panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

save ua while validating links

Browse Source
This commit is contained in:
Lei PAN
2024-06-17 10:53:36 +02:00
parent 14707e12f4
commit b55dcbb814
3 changed files with 9 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/mongo_manager.py
+2 -2
View File
@@ -152,7 +152,7 @@ class MongoDbManager:
return link_list return link_list
def link_validated_for_result(self, link: str, linkPojo: LinkPojo, state=True, is_duplicated=False, def link_validated_for_result(self, link: str, linkPojo: LinkPojo, state=True, is_duplicated=False,
is_invalid=False, segement_position=1): is_invalid=False, segement_position=1, ua=""):
print("link_validated_for_result() called with url = " + link) print("link_validated_for_result() called with url = " + link)
if is_duplicated: if is_duplicated:
_id = link.split("/")[-2] _id = link.split("/")[-2]
@@ -174,7 +174,7 @@ class MongoDbManager:
validated_by = "Double" validated_by = "Double"
collection.find_one_and_update({'_id': _id}, { collection.find_one_and_update({'_id': _id}, {
"$set": {"url_validated": state, "validated_at": validated_at, "id": _id, "email": linkPojo.email, "$set": {"url_validated": state, "validated_at": validated_at, "id": _id, "email": linkPojo.email,
"url": link, "url": link, "validated_by_ua": ua,
"validated_by": validated_by}}, "validated_by": validated_by}},
upsert=True) upsert=True)
# remove the link from db # remove the link from db
workers/link_validator.py
+3 -4
View File
@@ -80,7 +80,7 @@ class LinkValidator(threading.Thread):
if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content: if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content:
print(response.url) print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo,
segement_position=self.segment_position) segement_position=self.segment_position, ua=_ua)
# set new cookies # set new cookies
_cookies_to_set = response.headers['set-cookie'] _cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set) self.cookie.load(_cookies_to_set)
@@ -99,8 +99,7 @@ class LinkValidator(threading.Thread):
self.cookie_str = new_coolies_str self.cookie_str = new_coolies_str
return RequestResult.SUCCESS return RequestResult.SUCCESS
elif INVALID in _content: elif INVALID in _content:
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True, ua=_ua)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True)
# set new cookies # set new cookies
_cookies_to_set = response.headers['set-cookie'] _cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set) self.cookie.load(_cookies_to_set)
@@ -116,7 +115,7 @@ class LinkValidator(threading.Thread):
elif DOUBLE_MESSAGE in _content: elif DOUBLE_MESSAGE in _content:
print(response.url) print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True) MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True, ua=_ua)
# set new cookies # set new cookies
_cookies_to_set = response.headers['set-cookie'] _cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set) self.cookie.load(_cookies_to_set)
workers/link_validator_with_provided_list.py
+4 -3
View File
@@ -83,7 +83,8 @@ class LinkValidatorWithProvidedList(threading.Thread):
if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content: if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content:
print(response.url) print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo,
segement_position=threading.currentThread().name) segement_position=threading.currentThread().name,
ua=_ua)
# set new cookies # set new cookies
_cookies_to_set = response.headers['set-cookie'] _cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set) self.cookie.load(_cookies_to_set)
@@ -103,7 +104,7 @@ class LinkValidatorWithProvidedList(threading.Thread):
return RequestResult.SUCCESS return RequestResult.SUCCESS
elif INVALID in _content: elif INVALID in _content:
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True) MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True, ua=_ua)
# set new cookies # set new cookies
_cookies_to_set = response.headers['set-cookie'] _cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set) self.cookie.load(_cookies_to_set)
@@ -119,7 +120,7 @@ class LinkValidatorWithProvidedList(threading.Thread):
elif DOUBLE_MESSAGE in _content: elif DOUBLE_MESSAGE in _content:
print(response.url) print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True) MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True, ua=_ua)
# set new cookies # set new cookies
_cookies_to_set = response.headers['set-cookie'] _cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set) self.cookie.load(_cookies_to_set)