panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

save ua while validating links

Browse Source
This commit is contained in:
Lei PAN
2024-06-17 10:53:36 +02:00
parent 14707e12f4
commit b55dcbb814
3 changed files with 9 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/link_validator.py
+3 -4
View File
@@ -80,7 +80,7 @@ class LinkValidator(threading.Thread):
if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content:
print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo,
segement_position=self.segment_position)
segement_position=self.segment_position, ua=_ua)
# set new cookies
_cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set)
@@ -99,8 +99,7 @@ class LinkValidator(threading.Thread):
self.cookie_str = new_coolies_str
return RequestResult.SUCCESS
elif INVALID in _content:
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True, ua=_ua)
# set new cookies
_cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set)
@@ -116,7 +115,7 @@ class LinkValidator(threading.Thread):
elif DOUBLE_MESSAGE in _content:
print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True, ua=_ua)
# set new cookies
_cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set)
workers/link_validator_with_provided_list.py
+4 -3
View File
@@ -83,7 +83,8 @@ class LinkValidatorWithProvidedList(threading.Thread):
if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content:
print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo,
segement_position=threading.currentThread().name)
segement_position=threading.currentThread().name,
ua=_ua)
# set new cookies
_cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set)
@@ -103,7 +104,7 @@ class LinkValidatorWithProvidedList(threading.Thread):
return RequestResult.SUCCESS
elif INVALID in _content:
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True, ua=_ua)
# set new cookies
_cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set)
@@ -119,7 +120,7 @@ class LinkValidatorWithProvidedList(threading.Thread):
elif DOUBLE_MESSAGE in _content:
print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True, ua=_ua)
# set new cookies
_cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set)