diff --git a/db/mongo_manager.py b/db/mongo_manager.py index 999d3f5..434c538 100755 --- a/db/mongo_manager.py +++ b/db/mongo_manager.py @@ -152,7 +152,7 @@ class MongoDbManager: return link_list def link_validated_for_result(self, link: str, linkPojo: LinkPojo, state=True, is_duplicated=False, - is_invalid=False, segement_position=1): + is_invalid=False, segement_position=1, ua=""): print("link_validated_for_result() called with url = " + link) if is_duplicated: _id = link.split("/")[-2] @@ -174,7 +174,7 @@ class MongoDbManager: validated_by = "Double" collection.find_one_and_update({'_id': _id}, { "$set": {"url_validated": state, "validated_at": validated_at, "id": _id, "email": linkPojo.email, - "url": link, + "url": link, "validated_by_ua": ua, "validated_by": validated_by}}, upsert=True) # remove the link from db diff --git a/workers/link_validator.py b/workers/link_validator.py index 190a857..97ebbeb 100644 --- a/workers/link_validator.py +++ b/workers/link_validator.py @@ -80,7 +80,7 @@ class LinkValidator(threading.Thread): if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content: print(response.url) MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, - segement_position=self.segment_position) + segement_position=self.segment_position, ua=_ua) # set new cookies _cookies_to_set = response.headers['set-cookie'] self.cookie.load(_cookies_to_set) @@ -99,8 +99,7 @@ class LinkValidator(threading.Thread): self.cookie_str = new_coolies_str return RequestResult.SUCCESS elif INVALID in _content: - - MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True) + MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True, ua=_ua) # set new cookies _cookies_to_set = response.headers['set-cookie'] self.cookie.load(_cookies_to_set) @@ -116,7 +115,7 @@ class LinkValidator(threading.Thread): elif DOUBLE_MESSAGE in _content: print(response.url) - MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True) + MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True, ua=_ua) # set new cookies _cookies_to_set = response.headers['set-cookie'] self.cookie.load(_cookies_to_set) diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index 558ba71..233e834 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -83,7 +83,8 @@ class LinkValidatorWithProvidedList(threading.Thread): if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content: print(response.url) MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, - segement_position=threading.currentThread().name) + segement_position=threading.currentThread().name, + ua=_ua) # set new cookies _cookies_to_set = response.headers['set-cookie'] self.cookie.load(_cookies_to_set) @@ -103,7 +104,7 @@ class LinkValidatorWithProvidedList(threading.Thread): return RequestResult.SUCCESS elif INVALID in _content: - MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True) + MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True, ua=_ua) # set new cookies _cookies_to_set = response.headers['set-cookie'] self.cookie.load(_cookies_to_set) @@ -119,7 +120,7 @@ class LinkValidatorWithProvidedList(threading.Thread): elif DOUBLE_MESSAGE in _content: print(response.url) - MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True) + MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True, ua=_ua) # set new cookies _cookies_to_set = response.headers['set-cookie'] self.cookie.load(_cookies_to_set)