diff --git a/db/mongo_manager.py b/db/mongo_manager.py index 37b1302..8c0fdbc 100755 --- a/db/mongo_manager.py +++ b/db/mongo_manager.py @@ -84,7 +84,7 @@ class MongoDbManager: return link_list def link_validated_for_result(self, link: str, linkPojo: LinkPojo, state=True, is_duplicated=False, - is_invalid=False): + is_invalid=False, segement_position=1): print("link_validated_for_result() called with url = " + link) if is_duplicated: _id = link.split("/")[-2] @@ -99,7 +99,7 @@ class MongoDbManager: collection = self.db[collection_name] validated_at = time.strftime("%H:%M:%S", time.localtime()) - validated_by = "requests" + validated_by = "requests" + str(segement_position) if is_invalid: validated_by = "Invalid" if is_duplicated: diff --git a/proxy_manager/proxy_manager.py b/proxy_manager/proxy_manager.py index b3fa51a..0ab6d56 100644 --- a/proxy_manager/proxy_manager.py +++ b/proxy_manager/proxy_manager.py @@ -22,8 +22,8 @@ FR_PROXY_ASOCK_RES_2 = { class ProxyManager: def get_link_validate_proxy(self, links_to_validate: list) -> list: if len(links_to_validate) > 15: - # return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY] - return [FR_PROXY_RES_OXY] + return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY] + # return [FR_PROXY_RES_OXY] else: - # return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY] - return [FR_PROXY_RES_OXY] + return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY] + # return [FR_PROXY_RES_OXY] diff --git a/workers/link_validator.py b/workers/link_validator.py index bf313dd..fbf00db 100644 --- a/workers/link_validator.py +++ b/workers/link_validator.py @@ -67,14 +67,15 @@ class LinkValidator(threading.Thread): print("send request for link: " + linkPojo.url) try: response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use, - timeout=30) + timeout=60) print(response.status_code) if response.status_code == 200: _content = response.text print(response.text) if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content: print(response.url) - MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo) + MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, + segement_position=self.segment_position) # set new cookies _cookies_to_set = response.headers['set-cookie'] self.cookie.load(_cookies_to_set) @@ -169,7 +170,7 @@ class LinkValidator(threading.Thread): break time.sleep(random.randint(2, 5)) print("can continue, continue") - if can_continue == RequestResult.BAD_GATEWAY or can_continue == RequestResult.PROXY_ERROR: + if can_continue == RequestResult.BAD_GATEWAY or can_continue == RequestResult.PROXY_ERROR or can_continue == RequestResult.SUCCESS: print("will requeue the message") ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True) else: @@ -189,8 +190,7 @@ class LinkValidator(threading.Thread): def validate_with_FR_ip(segment_position=1): - # _queue_name = REQUEST_DATA_QUEUE - _queue_name = REQUEST_DATA_DE + _queue_name = REQUEST_DATA_QUEUE cookiesPublisher = CookiesPublisher(queue_name=_queue_name) cookiesPublisher.set_up_connection() print("filter links with ip_country")