support segment

workers/link_validator.py

@@ -67,14 +67,15 @@ class LinkValidator(threading.Thread):
         print("send request for link: " + linkPojo.url)
         try:
             response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use,
-                                    timeout=30)
+                                    timeout=60)
             print(response.status_code)
             if response.status_code == 200:
                 _content = response.text
                 print(response.text)
                 if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content:
                     print(response.url)
-                    MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo)
+                    MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo,
+                                                                  segement_position=self.segment_position)
                     # set new cookies
                     _cookies_to_set = response.headers['set-cookie']
                     self.cookie.load(_cookies_to_set)
@@ -169,7 +170,7 @@ class LinkValidator(threading.Thread):
                         break
                     time.sleep(random.randint(2, 5))
                     print("can continue, continue")
-            if can_continue == RequestResult.BAD_GATEWAY or can_continue == RequestResult.PROXY_ERROR:
+            if can_continue == RequestResult.BAD_GATEWAY or can_continue == RequestResult.PROXY_ERROR or can_continue == RequestResult.SUCCESS:
                 print("will requeue the message")
                 ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
             else:
@@ -189,8 +190,7 @@ class LinkValidator(threading.Thread):
 
 
 def validate_with_FR_ip(segment_position=1):
-    # _queue_name = REQUEST_DATA_QUEUE
-    _queue_name = REQUEST_DATA_DE
+    _queue_name = REQUEST_DATA_QUEUE
     cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
     cookiesPublisher.set_up_connection()
     print("filter links with ip_country")