support segment

db/mongo_manager.py

@@ -84,7 +84,7 @@ class MongoDbManager:
         return link_list
 
     def link_validated_for_result(self, link: str, linkPojo: LinkPojo, state=True, is_duplicated=False,
-                                  is_invalid=False):
+                                  is_invalid=False, segement_position=1):
         print("link_validated_for_result() called with url = " + link)
         if is_duplicated:
             _id = link.split("/")[-2]
@@ -99,7 +99,7 @@ class MongoDbManager:
 
         collection = self.db[collection_name]
         validated_at = time.strftime("%H:%M:%S", time.localtime())
-        validated_by = "requests"
+        validated_by = "requests" + str(segement_position)
         if is_invalid:
             validated_by = "Invalid"
         if is_duplicated:

proxy_manager/proxy_manager.py

@@ -22,8 +22,8 @@ FR_PROXY_ASOCK_RES_2 = {
 class ProxyManager:
     def get_link_validate_proxy(self, links_to_validate: list) -> list:
         if len(links_to_validate) > 15:
-            # return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY]
-            return [FR_PROXY_RES_OXY]
+            return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY]
+            # return [FR_PROXY_RES_OXY]
         else:
-            # return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY]
-            return [FR_PROXY_RES_OXY]
+            return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY]
+            # return [FR_PROXY_RES_OXY]

workers/link_validator.py

@@ -67,14 +67,15 @@ class LinkValidator(threading.Thread):
         print("send request for link: " + linkPojo.url)
         try:
             response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use,
-                                    timeout=30)
+                                    timeout=60)
             print(response.status_code)
             if response.status_code == 200:
                 _content = response.text
                 print(response.text)
                 if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content:
                     print(response.url)
-                    MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo)
+                    MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo,
+                                                                  segement_position=self.segment_position)
                     # set new cookies
                     _cookies_to_set = response.headers['set-cookie']
                     self.cookie.load(_cookies_to_set)
@@ -169,7 +170,7 @@ class LinkValidator(threading.Thread):
                         break
                     time.sleep(random.randint(2, 5))
                     print("can continue, continue")
-            if can_continue == RequestResult.BAD_GATEWAY or can_continue == RequestResult.PROXY_ERROR:
+            if can_continue == RequestResult.BAD_GATEWAY or can_continue == RequestResult.PROXY_ERROR or can_continue == RequestResult.SUCCESS:
                 print("will requeue the message")
                 ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
             else:
@@ -189,8 +190,7 @@ class LinkValidator(threading.Thread):
 
 
 def validate_with_FR_ip(segment_position=1):
-    # _queue_name = REQUEST_DATA_QUEUE
-    _queue_name = REQUEST_DATA_DE
+    _queue_name = REQUEST_DATA_QUEUE
     cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
     cookiesPublisher.set_up_connection()
     print("filter links with ip_country")