support 502 error

workers/link_validator.py

@@ -19,13 +19,16 @@ DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ce
 
 class LinkValidator(threading.Thread):
 
-    def __init__(self, link_to_validate_list: list,
-                 cookiesPublisher: CookiesPublisher, proxy_manager:ProxyManager, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR",
+    def __init__(self, cookiesPublisher: CookiesPublisher, proxy_manager: ProxyManager,
+                 queue_to_listen=REQUEST_DATA_QUEUE,
+                 ip_country="FR", segment_position=1,
                  limit=40):
         super().__init__()
+        self.link_to_validate_list = []
         self.cookie = SimpleCookie()
         self.cookiesPublisher = cookiesPublisher
-        self.link_to_validate_list = link_to_validate_list
+        self.segment_position = segment_position
+        self.update_validate_list()
         self.queue_to_listen = queue_to_listen
         self.ip_country = ip_country
         self.filter_with_ip_country()
@@ -62,7 +65,7 @@ class LinkValidator(threading.Thread):
         print("received cookie is " + str(self.cookie_str))
         try:
             response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use,
-                                    timeout=10)
+                                    timeout=30)
             print(response.status_code)
             if response.status_code == 200:
                 _content = response.text
@@ -99,17 +102,35 @@ class LinkValidator(threading.Thread):
                     return RequestResult.SUCCESS
                 else:
                     return RequestResult.UNKNOWN
+            elif response.status_code == 502:
+                return RequestResult.PROXY_ERROR
             else:
                 return RequestResult.BLOCKED
         except Exception as error:
             print(error)
             return RequestResult.PROXY_ERROR
 
+    def update_validate_list(self):
+        # for the moment, max segment is 2
+        _all_list = MONGO_STORE_MANAGER.get_links_to_validate()
+        list_length = len(_all_list)
+        middle = int(len(_all_list) / 2)
+        if self.segment_position == 1:
+            if list_length > 1:
+                self.link_to_validate_list = _all_list[0:middle]
+            else:
+                self.link_to_validate_list = _all_list
+        else:
+            if list_length > 1:
+                self.link_to_validate_list = _all_list[middle:]
+            else:
+                self.link_to_validate_list = []
+
     def on_message(self, ch, method, properties, body):
         print(f" [x] Received {body}")
         _message_in_queue_count = self.cookiesPublisher.message_count()
         print("message count in queue is {}".format(_message_in_queue_count))
-        self.link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
+        self.update_validate_list()
         self.filter_with_ip_country()
         self.cookie_str = body.decode("UTF-8")
         random.shuffle(self.link_to_validate_list)
@@ -141,24 +162,15 @@ class LinkValidator(threading.Thread):
         self.link_to_validate_list = _link_list_to_click
 
 
-def validate_with_FR_ip():
+def validate_with_FR_ip(segment_position=1):
     _queue_name = REQUEST_DATA_QUEUE
-    link_list = MONGO_STORE_MANAGER.get_links_to_validate()
     cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
     cookiesPublisher.set_up_connection()
     print("filter links with ip_country")
-    _link_list_to_click = []
-    for _link in link_list:
-        if _link.ip_country == "FR":
-            _link_list_to_click.append(_link)
-    for _l in _link_list_to_click:
-        print(_l.ip_country)
     _proxy_manager = ProxyManager()
-    # _fr_proxy_to_use = random.choice(PROXY_LIST_FR)
-    # random.shuffle(_link_list_to_click)
-    receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
+    receiver = LinkValidator(cookiesPublisher=cookiesPublisher,
                              proxy_manager=_proxy_manager,
-                             queue_to_listen=_queue_name, ip_country="FR", limit=50)
+                             queue_to_listen=_queue_name, ip_country="FR", segment_position=segment_position, limit=50)
     print("will connect to queue")
     receiver.set_up_connection()
     receiver.listen_to_queue(receiver.on_message)

workers/validate_links_DE.py

@@ -1,33 +0,0 @@
-import random
-
-from db.mongo_manager import MONGO_STORE_MANAGER
-from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher
-from workers.link_validator import LinkValidator
-from workers.proxies_constants import PROXY_LIST_DE
-
-
-def validate_with_DE_ip():
-    _queue_name = REQUEST_DATA_QUEUE_DE
-    link_list = MONGO_STORE_MANAGER.get_links_to_validate()
-    cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
-    cookiesPublisher.set_up_connection()
-    print("filter links with ip_country")
-    _link_list_to_click = []
-    for _link in link_list:
-        if _link.ip_country == "DE":
-            _link_list_to_click.append(_link)
-    for _l in _link_list_to_click:
-        print(_l.ip_country)
-        # else:
-        #     print(_link.ip_country)
-    _de_proxy_to_use = random.choice(PROXY_LIST_DE)
-    receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
-                             proxy_to_use=_de_proxy_to_use,
-                             queue_to_listen=_queue_name, ip_country="DE", limit=0)
-    receiver.set_up_connection()
-    receiver.listen_to_queue(receiver.on_message)
-    pass
-
-
-if __name__ == '__main__':
-    validate_with_DE_ip()