add ProxyManager

workers/link_validator.py

@@ -9,6 +9,7 @@ import requests
 from db.mongo_manager import MONGO_STORE_MANAGER
 from models.LinkPojo import LinkPojo
 from models.result_pojo import RequestResult
+from proxy_manager.proxy_manager import ProxyManager
 from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST
 from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
 from workers.proxies_constants import PROXY_LIST_FR
@@ -19,7 +20,7 @@ DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ce
 class LinkValidator(threading.Thread):
 
     def __init__(self, link_to_validate_list: list,
-                 cookiesPublisher: CookiesPublisher, proxy_to_use, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR",
+                 cookiesPublisher: CookiesPublisher, proxy_manager:ProxyManager, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR",
                  limit=40):
         super().__init__()
         self.cookie = SimpleCookie()
@@ -28,7 +29,7 @@ class LinkValidator(threading.Thread):
         self.queue_to_listen = queue_to_listen
         self.ip_country = ip_country
         self.filter_with_ip_country()
-        self.proxy_to_use = proxy_to_use
+        self.proxy_manager = proxy_manager
         self.limit = limit
         # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
 
@@ -56,11 +57,12 @@ class LinkValidator(threading.Thread):
             'Sec-Fetch-Site': 'same-origin',
             'Sec-Fetch-Dest': 'document',
             'Accept-Language': 'fr-FR,fr;q=0.6'}
-        print(self.proxy_to_use)
+        _proxy_to_use = random.choice(self.proxy_manager.get_link_validate_proxy())
+        print(_proxy_to_use)
         print("received cookie is " + str(self.cookie_str))
         try:
-            response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
-                                    timeout=15)
+            response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use,
+                                    timeout=10)
             print(response.status_code)
             if response.status_code == 200:
                 _content = response.text
@@ -151,10 +153,11 @@ def validate_with_FR_ip():
             _link_list_to_click.append(_link)
     for _l in _link_list_to_click:
         print(_l.ip_country)
-    _fr_proxy_to_use = random.choice(PROXY_LIST_FR)
+    _proxy_manager = ProxyManager()
+    # _fr_proxy_to_use = random.choice(PROXY_LIST_FR)
     # random.shuffle(_link_list_to_click)
     receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
-                             proxy_to_use=_fr_proxy_to_use,
+                             proxy_manager=_proxy_manager,
                              queue_to_listen=_queue_name, ip_country="FR", limit=50)
     print("will connect to queue")
     receiver.set_up_connection()