add ProxyManager

link_validator_executor.py

@@ -1,50 +1,5 @@
-import random
-import time
-from concurrent.futures import as_completed
-from concurrent.futures.thread import ThreadPoolExecutor
-from http.cookies import SimpleCookie
-
-from db.mongo_manager import MONGO_STORE_MANAGER
-from excel_reader import read_contacts
-from models.contact_pojo import ContactPojo
-from queue_message.link_validator_receiver import LinkValidatorReceiver
-from queue_message.appointmentrequestsender import AppointmentRequestSender, filter_contacts
-from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
-from workers.sender import Sender
-
-IPFIY = 'http://api.ipify.org'
-NGROK_TEST = "https://bcc6-193-164-156-53.ngrok-free.app"
-
-# def handle_cookie(cookie: str):
-#     contact_list = read_contacts('/Users/panlei/Desktop/yahoo_aol_valid_26-1.xlsx')
-#     # remove already booked contacts
-#     sub_contact_list = filter_contacts(contact_list)
-#     random.shuffle(sub_contact_list)
-#     # all_task = []
-#     # executor = ThreadPoolExecutor(max_workers=2)
-#     # urls = [3, 2, 4] # 并不是真的url
-#     # all_task = [executor.submit(delegate_sender, (url)) for con in sub_contact_list]
-#     with ThreadPoolExecutor(max_workers=2) as executor:
-#         for con in sub_contact_list:
-#             sender = Sender(cookie)
-#             _can_continue = executor.submit(delegate_sender, sender, con)
-#             print("_can_continue is {}".format(_can_continue))
-#             if not _can_continue:
-#                 break
-#     print("fin")
-#     # for future in as_completed(all_task):
-#     #     data = future.result()
-#     # print("in main: get page {}s success".format(data))
-
+from workers.link_validator import validate_with_FR_ip
 
 if __name__ == '__main__':
     # link_list = MONGO_STORE_MANAGER.get_links_to_validate()
-    # # remove already booked contacts
-    # random.shuffle(link_list)
-    # for l in link_list:
-    #     print(l)
-
-    # contact_list = read_contacts('/Users/panlei/Desktop/real_name_contacts_13_01_2024.xlsx')
-    # contact_list = filter_contacts(contact_list)
-    receiver = LinkValidatorReceiver()
-    receiver.run()
+    validate_with_FR_ip()

proxy_manager/proxy_manager.py

@@ -0,0 +1,9 @@
+FR_ASOCKS_MOBILE_PROXY = {
+    'http': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046',
+    'https': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046',
+}
+
+
+class ProxyManager:
+    def get_link_validate_proxy(self) -> list:
+        return [FR_ASOCKS_MOBILE_PROXY]

workers/link_validator.py

@@ -9,6 +9,7 @@ import requests
 from db.mongo_manager import MONGO_STORE_MANAGER
 from models.LinkPojo import LinkPojo
 from models.result_pojo import RequestResult
+from proxy_manager.proxy_manager import ProxyManager
 from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST
 from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
 from workers.proxies_constants import PROXY_LIST_FR
@@ -19,7 +20,7 @@ DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ce
 class LinkValidator(threading.Thread):
 
     def __init__(self, link_to_validate_list: list,
-                 cookiesPublisher: CookiesPublisher, proxy_to_use, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR",
+                 cookiesPublisher: CookiesPublisher, proxy_manager:ProxyManager, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR",
                  limit=40):
         super().__init__()
         self.cookie = SimpleCookie()
@@ -28,7 +29,7 @@ class LinkValidator(threading.Thread):
         self.queue_to_listen = queue_to_listen
         self.ip_country = ip_country
         self.filter_with_ip_country()
-        self.proxy_to_use = proxy_to_use
+        self.proxy_manager = proxy_manager
         self.limit = limit
         # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
 
@@ -56,11 +57,12 @@ class LinkValidator(threading.Thread):
             'Sec-Fetch-Site': 'same-origin',
             'Sec-Fetch-Dest': 'document',
             'Accept-Language': 'fr-FR,fr;q=0.6'}
-        print(self.proxy_to_use)
+        _proxy_to_use = random.choice(self.proxy_manager.get_link_validate_proxy())
+        print(_proxy_to_use)
         print("received cookie is " + str(self.cookie_str))
         try:
-            response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
-                                    timeout=15)
+            response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use,
+                                    timeout=10)
             print(response.status_code)
             if response.status_code == 200:
                 _content = response.text
@@ -151,10 +153,11 @@ def validate_with_FR_ip():
             _link_list_to_click.append(_link)
     for _l in _link_list_to_click:
         print(_l.ip_country)
-    _fr_proxy_to_use = random.choice(PROXY_LIST_FR)
+    _proxy_manager = ProxyManager()
+    # _fr_proxy_to_use = random.choice(PROXY_LIST_FR)
     # random.shuffle(_link_list_to_click)
     receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
-                             proxy_to_use=_fr_proxy_to_use,
+                             proxy_manager=_proxy_manager,
                              queue_to_listen=_queue_name, ip_country="FR", limit=50)
     print("will connect to queue")
     receiver.set_up_connection()