support data impulse

models/ReserveResultPojo.py

@@ -34,6 +34,7 @@ class ReserveResultPojo:
     url_validated = None
     created_at = None
     validated_at = None
+    proxy: str = None
 
     @staticmethod
     def from_firestore_dict(source):
@@ -127,6 +128,7 @@ class ReserveResultPojo:
             u'store_type': self.store_type,
             u'accepted': self.accepted,
             u'url_validated': self.url_validated,
+            u'proxy': self.proxy,
         }
 
         return dest

proxy_manager/proxy_manager.py

@@ -1,3 +1,5 @@
+import random
+
 FR_ASOCKS_MOBILE_PROXY = {
     'http': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046',
     'https': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046',
@@ -17,13 +19,42 @@ FR_PROXY_ASOCK_RES_2 = {
     'http': 'http://10488120-res-country-FR:k94fsbn9a@217.23.6.161:11287',
     'https': 'http://10488120-res-country-FR:k94fsbn9a@217.23.6.161:11287'
 }
+FR_PROXY_MOB_OXY_STICKY = {
+    'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}',
+    'https': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}'
+}
+
+FR_PROXY_DATA_IMPULSE_STICKY = {
+    'http': 'http://1d568220dbefeff21ad4__cr.fr:df99489a25fa72d9@gw.dataimpulse.com:{}',
+    'https': 'http://1d568220dbefeff21ad4__cr.fr:df99489a25fa72d9@gw.dataimpulse.com:{}'
+}
+
+MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_DATA_IMPULSE_STICKY]
 
 
 class ProxyManager:
+
+    def __init__(self, logger):
+        self.logger = logger
+
     def get_link_validate_proxy(self, links_to_validate: list) -> list:
         if len(links_to_validate) > 15:
-            return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY]
+            return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2]
             # return [FR_PROXY_RES_OXY]
         else:
-            return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY]
+            return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2]
             # return [FR_PROXY_RES_OXY]
+
+    def get_proxy_for_appointment_request(self) -> dict:
+        _chosen_proxy = random.choice(MOBILE_PROXY_LIST)
+        if "oxylabs" in _chosen_proxy["http"]:
+            self.logger.info("use oxylabs proxy")
+            _port = random.randint(40001, 49999)
+        else:
+            self.logger.info("use dataimpulse proxy")
+            _port = random.randint(10000, 20000)
+        self.logger.info("generated port is {}".format(_port))
+        _proxy_to_use = {}
+        _proxy_to_use["http"] = _chosen_proxy["http"].format(_port)
+        _proxy_to_use["https"] = _chosen_proxy["https"].format(_port)
+        return _proxy_to_use

queue_message/appointmentrequestsender.py

@@ -13,6 +13,7 @@ from models.contact_pojo import ContactPojo
 from models.jsdata_le_pojo import JsDataLeTypePojo
 from models.jsdata_pojo import JsDataPojo
 from models.result_pojo import RequestResult
+from proxy_manager.proxy_manager import ProxyManager
 from queue_message.CookiesPublisher import CookiesPublisher
 from utiles import is_time_between
 from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
@@ -62,7 +63,7 @@ def is_open():
 
 
 class AppointmentRequestSender(threading.Thread):
-    def __init__(self, sub_contact_list: list, proxy_to_use_list, logger, cookiesPublisher: CookiesPublisher,
+    def __init__(self, sub_contact_list: list, logger, cookiesPublisher: CookiesPublisher,
                  just_send=False,
                  queue_name=REQUEST_DATA_QUEUE):
         super().__init__()
@@ -77,7 +78,7 @@ class AppointmentRequestSender(threading.Thread):
         self.initial_contact_list = sub_contact_list
         self.contact_list = sub_contact_list
         self.queue_name = queue_name
-        self.proxy_to_use_list = proxy_to_use_list
+        self.proxy_manager = ProxyManager(logger)
         self.already_read_emails = False
 
     def set_up_connection(self):
@@ -96,13 +97,6 @@ class AppointmentRequestSender(threading.Thread):
         self.logger.info("message count in queue is {}".format(_message_count))
         _received_object = body.decode("UTF-8")
         self.logger.info(f" [x] Received {_received_object}")
-        _port = random.randint(40001, 49999)
-        self.logger.info("generated port is {}".format(_port))
-        _chosen_proxy = random.choice(self.proxy_to_use_list)
-        _proxy_to_use = {}
-        _proxy_to_use["http"] = _chosen_proxy["http"].format(_port)
-        _proxy_to_use["https"] = _chosen_proxy["https"].format(_port)
-        print(_proxy_to_use)
         if "glrd" in _received_object:
             _received_dict = json.loads(_received_object)
             js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'],
@@ -114,8 +108,6 @@ class AppointmentRequestSender(threading.Thread):
                                  rs_h=_received_dict['rs_h'],
                                  rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'])
             _received_cookies = _received_dict["cookiesStr"]
-            sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict,
-                            proxy_to_use=_proxy_to_use, logger=self.logger)
             # remove already sent contacts
             if self.just_send:
                 self.contact_list = filter_contacts(self.contact_list, self.already_tried_contact_list)
@@ -128,6 +120,10 @@ class AppointmentRequestSender(threading.Thread):
                 self.logger.info("contact number is {}".format(len(self.contact_list)))
                 # self.contact_list = filter_contacts(self.contact_list)
                 for con in self.contact_list:
+                    _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
+                    print(_proxy_to_use)
+                    sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict,
+                                    proxy_to_use=_proxy_to_use, logger=self.logger)
                     self.logger.info(con.mail)
                     if self.valid_csrf is None:
                         csrf_result = captchaResultGetter.get_csrf(

request_sender_test.py

@@ -60,7 +60,7 @@ def send_appointment_request(message_queue_name, _contact_list):
         logger.info(_contact)
     _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name)
     _cookiesPublisher.set_up_connection()
-    receiver = AppointmentRequestSender(sub_contact_list=_contact_list, proxy_to_use_list=MOBILE_PROXY_LIST_FR,
+    receiver = AppointmentRequestSender(sub_contact_list=_contact_list,
                                         queue_name=message_queue_name, just_send=True,
                                         cookiesPublisher=_cookiesPublisher, logger=logger)
     print("count is " + str(count))
@@ -79,11 +79,11 @@ def start_send_requests(thread_number, file_path):
         logger.info("segment is {}".format(i))
         _step = int(len(_contact_list_to_book) / _segment_number)
         _sublist = _contact_list_to_book[i * _step:_step * (i + 1)]
-        _thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE, _sublist))
+        _thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE_2, _sublist))
         last_thread = _thread1
         _thread1.start()
     last_thread.join()
 
 
 if __name__ == '__main__':
-    start_send_requests(thread_number=25, file_path='~/Desktop/contact_list_2024-04-17.xlsx')
+    start_send_requests(thread_number=5, file_path='~/Desktop/contact_list_2024-04-24.xlsx')

workers/sender.py

@@ -36,6 +36,10 @@ class Sender:
         result = ReserveResultPojo(type=status, phone=contact.phone, message=status.value, url=url,
                                    firstName=contact.first_name, lastName=contact.last_name, email=contact.mail,
                                    passport=contact.passport, ccid=contact.ccid)
+        if "oxylabs" in self.proxy_to_use["http"]:
+            result.proxy = "oxylabs"
+        else:
+            result.proxy = "data_impulse"
         result.id = id
         result.store_type = store_type
         result.created_at = time.strftime("%H:%M:%S", time.localtime())