From 8f432e08592bd853a6f46adf2fd2cb2183fa0caf Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 10 May 2024 17:40:33 +0200 Subject: [PATCH] support data impulse --- models/ReserveResultPojo.py | 2 ++ proxy_manager/proxy_manager.py | 35 +++++++++++++++++++++-- queue_message/appointmentrequestsender.py | 18 +++++------- request_sender_test.py | 6 ++-- workers/sender.py | 4 +++ 5 files changed, 49 insertions(+), 16 deletions(-) diff --git a/models/ReserveResultPojo.py b/models/ReserveResultPojo.py index 0d41ae1..2cf6892 100755 --- a/models/ReserveResultPojo.py +++ b/models/ReserveResultPojo.py @@ -34,6 +34,7 @@ class ReserveResultPojo: url_validated = None created_at = None validated_at = None + proxy: str = None @staticmethod def from_firestore_dict(source): @@ -127,6 +128,7 @@ class ReserveResultPojo: u'store_type': self.store_type, u'accepted': self.accepted, u'url_validated': self.url_validated, + u'proxy': self.proxy, } return dest diff --git a/proxy_manager/proxy_manager.py b/proxy_manager/proxy_manager.py index 0ab6d56..589c2c0 100644 --- a/proxy_manager/proxy_manager.py +++ b/proxy_manager/proxy_manager.py @@ -1,3 +1,5 @@ +import random + FR_ASOCKS_MOBILE_PROXY = { 'http': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046', 'https': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046', @@ -17,13 +19,42 @@ FR_PROXY_ASOCK_RES_2 = { 'http': 'http://10488120-res-country-FR:k94fsbn9a@217.23.6.161:11287', 'https': 'http://10488120-res-country-FR:k94fsbn9a@217.23.6.161:11287' } +FR_PROXY_MOB_OXY_STICKY = { + 'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}', + 'https': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}' +} + +FR_PROXY_DATA_IMPULSE_STICKY = { + 'http': 'http://1d568220dbefeff21ad4__cr.fr:df99489a25fa72d9@gw.dataimpulse.com:{}', + 'https': 'http://1d568220dbefeff21ad4__cr.fr:df99489a25fa72d9@gw.dataimpulse.com:{}' +} + +MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_DATA_IMPULSE_STICKY] class ProxyManager: + + def __init__(self, logger): + self.logger = logger + def get_link_validate_proxy(self, links_to_validate: list) -> list: if len(links_to_validate) > 15: - return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY] + return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2] # return [FR_PROXY_RES_OXY] else: - return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY] + return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2] # return [FR_PROXY_RES_OXY] + + def get_proxy_for_appointment_request(self) -> dict: + _chosen_proxy = random.choice(MOBILE_PROXY_LIST) + if "oxylabs" in _chosen_proxy["http"]: + self.logger.info("use oxylabs proxy") + _port = random.randint(40001, 49999) + else: + self.logger.info("use dataimpulse proxy") + _port = random.randint(10000, 20000) + self.logger.info("generated port is {}".format(_port)) + _proxy_to_use = {} + _proxy_to_use["http"] = _chosen_proxy["http"].format(_port) + _proxy_to_use["https"] = _chosen_proxy["https"].format(_port) + return _proxy_to_use diff --git a/queue_message/appointmentrequestsender.py b/queue_message/appointmentrequestsender.py index 2b4aadc..5ddb2d9 100644 --- a/queue_message/appointmentrequestsender.py +++ b/queue_message/appointmentrequestsender.py @@ -13,6 +13,7 @@ from models.contact_pojo import ContactPojo from models.jsdata_le_pojo import JsDataLeTypePojo from models.jsdata_pojo import JsDataPojo from models.result_pojo import RequestResult +from proxy_manager.proxy_manager import ProxyManager from queue_message.CookiesPublisher import CookiesPublisher from utiles import is_time_between from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER @@ -62,7 +63,7 @@ def is_open(): class AppointmentRequestSender(threading.Thread): - def __init__(self, sub_contact_list: list, proxy_to_use_list, logger, cookiesPublisher: CookiesPublisher, + def __init__(self, sub_contact_list: list, logger, cookiesPublisher: CookiesPublisher, just_send=False, queue_name=REQUEST_DATA_QUEUE): super().__init__() @@ -77,7 +78,7 @@ class AppointmentRequestSender(threading.Thread): self.initial_contact_list = sub_contact_list self.contact_list = sub_contact_list self.queue_name = queue_name - self.proxy_to_use_list = proxy_to_use_list + self.proxy_manager = ProxyManager(logger) self.already_read_emails = False def set_up_connection(self): @@ -96,13 +97,6 @@ class AppointmentRequestSender(threading.Thread): self.logger.info("message count in queue is {}".format(_message_count)) _received_object = body.decode("UTF-8") self.logger.info(f" [x] Received {_received_object}") - _port = random.randint(40001, 49999) - self.logger.info("generated port is {}".format(_port)) - _chosen_proxy = random.choice(self.proxy_to_use_list) - _proxy_to_use = {} - _proxy_to_use["http"] = _chosen_proxy["http"].format(_port) - _proxy_to_use["https"] = _chosen_proxy["https"].format(_port) - print(_proxy_to_use) if "glrd" in _received_object: _received_dict = json.loads(_received_object) js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'], @@ -114,8 +108,6 @@ class AppointmentRequestSender(threading.Thread): rs_h=_received_dict['rs_h'], rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd']) _received_cookies = _received_dict["cookiesStr"] - sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict, - proxy_to_use=_proxy_to_use, logger=self.logger) # remove already sent contacts if self.just_send: self.contact_list = filter_contacts(self.contact_list, self.already_tried_contact_list) @@ -128,6 +120,10 @@ class AppointmentRequestSender(threading.Thread): self.logger.info("contact number is {}".format(len(self.contact_list))) # self.contact_list = filter_contacts(self.contact_list) for con in self.contact_list: + _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() + print(_proxy_to_use) + sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict, + proxy_to_use=_proxy_to_use, logger=self.logger) self.logger.info(con.mail) if self.valid_csrf is None: csrf_result = captchaResultGetter.get_csrf( diff --git a/request_sender_test.py b/request_sender_test.py index 05d19a1..95ee5df 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -60,7 +60,7 @@ def send_appointment_request(message_queue_name, _contact_list): logger.info(_contact) _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name) _cookiesPublisher.set_up_connection() - receiver = AppointmentRequestSender(sub_contact_list=_contact_list, proxy_to_use_list=MOBILE_PROXY_LIST_FR, + receiver = AppointmentRequestSender(sub_contact_list=_contact_list, queue_name=message_queue_name, just_send=True, cookiesPublisher=_cookiesPublisher, logger=logger) print("count is " + str(count)) @@ -79,11 +79,11 @@ def start_send_requests(thread_number, file_path): logger.info("segment is {}".format(i)) _step = int(len(_contact_list_to_book) / _segment_number) _sublist = _contact_list_to_book[i * _step:_step * (i + 1)] - _thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE, _sublist)) + _thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE_2, _sublist)) last_thread = _thread1 _thread1.start() last_thread.join() if __name__ == '__main__': - start_send_requests(thread_number=25, file_path='~/Desktop/contact_list_2024-04-17.xlsx') + start_send_requests(thread_number=5, file_path='~/Desktop/contact_list_2024-04-24.xlsx') diff --git a/workers/sender.py b/workers/sender.py index 6ea2316..f35b034 100644 --- a/workers/sender.py +++ b/workers/sender.py @@ -36,6 +36,10 @@ class Sender: result = ReserveResultPojo(type=status, phone=contact.phone, message=status.value, url=url, firstName=contact.first_name, lastName=contact.last_name, email=contact.mail, passport=contact.passport, ccid=contact.ccid) + if "oxylabs" in self.proxy_to_use["http"]: + result.proxy = "oxylabs" + else: + result.proxy = "data_impulse" result.id = id result.store_type = store_type result.created_at = time.strftime("%H:%M:%S", time.localtime())