panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support data impulse

Browse Source
This commit is contained in:
PAN Lei
2024-05-10 17:40:33 +02:00
parent 20e6c0b5e0
commit 8f432e0859
5 changed files with 49 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
models/ReserveResultPojo.py
+2
View File
@@ -34,6 +34,7 @@ class ReserveResultPojo:
url_validated = None
created_at = None
validated_at = None
proxy: str = None
@staticmethod
def from_firestore_dict(source):
@@ -127,6 +128,7 @@ class ReserveResultPojo:
u'store_type': self.store_type,
u'accepted': self.accepted,
u'url_validated': self.url_validated,
u'proxy': self.proxy,
}
return dest
proxy_manager/proxy_manager.py
+33 -2
View File
@@ -1,3 +1,5 @@
import random
FR_ASOCKS_MOBILE_PROXY = {
'http': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046',
'https': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046',
@@ -17,13 +19,42 @@ FR_PROXY_ASOCK_RES_2 = {
'http': 'http://10488120-res-country-FR:k94fsbn9a@217.23.6.161:11287',
'https': 'http://10488120-res-country-FR:k94fsbn9a@217.23.6.161:11287'
}
FR_PROXY_MOB_OXY_STICKY = {
'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}',
'https': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}'
}
FR_PROXY_DATA_IMPULSE_STICKY = {
'http': 'http://1d568220dbefeff21ad4__cr.fr:df99489a25fa72d9@gw.dataimpulse.com:{}',
'https': 'http://1d568220dbefeff21ad4__cr.fr:df99489a25fa72d9@gw.dataimpulse.com:{}'
}
MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_DATA_IMPULSE_STICKY]
class ProxyManager:
def __init__(self, logger):
self.logger = logger
def get_link_validate_proxy(self, links_to_validate: list) -> list:
if len(links_to_validate) > 15:
return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY]
return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2]
# return [FR_PROXY_RES_OXY]
else:
return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY]
return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2]
# return [FR_PROXY_RES_OXY]
def get_proxy_for_appointment_request(self) -> dict:
_chosen_proxy = random.choice(MOBILE_PROXY_LIST)
if "oxylabs" in _chosen_proxy["http"]:
self.logger.info("use oxylabs proxy")
_port = random.randint(40001, 49999)
else:
self.logger.info("use dataimpulse proxy")
_port = random.randint(10000, 20000)
self.logger.info("generated port is {}".format(_port))
_proxy_to_use = {}
_proxy_to_use["http"] = _chosen_proxy["http"].format(_port)
_proxy_to_use["https"] = _chosen_proxy["https"].format(_port)
return _proxy_to_use
queue_message/appointmentrequestsender.py
+7 -11
View File
@@ -13,6 +13,7 @@ from models.contact_pojo import ContactPojo
from models.jsdata_le_pojo import JsDataLeTypePojo
from models.jsdata_pojo import JsDataPojo
from models.result_pojo import RequestResult
from proxy_manager.proxy_manager import ProxyManager
from queue_message.CookiesPublisher import CookiesPublisher
from utiles import is_time_between
from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
@@ -62,7 +63,7 @@ def is_open():
class AppointmentRequestSender(threading.Thread):
def __init__(self, sub_contact_list: list, proxy_to_use_list, logger, cookiesPublisher: CookiesPublisher,
def __init__(self, sub_contact_list: list, logger, cookiesPublisher: CookiesPublisher,
just_send=False,
queue_name=REQUEST_DATA_QUEUE):
super().__init__()
@@ -77,7 +78,7 @@ class AppointmentRequestSender(threading.Thread):
self.initial_contact_list = sub_contact_list
self.contact_list = sub_contact_list
self.queue_name = queue_name
self.proxy_to_use_list = proxy_to_use_list
self.proxy_manager = ProxyManager(logger)
self.already_read_emails = False
def set_up_connection(self):
@@ -96,13 +97,6 @@ class AppointmentRequestSender(threading.Thread):
self.logger.info("message count in queue is {}".format(_message_count))
_received_object = body.decode("UTF-8")
self.logger.info(f" [x] Received {_received_object}")
_port = random.randint(40001, 49999)
self.logger.info("generated port is {}".format(_port))
_chosen_proxy = random.choice(self.proxy_to_use_list)
_proxy_to_use = {}
_proxy_to_use["http"] = _chosen_proxy["http"].format(_port)
_proxy_to_use["https"] = _chosen_proxy["https"].format(_port)
print(_proxy_to_use)
if "glrd" in _received_object:
_received_dict = json.loads(_received_object)
js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'],
@@ -114,8 +108,6 @@ class AppointmentRequestSender(threading.Thread):
rs_h=_received_dict['rs_h'],
rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'])
_received_cookies = _received_dict["cookiesStr"]
sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict,
proxy_to_use=_proxy_to_use, logger=self.logger)
# remove already sent contacts
if self.just_send:
self.contact_list = filter_contacts(self.contact_list, self.already_tried_contact_list)
@@ -128,6 +120,10 @@ class AppointmentRequestSender(threading.Thread):
self.logger.info("contact number is {}".format(len(self.contact_list)))
# self.contact_list = filter_contacts(self.contact_list)
for con in self.contact_list:
_proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
print(_proxy_to_use)
sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict,
proxy_to_use=_proxy_to_use, logger=self.logger)
self.logger.info(con.mail)
if self.valid_csrf is None:
csrf_result = captchaResultGetter.get_csrf(
request_sender_test.py
+3 -3
View File
@@ -60,7 +60,7 @@ def send_appointment_request(message_queue_name, _contact_list):
logger.info(_contact)
_cookiesPublisher = CookiesPublisher(queue_name=message_queue_name)
_cookiesPublisher.set_up_connection()
receiver = AppointmentRequestSender(sub_contact_list=_contact_list, proxy_to_use_list=MOBILE_PROXY_LIST_FR,
receiver = AppointmentRequestSender(sub_contact_list=_contact_list,
queue_name=message_queue_name, just_send=True,
cookiesPublisher=_cookiesPublisher, logger=logger)
print("count is " + str(count))
@@ -79,11 +79,11 @@ def start_send_requests(thread_number, file_path):
logger.info("segment is {}".format(i))
_step = int(len(_contact_list_to_book) / _segment_number)
_sublist = _contact_list_to_book[i * _step:_step * (i + 1)]
_thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE, _sublist))
_thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE_2, _sublist))
last_thread = _thread1
_thread1.start()
last_thread.join()
if __name__ == '__main__':
start_send_requests(thread_number=25, file_path='~/Desktop/contact_list_2024-04-17.xlsx')
start_send_requests(thread_number=5, file_path='~/Desktop/contact_list_2024-04-24.xlsx')
workers/sender.py
+4
View File
@@ -36,6 +36,10 @@ class Sender:
result = ReserveResultPojo(type=status, phone=contact.phone, message=status.value, url=url,
firstName=contact.first_name, lastName=contact.last_name, email=contact.mail,
passport=contact.passport, ccid=contact.ccid)
if "oxylabs" in self.proxy_to_use["http"]:
result.proxy = "oxylabs"
else:
result.proxy = "data_impulse"
result.id = id
result.store_type = store_type
result.created_at = time.strftime("%H:%M:%S", time.localtime())