diff --git a/docs/generate_jsdata.json b/docs/generate_jsdata.json new file mode 100644 index 0000000..27d1120 --- /dev/null +++ b/docs/generate_jsdata.json @@ -0,0 +1,127 @@ +{ + "opts": "endpoint,ajaxListenerPath", + "ttst": 93, + "ifov": false, + "hc": 2, + "br_oh": 663, + "br_ow": 384, + "ua": "Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 9.185432887936322, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 663, + "br_w": 384, + "isf": true, + "nddc": 1, + "rs_h": 811, + "rs_w": 384, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.8125, + "ars_h": 811, + "ars_w": 384, + "tz": -60, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "5gQIEKN,Iv26GLka", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 4, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzMDIpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWIgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTA2MjIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxNzMz", + "ckwa": true, + "glrd": "Adreno (TM) 630", + "glvd": "Qualcomm", + "wwl": false, + "jset": 1710598314 +} diff --git a/excel_reader.py b/excel_reader.py index 0f98b58..ad75e5b 100755 --- a/excel_reader.py +++ b/excel_reader.py @@ -30,6 +30,7 @@ def read_contacts(file_name) -> list: first_name=first_name, passport_number=contact_dict['passport'], mail=contact_dict['email']) + contact.store = contact_dict['store'] contact_list.append(contact) return contact_list diff --git a/models/contact_pojo.py b/models/contact_pojo.py index b841c20..54102f3 100755 --- a/models/contact_pojo.py +++ b/models/contact_pojo.py @@ -10,6 +10,7 @@ class ContactPojo: mail: str ccid: str position: int + store: str note: str def __init__(self, phone_number: str, passport_number: str, last_name: str, first_name: str, mail: str, @@ -31,6 +32,7 @@ class ContactPojo: u'last_name': self.last_name, u'first_name': self.first_name, u'mail': self.mail, + u'store': self.store, u'ccid': self.ccid, u'position': self.position } diff --git a/models/jsdata_pojo.py b/models/jsdata_pojo.py index 5e399f9..93f29f2 100644 --- a/models/jsdata_pojo.py +++ b/models/jsdata_pojo.py @@ -9,7 +9,7 @@ import jsonpickle @dataclass class JsDataPojo: - opts: str + # opts: str ttst: int ifov: bool hc: int @@ -137,7 +137,7 @@ class JsDataPojo: def __init__(self, glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu, vnd, pr, ts_mtp, dvm): - self.opts = "endpoint,ajaxListenerPath" + # self.opts = "endpoint,ajaxListenerPath" self.ttst = random.randint(38, 148) self.ifov = False self.hc = hc @@ -265,8 +265,8 @@ class JsDataPojo: self.emt = False self.bfr = False self.dbov = False - self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigzODYpXSgnXHg2Mlx4NzJceDZmXHg3N1x4NzNceDY1XHg3Mlx4NjZceDZjXHg2Zlx4NzdceDJkXHg2M1x4NmZceDZlXHg3NFx4NjFceDY5XHg2ZVx4NjVceDcyJyk7aWYobyl7IWZ1bmN0aW9uIHQoKXt2YXIg" # 4.24.3 - self.stcfp = "dy5oZXJtZXMuY29tL2ZyL2ZyL3BvbHlmaWxscy43ZTFjYzMyM2Y3OWMwNjZlLmpzOjE6ODI2MykKICAgIGF0IG0uYXJncy48Y29tcHV0ZWQ+IChodHRwczovL3d3dy5oZXJtZXMuY29tL2ZyL2ZyL3BvbHlmaWxscy43ZTFjYzMyM2Y3OWMwNjZlLmpzOjE6Mjc5NTEp" # 4.24.3 + self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.25.0 + self.stcfp = "dy5oZXJtZXMuY29tL2ZyL2ZyL3BvbHlmaWxscy43ZTFjYzMyM2Y3OWMwNjZlLmpzOjE6ODI2MykKICAgIGF0IG0uYXJncy48Y29tcHV0ZWQ+IChodHRwczovL3d3dy5oZXJtZXMuY29tL2ZyL2ZyL3BvbHlmaWxscy43ZTFjYzMyM2Y3OWMwNjZlLmpzOjE6Mjc5NTEp" # 4.25.0 self.ckwa = True def to_url_encoded_json(self): diff --git a/queue_message/appointmentrequestsender.py b/queue_message/appointmentrequestsender.py index b7e0849..a1bbec6 100644 --- a/queue_message/appointmentrequestsender.py +++ b/queue_message/appointmentrequestsender.py @@ -34,6 +34,7 @@ def is_already_sent(contact: ContactPojo) -> bool: def filter_contacts(_contact_list: list, provided_list=[]) -> list: already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day() already_sent_contacts.extend(provided_list) + _link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate() _contact_list_to_book = [] for contact in _contact_list: _to_add = True @@ -44,6 +45,11 @@ def filter_contacts(_contact_list: list, provided_list=[]) -> list: else: if contact.mail == booked.mail: _to_add = False + # 如果已经收到链接了，就不要再请求 + for link_to_validate in _link_to_validate_list: + if contact.mail == link_to_validate.email: + print("{}: link already received".format(contact.mail)) + _to_add = False if _to_add: _contact_list_to_book.append(contact) @@ -127,7 +133,7 @@ class AppointmentRequestSender(threading.Thread): _new_cookies = captchaResultGetter.get_valid_ch_cookie(sender.proxy_to_use, js_data, old_valid_cookie=_received_cookies) if _new_cookies is not None: - # self.logger.info("new cookie is " + _new_cookies) + self.logger.info("new cookie is " + _new_cookies) time.sleep(random.randint(1, 3)) # m_s_c = f.scroll m_s_c = random.randint(0, 3) @@ -160,6 +166,14 @@ class AppointmentRequestSender(threading.Thread): time.sleep(random.randint(1, 3)) self.already_tried_contact_list.append(con) can_continue = sender.send_request(HERMES_REGISTER, js_data, con, csrf=self.valid_csrf) + if can_continue == RequestResult.SUCCESS: + # 让服务器读取成功的约会 + try: + self.logger.info("try to remove success contact from list to retrieve mails") + self.list_to_retrieve_mails.remove(con) + except Exception as e: + self.logger.info("exception while remove success contact from list to retrieve mails") + print(e) else: can_continue = RequestResult.COOKIES_ERROR else: @@ -196,6 +210,7 @@ class AppointmentRequestSender(threading.Thread): def retreive_invalidate_urls(self): if not self.already_read_emails: self.logger.info("will retreive_invalidate_urls") + time.sleep(30) _mail_list = MONGO_STORE_MANAGER.get_destination_emails() _mail_list_filtred = [] for mail in _mail_list: diff --git a/request_sender.py b/request_sender.py index 27bff36..f148f6a 100644 --- a/request_sender.py +++ b/request_sender.py @@ -26,12 +26,18 @@ def is_already_sent(contact: ContactPojo) -> bool: def filter_contacts(_contact_list: list) -> list: already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day() + _link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate() _contact_list_to_book = [] for contact in _contact_list: _to_add = True for booked in already_sent_contacts: if contact.mail == booked.email: _to_add = False + #如果已经收到链接了，就不要再请求 + for link_to_validate in _link_to_validate_list: + if contact.mail == link_to_validate.email: + logger.info("{}: link already received".format(contact.mail)) + _to_add = False if _to_add: _contact_list_to_book.append(contact) @@ -64,10 +70,10 @@ def send_appointment_request(message_queue_name, _contact_list): if __name__ == '__main__': - contacts_file_path = '~/Desktop/contact_list_2024-03-26-2.xlsx' + contacts_file_path = '~/Desktop/contact_list_2024-03-25.xlsx' _contact_list = read_contacts(contacts_file_path) _contact_list_to_book = filter_contacts(_contact_list) - _segment_number = 5 + _segment_number = 2 logger.info("{} contacts to book".format(len(_contact_list_to_book))) last_thread = None for i in range(0, _segment_number): diff --git a/workers/captcha_result_getter.py b/workers/captcha_result_getter.py index e4b2ae2..bdf9243 100644 --- a/workers/captcha_result_getter.py +++ b/workers/captcha_result_getter.py @@ -101,7 +101,7 @@ class CaptchaResultGetter: return None def get_ch_raw_data_from_js_data(self, js_data: JsDataPojo, old_valid_cookie) -> str: - _tag_version = "4.24.3" + _tag_version = "4.25.0" _raw_data = "jsData={}&eventCounters=%5B%5D&jsType=ch&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv={}".format( js_data.to_url_encoded_json(), old_valid_cookie, _tag_version) print("raw data is " + _raw_data) @@ -168,7 +168,7 @@ class CaptchaResultGetter: # _le_js_raw_data = self.get_le_raw_data_from_js_data(js_le_type_data=js_le_type_data, # old_valid_cookie=old_valid_cookie) _cid = get_datadome_cookies(old_valid_cookie) - _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.24.3".format( + _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.25.0".format( js_le_type_data.to_url_encoded_json(), mousemove_count, click_count, scroll_count, touch_count, touch_count, touch_move, key_count, diff --git a/workers/proxies_constants.py b/workers/proxies_constants.py index fbdaa74..3709d9a 100644 --- a/workers/proxies_constants.py +++ b/workers/proxies_constants.py @@ -72,7 +72,4 @@ DE_PROXY_RES = { # PROXY_LIST_FR = [FR_PROXY_MOBILE_2, FR_PROXY_MOBILE, FR_PROXY_RES, FR_PROXY_RES_4, FR_PROXY_RES_2] # PROXY_LIST_FR = [FR_PROXY_RES_OXY, FR_PROXY_MOBILE_3] PROXY_LIST_FR = [FR_PROXY_MOB_OXY] -MOBILE_PROXY_LIST_FR = [FR_PROXY_MOB_OXY_STICKY] -# PROXY_LIST = [FR_PROXY_MOBILE, FR_ -# PROXY_RES] -PROXY_LIST_DE = [DE_PROXY_RES, DE_PROXY_MOBILE] +MOBILE_PROXY_LIST_FR = [FR_PROXY_MOB_OXY] diff --git a/workers/sender.py b/workers/sender.py index 99b0ec1..6ea2316 100644 --- a/workers/sender.py +++ b/workers/sender.py @@ -28,7 +28,7 @@ class Sender: self.proxy_to_use = proxy_to_use self.cookie.load(self.cookie_str) - def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str): + def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str, store_type: str): # create the message if url == "https://rendezvousparis.hermes.com/client/welcome": return @@ -37,7 +37,7 @@ class Sender: firstName=contact.first_name, lastName=contact.last_name, email=contact.mail, passport=contact.passport, ccid=contact.ccid) result.id = id - result.store_type = self.store_type + result.store_type = store_type result.created_at = time.strftime("%H:%M:%S", time.localtime()) collection_name = str(datetime.date.today()) MONGO_STORE_MANAGER.insert_reserve_result(collection_name=collection_name, reserve=result) @@ -57,6 +57,11 @@ class Sender: else: self._csrf = csrf _csrf = csrf + # 默认为空，也就是随机 + _selected_store = '' + if contact.store is not None and contact.store != "random": + _selected_store = contact.store + self.logger.info("store is:{}".format(_selected_store)) headers = {'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': js_data.ua, 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8', @@ -64,7 +69,7 @@ class Sender: 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Dest': 'document', 'Accept-Language': 'fr-FR,fr;q=0.6'} - data = {'check': '', '_csrf': _csrf, 'prefer': '', + data = {'check': '', '_csrf': _csrf, 'prefer': _selected_store, 'surname': contact.last_name.capitalize(), 'name': contact.first_name.capitalize(), 'phone_country': "FR", 'phone_number': "+33 " + contact.phone, 'email': contact.mail, 'passport_id': contact.passport, 'processing': 'on', 'cgu': 'on'} @@ -81,7 +86,8 @@ class Sender: # add to mongodb self.logger.info(response.text) self.logger.info("{}:{}".format(contact.mail, response.url)) - self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=response.url) + self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=response.url, + store_type=_selected_store) cookies_to_set = response.headers['set-cookie'] self.cookie.load(cookies_to_set) new_cookies = {k: v.value for k, v in self.cookie.items()}