From 692afc3a39de1b00e8dc8129da66420c71d5af1d Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Wed, 31 Jul 2024 04:23:31 +0200 Subject: [PATCH 01/16] add possibility to stop at hour --- queue_message/appointmentrequestsender.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/queue_message/appointmentrequestsender.py b/queue_message/appointmentrequestsender.py index df45dfc..1f6b834 100644 --- a/queue_message/appointmentrequestsender.py +++ b/queue_message/appointmentrequestsender.py @@ -93,6 +93,7 @@ class AppointmentRequestSender(threading.Thread): self.channel.start_consuming() def on_message(self, ch, method, properties, body): + self.check_and_stop_if_necessary(hour_to_check=11, mins_to_check=40) _message_count = self.cookiesPublisher.message_count() self.logger.info("message count in queue is {}".format(_message_count)) _received_object = body.decode("UTF-8") @@ -261,3 +262,13 @@ class AppointmentRequestSender(threading.Thread): else: self.logger.info("already read emails, no contact to use -> stop") self.channel.stop_consuming() + + def check_and_stop_if_necessary(self, hour_to_check, mins_to_check): + hour = datetime.datetime.now().hour + mins = datetime.datetime.now().minute + if hour == hour_to_check and mins >= mins_to_check: + self.logger.info("will stop") + self.channel.stop_consuming() + elif hour > hour_to_check: + self.logger.info("will stop") + self.channel.stop_consuming() From cb979ebb2dfe42ae3ba5252ea60f279fef7504c0 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Sat, 3 Aug 2024 13:59:20 +0200 Subject: [PATCH 02/16] add possibility to stop at hour --- db/mongo_manager.py | 2 +- queue_message/appointmentrequestsender.py | 6 ++++-- request_sender_test.py | 25 +++++++++++------------ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/db/mongo_manager.py b/db/mongo_manager.py index 434c538..c2739af 100755 --- a/db/mongo_manager.py +++ b/db/mongo_manager.py @@ -11,7 +11,7 @@ from models.contact_pojo import ContactPojo from models.mail_pojo import MailAddress from models.regisered_user_pojo import RegisteredUserPojo -MONGO_DB_URL = "mongo.lpaconsulting.fr" +MONGO_DB_URL = "mongodb://mongo.lpaconsulting.fr/?timeoutMS=100000" CAPTCHA_ERROR_COLLECTION_PREFIX = "CAPTCHA_ERROR_" BLACK_LIST = "BLACK_LIST" ACCEPTED_APPOINTMENT_LIST = "ACCEPTED_APPOINTMENT_LIST" diff --git a/queue_message/appointmentrequestsender.py b/queue_message/appointmentrequestsender.py index 1f6b834..aa0d361 100644 --- a/queue_message/appointmentrequestsender.py +++ b/queue_message/appointmentrequestsender.py @@ -65,7 +65,7 @@ def is_open(): class AppointmentRequestSender(threading.Thread): def __init__(self, sub_contact_list: list, logger, cookiesPublisher: CookiesPublisher, bakeUpCookiesPublisher: CookiesPublisher, - queue_name=REQUEST_DATA_QUEUE): + queue_name=REQUEST_DATA_QUEUE, stop_at_hour=11, stop_at_mins=30): super().__init__() self.connection = None self.logger = logger @@ -80,6 +80,8 @@ class AppointmentRequestSender(threading.Thread): self.queue_name = queue_name self.proxy_manager = ProxyManager(logger) self.already_read_emails = False + self.stop_at_hour = stop_at_hour + self.stop_at_mins = stop_at_mins def set_up_connection(self): self.connection = pika.BlockingConnection( @@ -93,7 +95,7 @@ class AppointmentRequestSender(threading.Thread): self.channel.start_consuming() def on_message(self, ch, method, properties, body): - self.check_and_stop_if_necessary(hour_to_check=11, mins_to_check=40) + self.check_and_stop_if_necessary(hour_to_check=self.stop_at_hour, mins_to_check=self.stop_at_mins) _message_count = self.cookiesPublisher.message_count() self.logger.info("message count in queue is {}".format(_message_count)) _received_object = body.decode("UTF-8") diff --git a/request_sender_test.py b/request_sender_test.py index 2fe77a2..f7c8093 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -42,10 +42,6 @@ def filter_contacts(_contact_list: list) -> list: return _contact_list_to_book -def is_open(): - return is_time_between(datetime.time(10, 30), datetime.time(19, 00)) - - count = 0 init_logger() logger = logging.getLogger() @@ -53,7 +49,7 @@ logger = logging.getLogger() logger.addHandler(logging.StreamHandler(stream=sys.stdout)) -def send_appointment_request(message_queue_name, _contact_list): +def send_appointment_request(message_queue_name, _contact_list, stop_at_hour=11, stop_at_mins=30): global count count = count + 1 for _contact in _contact_list: @@ -65,12 +61,14 @@ def send_appointment_request(message_queue_name, _contact_list): receiver = AppointmentRequestSender(sub_contact_list=_contact_list, queue_name=message_queue_name, cookiesPublisher=_cookiesPublisher, - bakeUpCookiesPublisher=_backUp_cookiesPublisher, logger=logger) + bakeUpCookiesPublisher=_backUp_cookiesPublisher, logger=logger, + stop_at_hour=stop_at_hour, stop_at_mins=stop_at_mins) print("count is " + str(count)) receiver.run() -def start_send_requests(thread_number, contact_list, data_queue_name=MORNING_DATA_CACHE): +def start_send_requests(thread_number, contact_list, data_queue_name=MORNING_DATA_CACHE, stop_at_hour=14, + stop_at_mins=56): print("start send requests") _contact_list_to_book = filter_contacts(contact_list) _segment_number = thread_number @@ -81,19 +79,20 @@ def start_send_requests(thread_number, contact_list, data_queue_name=MORNING_DAT logger.info("segment is {}".format(i)) _step = int(len(_contact_list_to_book) / _segment_number) _sublist = _contact_list_to_book[i * _step:_step * (i + 1)] - _thread1 = Thread(target=send_appointment_request, args=(data_queue_name, _sublist)) + _thread1 = Thread(target=send_appointment_request, args=(data_queue_name, _sublist, stop_at_hour, stop_at_mins)) thread_list.append(_thread1) _thread1.start() for _thread in thread_list: _thread.join() -def send_request_for_file_list(file_list: list[str], thread_number: int = 20, data_queue_name=MORNING_DATA_CACHE): +def send_request_for_file_list(file_list: list, thread_number: int = 20, data_queue_name=MORNING_DATA_CACHE, + stop_at_hour=11, stop_at_mins=30): for _file_path in file_list: logger.info("send request for file: " + _file_path) _contact_list = read_contacts(_file_path) start_send_requests(thread_number=thread_number, contact_list=_contact_list, - data_queue_name=data_queue_name) + data_queue_name=data_queue_name, stop_at_hour=stop_at_hour, stop_at_mins=stop_at_mins) if __name__ == '__main__': @@ -102,6 +101,6 @@ if __name__ == '__main__': # '~/Desktop/contact_list_2024-05-21.xlsx', # '~/Desktop/15_05_to_test.xlsx'] # file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx'] - file_list = ['~/Desktop/contact_list_2024-06-28.xlsx'] - send_request_for_file_list(file_list=file_list, thread_number=10, - data_queue_name=MORNING_DATA_CACHE) + file_list = ['~/Desktop/contact_list_all.xlsx'] + send_request_for_file_list(file_list=file_list, thread_number=2, + data_queue_name=MORNING_DATA_CACHE, stop_at_hour=16, stop_at_mins=30) From a1df129225795b18d9b24c24f0ea340e2b50407c Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Sun, 4 Aug 2024 22:32:16 +0200 Subject: [PATCH 03/16] handle gateway error --- workers/sender.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/workers/sender.py b/workers/sender.py index 09ced4e..5ac8c65 100644 --- a/workers/sender.py +++ b/workers/sender.py @@ -122,6 +122,9 @@ class Sender: # session = HTMLSession() self.logger.info("blocked by hermes, mail:{}".format(contact.mail)) return RequestResult.BLOCKED + elif response.status_code == 502: + self.logger.info("proxy error, mail:{}".format(contact.mail)) + return RequestResult.PROXY_ERROR return RequestResult.BLOCKED except Exception as Error: print(Error) From 981173317e5c9f11364fdff50dfbfae92edc797d Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Mon, 5 Aug 2024 14:29:44 +0200 Subject: [PATCH 04/16] shuffle contacts before send requests --- request_sender_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/request_sender_test.py b/request_sender_test.py index 2fe77a2..efb2f75 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -1,5 +1,6 @@ import datetime import logging +import random import sys from threading import Thread @@ -92,6 +93,7 @@ def send_request_for_file_list(file_list: list[str], thread_number: int = 20, da for _file_path in file_list: logger.info("send request for file: " + _file_path) _contact_list = read_contacts(_file_path) + random.shuffle(_contact_list) start_send_requests(thread_number=thread_number, contact_list=_contact_list, data_queue_name=data_queue_name) @@ -102,6 +104,6 @@ if __name__ == '__main__': # '~/Desktop/contact_list_2024-05-21.xlsx', # '~/Desktop/15_05_to_test.xlsx'] # file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx'] - file_list = ['~/Desktop/contact_list_2024-06-28.xlsx'] - send_request_for_file_list(file_list=file_list, thread_number=10, + file_list = ['~/Desktop/contact_list_2024-07-25.xlsx'] + send_request_for_file_list(file_list=file_list, thread_number=100, data_queue_name=MORNING_DATA_CACHE) From df833b5ac834c025f3e41b4309cb0c605eff9abe Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 5 Aug 2024 15:33:09 +0200 Subject: [PATCH 05/16] change thread_number to 60 --- request_sender_test.py | 1 + scheduler_test.py | 11 +++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/request_sender_test.py b/request_sender_test.py index f8b813d..288a869 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -89,6 +89,7 @@ def start_send_requests(thread_number, contact_list, data_queue_name=MORNING_DAT def send_request_for_file_list(file_list: list, thread_number: int = 20, data_queue_name=MORNING_DATA_CACHE, stop_at_hour=11, stop_at_mins=30): + logger.info("stop_at_hour is " + str(stop_at_hour) + " stop_at_mins is " + str(stop_at_mins)) for _file_path in file_list: logger.info("send request for file: " + _file_path) _contact_list = read_contacts(_file_path) diff --git a/scheduler_test.py b/scheduler_test.py index 656a5b3..eb60245 100644 --- a/scheduler_test.py +++ b/scheduler_test.py @@ -6,20 +6,19 @@ from request_sender_test import send_request_for_file_list def start_book_appointment(): - file_list = ['~/Desktop/contact_list_2024-05-27.xlsx', + file_list = ['~/Desktop/contact_list_2024-07-31.xlsx', '~/Desktop/contact_list_2024-05-25.xlsx', '~/Desktop/contact_list_2024-05-24.xlsx', - '~/Desktop/contact_list_2024-05-23.xlsx', - '~/Desktop/contact_list_2024-05-22.xlsx'] - send_request_for_file_list(file_list=file_list, thread_number=40, - data_queue_name=MORNING_DATA_CACHE) + '~/Desktop/contact_list_2024-06-26.xlsx'] + send_request_for_file_list(file_list=file_list, thread_number=60, + data_queue_name=MORNING_DATA_CACHE, stop_at_hour=11, stop_at_mins=40) def start_check_results_job(sched): sched.add_job(start_book_appointment, 'cron', day_of_week='mon-sat', hour='10', minute='30', misfire_grace_time=10, - second='20', timezone='Europe/Paris', max_instances=1, args=[]) + second='10', timezone='Europe/Paris', max_instances=1, args=[]) def config_and_start_jobs(): From 3b18a0756bcaf17f6a534f1c087babfb33f0d4d9 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Mon, 5 Aug 2024 15:33:32 +0200 Subject: [PATCH 06/16] new files --- scheduler_test.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/scheduler_test.py b/scheduler_test.py index 656a5b3..8a1bebb 100644 --- a/scheduler_test.py +++ b/scheduler_test.py @@ -6,12 +6,18 @@ from request_sender_test import send_request_for_file_list def start_book_appointment(): - file_list = ['~/Desktop/contact_list_2024-05-27.xlsx', - '~/Desktop/contact_list_2024-05-25.xlsx', - '~/Desktop/contact_list_2024-05-24.xlsx', - '~/Desktop/contact_list_2024-05-23.xlsx', - '~/Desktop/contact_list_2024-05-22.xlsx'] - send_request_for_file_list(file_list=file_list, thread_number=40, + file_list = ['~/Desktop/contact_list_2024-07-25.xlsx', + '~/Desktop/contact_list_2024-07-26.xlsx', + '~/Desktop/contact_list_2024-07-30.xlsx', + '~/Desktop/contact_list_2024-07-24.xlsx', + '~/Desktop/contact_list_2024-07-22.xlsx', + '~/Desktop/contact_list_2024-07-05.xlsx', + '~/Desktop/contact_list_2024-07-04.xlsx', + '~/Desktop/contact_list_2024-06-22.xlsx', + '~/Desktop/contact_list_2024-06-20.xlsx', + '~/Desktop/contact_list_2024-06-19.xlsx', + '~/Desktop/contact_list_2024-05-27.xlsx'] + send_request_for_file_list(file_list=file_list, thread_number=58, data_queue_name=MORNING_DATA_CACHE) @@ -19,7 +25,7 @@ def start_check_results_job(sched): sched.add_job(start_book_appointment, 'cron', day_of_week='mon-sat', hour='10', minute='30', misfire_grace_time=10, - second='20', timezone='Europe/Paris', max_instances=1, args=[]) + second='10', timezone='Europe/Paris', max_instances=1, args=[]) def config_and_start_jobs(): From bff6c68b69077ae20fe0ba7f5fe2deb77656827a Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Fri, 9 Aug 2024 14:05:47 +0200 Subject: [PATCH 07/16] send to MORING_DATA_CACHE_2 when blocked --- request_sender_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request_sender_test.py b/request_sender_test.py index 288a869..a973b80 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -57,7 +57,7 @@ def send_appointment_request(message_queue_name, _contact_list, stop_at_hour=11, logger.info(_contact) _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name) _cookiesPublisher.set_up_connection() - _backUp_cookiesPublisher = CookiesPublisher(queue_name=MORNING_DATA_CACHE_BAK) + _backUp_cookiesPublisher = CookiesPublisher(queue_name=MORNING_DATA_CACHE_2) _backUp_cookiesPublisher.set_up_connection() receiver = AppointmentRequestSender(sub_contact_list=_contact_list, queue_name=message_queue_name, From 786bbc9968d2c9c6f999a6a092f732aa54186a6e Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Sat, 10 Aug 2024 16:49:02 +0200 Subject: [PATCH 08/16] add file to migrate data to data_cache_2 queue --- migrate_to_data_cache_2.py | 6 ++++++ request_sender_test.py | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 migrate_to_data_cache_2.py diff --git a/migrate_to_data_cache_2.py b/migrate_to_data_cache_2.py new file mode 100644 index 0000000..51e29f1 --- /dev/null +++ b/migrate_to_data_cache_2.py @@ -0,0 +1,6 @@ + +from queue_message.CookiesPublisher import MORNING_DATA_CACHE_2, MORNING_DATA_CACHE, MORNING_DATA_CACHE_BAK +from workers.MessagerTransporter import migrate_message_to_queue + +if __name__ == '__main__': + migrate_message_to_queue(from_queue=MORNING_DATA_CACHE,to_queue=MORNING_DATA_CACHE_2) diff --git a/request_sender_test.py b/request_sender_test.py index 288a869..bf0fe8b 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -57,7 +57,7 @@ def send_appointment_request(message_queue_name, _contact_list, stop_at_hour=11, logger.info(_contact) _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name) _cookiesPublisher.set_up_connection() - _backUp_cookiesPublisher = CookiesPublisher(queue_name=MORNING_DATA_CACHE_BAK) + _backUp_cookiesPublisher = CookiesPublisher(queue_name=MORNING_DATA_CACHE_2) _backUp_cookiesPublisher.set_up_connection() receiver = AppointmentRequestSender(sub_contact_list=_contact_list, queue_name=message_queue_name, @@ -106,4 +106,4 @@ if __name__ == '__main__': # file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx'] file_list = ['~/Desktop/contact_list_all.xlsx'] send_request_for_file_list(file_list=file_list, thread_number=2, - data_queue_name=MORNING_DATA_CACHE, stop_at_hour=16, stop_at_mins=30) + data_queue_name=MORNING_DATA_CACHE, stop_at_hour=17, stop_at_mins=30) From 1cd870a9226cc5c64246e1c1ffd312e25d6603a4 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 16 Aug 2024 14:43:07 +0200 Subject: [PATCH 09/16] support 4.33.0 --- docs/4_33_0/jsdata_ch_4_33_0.json | 126 ++++++++++++++++ docs/4_33_0/jsdata_ch_4_33_0_generated.json | 126 ++++++++++++++++ docs/4_33_0/jsdata_le_4_33_0.json | 151 ++++++++++++++++++++ docs/4_33_0/jsdata_le_4_33_0_generated.json | 151 ++++++++++++++++++++ models/jsdata_le_pojo.py | 11 +- models/jsdata_pojo.py | 6 +- request_sender_test.py | 4 +- scheduler_test.py | 13 +- workers/captcha_result_getter.py | 4 +- 9 files changed, 568 insertions(+), 24 deletions(-) create mode 100644 docs/4_33_0/jsdata_ch_4_33_0.json create mode 100644 docs/4_33_0/jsdata_ch_4_33_0_generated.json create mode 100644 docs/4_33_0/jsdata_le_4_33_0.json create mode 100644 docs/4_33_0/jsdata_le_4_33_0_generated.json diff --git a/docs/4_33_0/jsdata_ch_4_33_0.json b/docs/4_33_0/jsdata_ch_4_33_0.json new file mode 100644 index 0000000..1e6ef6b --- /dev/null +++ b/docs/4_33_0/jsdata_ch_4_33_0.json @@ -0,0 +1,126 @@ +{ + "ttst": 26, + "ifov": false, + "hc": 7, + "br_oh": 755, + "br_ow": 407, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 4.698411895004195, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 755, + "br_w": 407, + "isf": true, + "nddc": 1, + "rs_h": 904, + "rs_w": 407, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 904, + "ars_w": 407, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": false, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "RnTRQvA,wYUxYUpz", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 1, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbig1NDgpXShuKDMyMSkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1snXHg3M1x4NjhceDYxXHg2NFx4NmZceDc3XHg1Mlx4NmZceDZmXHg3NCddWydceDcxXHg3NVx4NjVceDcyXHg3", + "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzk0NzcpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc2MDIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU1MjIw", + "ckwa": true, + "glvd": "ARM", + "glrd": "Mali-G610 MC6", + "wwl": false, + "jset": 1723810343 +} \ No newline at end of file diff --git a/docs/4_33_0/jsdata_ch_4_33_0_generated.json b/docs/4_33_0/jsdata_ch_4_33_0_generated.json new file mode 100644 index 0000000..3cadeda --- /dev/null +++ b/docs/4_33_0/jsdata_ch_4_33_0_generated.json @@ -0,0 +1,126 @@ +{ + "ttst": 142, + "ifov": false, + "hc": 2, + "br_oh": 755, + "br_ow": 407, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 5.668757988742678, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 755, + "br_w": 407, + "isf": true, + "nddc": 1, + "rs_h": 904, + "rs_w": 407, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 904, + "ars_w": 407, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "0iRvAfP,asWyZrVp", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 8, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbig1NDgpXShuKDMyMSkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1snXHg3M1x4NjhceDYxXHg2NFx4NmZceDc3XHg1Mlx4NmZceDZmXHg3NCddWydceDcxXHg3NVx4NjVceDcyXHg3", + "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzk0NzcpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc2MDIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU1MjIw", + "ckwa": true, + "glvd": "ARM", + "glrd": "Mali-G610 MC6", + "wwl": false, + "jset": 1723811967 +} \ No newline at end of file diff --git a/docs/4_33_0/jsdata_le_4_33_0.json b/docs/4_33_0/jsdata_le_4_33_0.json new file mode 100644 index 0000000..9f88c87 --- /dev/null +++ b/docs/4_33_0/jsdata_le_4_33_0.json @@ -0,0 +1,151 @@ +{ + "ttst": 26, + "ifov": false, + "hc": 7, + "br_oh": 755, + "br_ow": 407, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 4.698411895004195, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 755, + "br_w": 407, + "isf": true, + "nddc": 1, + "rs_h": 904, + "rs_w": 407, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 904, + "ars_w": 407, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": false, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "RnTRQvA,wYUxYUpz", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 1, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbig1NDgpXShuKDMyMSkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1snXHg3M1x4NjhceDYxXHg2NFx4NmZceDc3XHg1Mlx4NmZceDZmXHg3NCddWydceDcxXHg3NVx4NjVceDcyXHg3", + "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzk0NzcpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc2MDIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU1MjIw", + "ckwa": true, + "glvd": "ARM", + "glrd": "Mali-G610 MC6", + "wwl": false, + "jset": 1723810376, + "emd": "k:ai,ao,vi", + "log2": true, + "dcok": ".hermes.com", + "m_fmi": false, + "tbce": 0, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "k_hA": 0, + "k_hSD": 0, + "k_pA": 455.10526315789474, + "k_pSD": 553.9116303011027, + "k_rA": 455.10526315789474, + "k_rSD": 553.9116303011027, + "k_ikA": 455.10526315789474, + "k_ikSD": 553.9116303011027, + "k_kdc": 20, + "k_kuc": 20, + "m_s_c": 0, + "m_m_c": 4, + "m_c_c": 4, + "m_cm_r": 1, + "m_ms_r": -1 +} diff --git a/docs/4_33_0/jsdata_le_4_33_0_generated.json b/docs/4_33_0/jsdata_le_4_33_0_generated.json new file mode 100644 index 0000000..e98b16f --- /dev/null +++ b/docs/4_33_0/jsdata_le_4_33_0_generated.json @@ -0,0 +1,151 @@ +{ + "ttst": 122, + "ifov": false, + "hc": 2, + "br_oh": 755, + "br_ow": 407, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 3.517094923360057, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 755, + "br_w": 407, + "isf": true, + "nddc": 1, + "rs_h": 904, + "rs_w": 407, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 904, + "ars_w": 407, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "0iRvAfP,asWyZrVp", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 8, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbig1NDgpXShuKDMyMSkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1snXHg3M1x4NjhceDYxXHg2NFx4NmZceDc3XHg1Mlx4NmZceDZmXHg3NCddWydceDcxXHg3NVx4NjVceDcyXHg3", + "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzk0NzcpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc2MDIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU1MjIw", + "ckwa": true, + "glvd": "ARM", + "glrd": "Mali-G610 MC6", + "wwl": false, + "jset": 1723811972, + "emd": "k:ai,ao,vi", + "log2": true, + "dcok": ".hermes.com", + "m_fmi": false, + "tbce": 0, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "k_hA": null, + "k_hSD": null, + "k_pA": null, + "k_pSD": null, + "k_rA": null, + "k_rSD": null, + "k_ikA": null, + "k_ikSD": null, + "k_kdc": 0, + "k_kuc": 0, + "m_s_c": 1, + "m_m_c": 3, + "m_c_c": 3, + "m_cm_r": 1.0, + "m_ms_r": 3.0 +} \ No newline at end of file diff --git a/models/jsdata_le_pojo.py b/models/jsdata_le_pojo.py index 1350a1c..0af127b 100644 --- a/models/jsdata_le_pojo.py +++ b/models/jsdata_le_pojo.py @@ -23,10 +23,11 @@ class JsDataLeTypePojo(JsDataPojo): vnd, pr, ts_mtp, dvm, m_s_c, m_m_c, m_c_c, m_cm_r, m_ms_r, emd: str): super().__init__(glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu, vnd, pr, ts_mtp, dvm, emd=emd) - self.dcok = ".hermes.com" + self.emd = emd self.log2 = True - # self.m_fmi = False # 这2个字段在4.29.0中没有再出现了 - # self.tbce = 0 + self.dcok = ".hermes.com" + self.m_fmi = False # 这2个字段在4.29.0中没有再出现了 + self.tbce = 0 self.es_sigmdn = None self.es_mumdn = None self.es_distmdn = None @@ -47,8 +48,8 @@ class JsDataLeTypePojo(JsDataPojo): self.m_c_c = m_c_c self.m_cm_r = m_cm_r self.m_ms_r = m_ms_r - self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.32.5 - self.stcfp = "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzkwMTMpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTI5OTIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUzODQ2" # 4.32.5 + self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbig1NDgpXShuKDMyMSkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1snXHg3M1x4NjhceDYxXHg2NFx4NmZceDc3XHg1Mlx4NmZceDZmXHg3NCddWydceDcxXHg3NVx4NjVceDcyXHg3" # 4.33.0 + self.stcfp = "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzk0NzcpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc2MDIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU1MjIw" # 4.33.0 test_data_json = """{"glvd":"ARM","glrd":"Mali-G57 MC2","hc":5,"br_oh":620,"br_ow":360,"br_h":620,"br_w":360,"rs_h":804,"rs_w":360,"rs_cd":24,"ars_h":804,"ars_w":360,"plg":2,"eva":33,"vnd":"Google Inc.","plu":["eTJjZUp","iZMOHLs1"],"ua":"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36","dvm":4,"acw":"probably","pr":3,"ts_mtp":5,"tz":-120,"emd":"k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801","cookiesStr":"app.sig=_FqS3_SPN2D40Z4Qlq-_ybxzRvQ;datadome=VSGCNMYkY6L81typRJUpurlyd39dHxdfxIYc6DzRWDT7_rtZ8UodGcbow1okQR2zg_t4d_NcP0rC1ZHNz3XSDAGFZCJhFI49lai~x2Nb2unkhOlY8o4E04Wjmde~auDZ;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiLW9jbktLVXAzMjVTOEFEdFpLUU9kSXlMIn0=;policy=accepted;lang=fr;"}""" diff --git a/models/jsdata_pojo.py b/models/jsdata_pojo.py index 44f52e0..7dd9525 100644 --- a/models/jsdata_pojo.py +++ b/models/jsdata_pojo.py @@ -130,7 +130,6 @@ class JsDataPojo: cfpfe: str stcfp: str ckwa: bool - emd: str glvd: str glrd: str wwl: bool @@ -237,7 +236,6 @@ class JsDataPojo: self.vc1ts = True self.dvm = dvm self.set_default_values() - self.emd = emd self.glvd = glvd self.glrd = glrd self.wwl = False @@ -267,8 +265,8 @@ class JsDataPojo: self.emt = False self.bfr = False self.dbov = False - self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.32.5 - self.stcfp = "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzkwMTMpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTI5OTIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUzODQ2" # 4.32.5 + self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbig1NDgpXShuKDMyMSkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1snXHg3M1x4NjhceDYxXHg2NFx4NmZceDc3XHg1Mlx4NmZceDZmXHg3NCddWydceDcxXHg3NVx4NjVceDcyXHg3" # 4.33.0 + self.stcfp = "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzk0NzcpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc2MDIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU1MjIw" # 4.33.0 self.ckwa = True def to_url_encoded_json(self): diff --git a/request_sender_test.py b/request_sender_test.py index bf0fe8b..cc6658c 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -105,5 +105,5 @@ if __name__ == '__main__': # '~/Desktop/15_05_to_test.xlsx'] # file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx'] file_list = ['~/Desktop/contact_list_all.xlsx'] - send_request_for_file_list(file_list=file_list, thread_number=2, - data_queue_name=MORNING_DATA_CACHE, stop_at_hour=17, stop_at_mins=30) + send_request_for_file_list(file_list=file_list, thread_number=1, + data_queue_name=MORNING_DATA_CACHE_2, stop_at_hour=17, stop_at_mins=30) diff --git a/scheduler_test.py b/scheduler_test.py index 85c7e4d..1177e3d 100644 --- a/scheduler_test.py +++ b/scheduler_test.py @@ -6,18 +6,9 @@ from request_sender_test import send_request_for_file_list def start_book_appointment(): - file_list = ['~/Desktop/contact_list_2024-07-25.xlsx', - '~/Desktop/contact_list_2024-07-26.xlsx', - '~/Desktop/contact_list_2024-07-30.xlsx', - '~/Desktop/contact_list_2024-07-24.xlsx', - '~/Desktop/contact_list_2024-07-22.xlsx', - '~/Desktop/contact_list_2024-07-05.xlsx', - '~/Desktop/contact_list_2024-07-04.xlsx', - '~/Desktop/contact_list_2024-06-22.xlsx', - '~/Desktop/contact_list_2024-06-20.xlsx', - '~/Desktop/contact_list_2024-06-19.xlsx', + file_list = ['~/Desktop/contact_list_2024-08-09.xlsx', '~/Desktop/contact_list_2024-05-27.xlsx'] - send_request_for_file_list(file_list=file_list, thread_number=60, + send_request_for_file_list(file_list=file_list, thread_number=10, data_queue_name=MORNING_DATA_CACHE, stop_at_hour=11, stop_at_mins=40) diff --git a/workers/captcha_result_getter.py b/workers/captcha_result_getter.py index 1e7d963..828dfe8 100644 --- a/workers/captcha_result_getter.py +++ b/workers/captcha_result_getter.py @@ -105,7 +105,7 @@ class CaptchaResultGetter: return None def get_ch_raw_data_from_js_data(self, js_data: JsDataPojo, old_valid_cookie) -> str: - _tag_version = "4.32.5" + _tag_version = "4.33.0" _raw_data = "jsData={}&eventCounters=%5B%5D&jsType=ch&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv={}".format( js_data.to_url_encoded_json(), old_valid_cookie, _tag_version) print("raw ch data is " + _raw_data) @@ -172,7 +172,7 @@ class CaptchaResultGetter: # _le_js_raw_data = self.get_le_raw_data_from_js_data(js_le_type_data=js_le_type_data, # old_valid_cookie=old_valid_cookie) _cid = get_datadome_cookies(old_valid_cookie) - _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.32.5".format( + _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.33.0".format( js_le_type_data.to_url_encoded_json(), mousemove_count, click_count, scroll_count, touch_count, touch_count, touch_move, key_count, From 2f0a41a8a80dd34d94834063c9c8ab8871823709 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Wed, 21 Aug 2024 21:48:28 +0200 Subject: [PATCH 10/16] more anyIp proxy --- proxy_manager/proxy_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/proxy_manager/proxy_manager.py b/proxy_manager/proxy_manager.py index be9a578..f6f0140 100644 --- a/proxy_manager/proxy_manager.py +++ b/proxy_manager/proxy_manager.py @@ -45,6 +45,7 @@ FR_MOBILE_ANY_IP_STICKY = { # 八分之一用data_impulse MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY, + FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_PROXY_RES_OXY_STICKY, FR_PROXY_DATA_IMPULSE_STICKY] From 9dea33a10667627298c060d2d2a6b085cbb199d0 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Thu, 22 Aug 2024 14:12:18 +0200 Subject: [PATCH 11/16] do not get ip after request --- utils/address_ip.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/utils/address_ip.py b/utils/address_ip.py index 4d46542..0924b55 100644 --- a/utils/address_ip.py +++ b/utils/address_ip.py @@ -6,17 +6,18 @@ from proxy_manager.proxy_manager import ProxyManager def get_address_ip(proxy_to_use): - try: - response = requests.get(url="https://api.ipify.org", proxies=proxy_to_use, verify=False, - timeout=15) - print(response.status_code) - if response.status_code == 200: - return response.text - else: - return "" - except Exception as Error: - print(Error) - return "error" + return "" + # try: + # response = requests.get(url="https://api.ipify.org", proxies=proxy_to_use, verify=False, + # timeout=15) + # print(response.status_code) + # if response.status_code == 200: + # return response.text + # else: + # return "" + # except Exception as Error: + # print(Error) + # return "error" if __name__ == '__main__': From f7b26d8de260b9a5b12328f6f618f230bec6e054 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 26 Aug 2024 12:22:12 +0200 Subject: [PATCH 12/16] use bak queue for url validation --- request_sender_test.py | 8 ++++---- workers/link_validator_with_provided_list.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/request_sender_test.py b/request_sender_test.py index cc6658c..daab207 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -57,7 +57,7 @@ def send_appointment_request(message_queue_name, _contact_list, stop_at_hour=11, logger.info(_contact) _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name) _cookiesPublisher.set_up_connection() - _backUp_cookiesPublisher = CookiesPublisher(queue_name=MORNING_DATA_CACHE_2) + _backUp_cookiesPublisher = CookiesPublisher(queue_name=MORNING_DATA_CACHE_BAK) _backUp_cookiesPublisher.set_up_connection() receiver = AppointmentRequestSender(sub_contact_list=_contact_list, queue_name=message_queue_name, @@ -104,6 +104,6 @@ if __name__ == '__main__': # '~/Desktop/contact_list_2024-05-21.xlsx', # '~/Desktop/15_05_to_test.xlsx'] # file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx'] - file_list = ['~/Desktop/contact_list_all.xlsx'] - send_request_for_file_list(file_list=file_list, thread_number=1, - data_queue_name=MORNING_DATA_CACHE_2, stop_at_hour=17, stop_at_mins=30) + file_list = ['~/Desktop/contact_list_2024-08-22_aol.xlsx'] + send_request_for_file_list(file_list=file_list, thread_number=5, + data_queue_name=MORNING_DATA_CACHE_2, stop_at_hour=18, stop_at_mins=30) diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index 7f65b76..321eb24 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -215,8 +215,8 @@ def validate_all_links(): divided = 3 default_segment_number = 20 _first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))] - _queue_name = MORNING_DATA_CACHE_2 - # _queue_name = MORNING_DATA_CACHE_BAK + # _queue_name = MORNING_DATA_CACHE_2 + _queue_name = MORNING_DATA_CACHE_BAK if len(all_link_list) > divided * default_segment_number: _segment_number = default_segment_number else: From 9f175575d05782263a31177217bf95ec608ed7ae Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 26 Aug 2024 17:04:51 +0200 Subject: [PATCH 13/16] support 4.33.1 --- docs/4_33_1/jsdata_ch_4_33_1.json | 126 ++++++++++++++++ docs/4_33_1/jsdata_ch_4_33_1_generated.json | 126 ++++++++++++++++ docs/4_33_1/jsdata_le_4_33_1.json | 151 ++++++++++++++++++++ docs/4_33_1/jsdata_le_4_33_1_generated.json | 151 ++++++++++++++++++++ migrate_cached_request_data.py | 6 +- migrate_to_data_cache_2.py | 2 +- models/jsdata_le_pojo.py | 8 +- models/jsdata_pojo.py | 4 +- request_sender_test.py | 6 +- workers/captcha_result_getter.py | 4 +- 10 files changed, 569 insertions(+), 15 deletions(-) create mode 100644 docs/4_33_1/jsdata_ch_4_33_1.json create mode 100644 docs/4_33_1/jsdata_ch_4_33_1_generated.json create mode 100644 docs/4_33_1/jsdata_le_4_33_1.json create mode 100644 docs/4_33_1/jsdata_le_4_33_1_generated.json diff --git a/docs/4_33_1/jsdata_ch_4_33_1.json b/docs/4_33_1/jsdata_ch_4_33_1.json new file mode 100644 index 0000000..55b8724 --- /dev/null +++ b/docs/4_33_1/jsdata_ch_4_33_1.json @@ -0,0 +1,126 @@ +{ + "ttst": 57, + "ifov": false, + "hc": 7, + "br_oh": 755, + "br_ow": 407, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 6.190817050206077, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 755, + "br_w": 407, + "isf": true, + "nddc": 1, + "rs_h": 904, + "rs_w": 407, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 904, + "ars_w": 407, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": false, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "t157dt15,9HqVp78", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 8, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzkwMDkpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc5MDQpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU0NzY4", + "ckwa": true, + "glvd": "ARM", + "glrd": "Mali-G610 MC6", + "wwl": false, + "jset": 1724683091 +} \ No newline at end of file diff --git a/docs/4_33_1/jsdata_ch_4_33_1_generated.json b/docs/4_33_1/jsdata_ch_4_33_1_generated.json new file mode 100644 index 0000000..ffb4c69 --- /dev/null +++ b/docs/4_33_1/jsdata_ch_4_33_1_generated.json @@ -0,0 +1,126 @@ +{ + "ttst": 145, + "ifov": false, + "hc": 5, + "br_oh": 693, + "br_ow": 393, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 9.388659029920017, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 693, + "br_w": 393, + "isf": true, + "nddc": 1, + "rs_h": 873, + "rs_w": 393, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.75, + "ars_h": 873, + "ars_w": 393, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "T0DozZr,oc1iZMOP", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 4, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzkwMDkpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc5MDQpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU0NzY4", + "ckwa": true, + "glvd": "Google Inc. (ARM)", + "glrd": "ANGLE (ARM,Mali-G68 MC4,OpenGL ES 3.2)", + "wwl": false, + "jset": 1724684158 +} \ No newline at end of file diff --git a/docs/4_33_1/jsdata_le_4_33_1.json b/docs/4_33_1/jsdata_le_4_33_1.json new file mode 100644 index 0000000..7b168f1 --- /dev/null +++ b/docs/4_33_1/jsdata_le_4_33_1.json @@ -0,0 +1,151 @@ +{ + "ttst": 57, + "ifov": false, + "hc": 7, + "br_oh": 755, + "br_ow": 407, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 6.190817050206077, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 755, + "br_w": 407, + "isf": true, + "nddc": 1, + "rs_h": 904, + "rs_w": 407, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 904, + "ars_w": 407, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": false, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "t157dt15,9HqVp78", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 8, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzkwMDkpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc5MDQpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU0NzY4", + "ckwa": true, + "glvd": "ARM", + "glrd": "Mali-G610 MC6", + "wwl": false, + "jset": 1724683113, + "dcok": ".hermes.com", + "log2": true, + "emd": "k:ai,vi,ao", + "m_fmi": false, + "tbce": 0, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "k_hA": 0, + "k_hSD": 0, + "k_pA": 449.35714285714283, + "k_pSD": 611.1603258372969, + "k_rA": 449.35714285714283, + "k_rSD": 611.1603258372969, + "k_ikA": 449.35714285714283, + "k_ikSD": 611.1603258372969, + "k_kdc": 15, + "k_kuc": 15, + "m_s_c": 0, + "m_m_c": 4, + "m_c_c": 4, + "m_cm_r": 1, + "m_ms_r": -1 +} \ No newline at end of file diff --git a/docs/4_33_1/jsdata_le_4_33_1_generated.json b/docs/4_33_1/jsdata_le_4_33_1_generated.json new file mode 100644 index 0000000..62ec188 --- /dev/null +++ b/docs/4_33_1/jsdata_le_4_33_1_generated.json @@ -0,0 +1,151 @@ +{ + "ttst": 146, + "ifov": false, + "hc": 3, + "br_oh": 697, + "br_ow": 393, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 6.026548169863489, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 697, + "br_w": 393, + "isf": true, + "nddc": 1, + "rs_h": 876, + "rs_w": 393, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.75, + "ars_h": 876, + "ars_w": 393, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "AIr89999,yCBn6lS", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 2, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzkwMDkpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc5MDQpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU0NzY4", + "ckwa": true, + "glvd": "Google Inc. (ARM)", + "glrd": "ANGLE (ARM,Mali-G57 MC2,OpenGL ES 3.2)", + "wwl": false, + "jset": 1724684234, + "dcok": ".hermes.com", + "log2": true, + "emd": "k:ai,vi,ao", + "m_fmi": false, + "tbce": 0, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "k_hA": null, + "k_hSD": null, + "k_pA": null, + "k_pSD": null, + "k_rA": null, + "k_rSD": null, + "k_ikA": null, + "k_ikSD": null, + "k_kdc": 0, + "k_kuc": 0, + "m_s_c": 2, + "m_m_c": 5, + "m_c_c": 3, + "m_cm_r": 0.6, + "m_ms_r": 2.5 +} \ No newline at end of file diff --git a/migrate_cached_request_data.py b/migrate_cached_request_data.py index 5882901..9238407 100644 --- a/migrate_cached_request_data.py +++ b/migrate_cached_request_data.py @@ -4,10 +4,10 @@ from queue_message.CookiesPublisher import MORNING_DATA_CACHE_2, MORNING_DATA_CA from workers.MessagerTransporter import migrate_message_to_queue if __name__ == '__main__': - p1 = Process(target=migrate_message_to_queue, args=(MORNING_DATA_CACHE_2, MORNING_DATA_CACHE_BAK)) - p1.start() + # p1 = Process(target=migrate_message_to_queue, args=(MORNING_DATA_CACHE_2, MORNING_DATA_CACHE_BAK)) + # p1.start() p2 = Process(target=migrate_message_to_queue, args=(MORNING_DATA_CACHE, MORNING_DATA_CACHE_BAK)) p2.start() - p2.join() + # p2.join() # migrate_message_to_queue(from_queue=MORNING_DATA_CACHE_2) # migrate_message_to_queue(from_queue=MORNING_DATA_CACHE) diff --git a/migrate_to_data_cache_2.py b/migrate_to_data_cache_2.py index 51e29f1..f6bef00 100644 --- a/migrate_to_data_cache_2.py +++ b/migrate_to_data_cache_2.py @@ -3,4 +3,4 @@ from queue_message.CookiesPublisher import MORNING_DATA_CACHE_2, MORNING_DATA_CA from workers.MessagerTransporter import migrate_message_to_queue if __name__ == '__main__': - migrate_message_to_queue(from_queue=MORNING_DATA_CACHE,to_queue=MORNING_DATA_CACHE_2) + migrate_message_to_queue(from_queue=MORNING_DATA_CACHE,to_queue=MORNING_DATA_CACHE_BAK) diff --git a/models/jsdata_le_pojo.py b/models/jsdata_le_pojo.py index 0af127b..24f2997 100644 --- a/models/jsdata_le_pojo.py +++ b/models/jsdata_le_pojo.py @@ -23,9 +23,9 @@ class JsDataLeTypePojo(JsDataPojo): vnd, pr, ts_mtp, dvm, m_s_c, m_m_c, m_c_c, m_cm_r, m_ms_r, emd: str): super().__init__(glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu, vnd, pr, ts_mtp, dvm, emd=emd) - self.emd = emd - self.log2 = True self.dcok = ".hermes.com" + self.log2 = True + self.emd = emd self.m_fmi = False # 这2个字段在4.29.0中没有再出现了 self.tbce = 0 self.es_sigmdn = None @@ -48,8 +48,8 @@ class JsDataLeTypePojo(JsDataPojo): self.m_c_c = m_c_c self.m_cm_r = m_cm_r self.m_ms_r = m_ms_r - self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbig1NDgpXShuKDMyMSkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1snXHg3M1x4NjhceDYxXHg2NFx4NmZceDc3XHg1Mlx4NmZceDZmXHg3NCddWydceDcxXHg3NVx4NjVceDcyXHg3" # 4.33.0 - self.stcfp = "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzk0NzcpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc2MDIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU1MjIw" # 4.33.0 + self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.33.1 + self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzkwMDkpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc5MDQpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU0NzY4" # 4.33.1 test_data_json = """{"glvd":"ARM","glrd":"Mali-G57 MC2","hc":5,"br_oh":620,"br_ow":360,"br_h":620,"br_w":360,"rs_h":804,"rs_w":360,"rs_cd":24,"ars_h":804,"ars_w":360,"plg":2,"eva":33,"vnd":"Google Inc.","plu":["eTJjZUp","iZMOHLs1"],"ua":"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36","dvm":4,"acw":"probably","pr":3,"ts_mtp":5,"tz":-120,"emd":"k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801","cookiesStr":"app.sig=_FqS3_SPN2D40Z4Qlq-_ybxzRvQ;datadome=VSGCNMYkY6L81typRJUpurlyd39dHxdfxIYc6DzRWDT7_rtZ8UodGcbow1okQR2zg_t4d_NcP0rC1ZHNz3XSDAGFZCJhFI49lai~x2Nb2unkhOlY8o4E04Wjmde~auDZ;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiLW9jbktLVXAzMjVTOEFEdFpLUU9kSXlMIn0=;policy=accepted;lang=fr;"}""" diff --git a/models/jsdata_pojo.py b/models/jsdata_pojo.py index 7dd9525..7498c8e 100644 --- a/models/jsdata_pojo.py +++ b/models/jsdata_pojo.py @@ -265,8 +265,8 @@ class JsDataPojo: self.emt = False self.bfr = False self.dbov = False - self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbig1NDgpXShuKDMyMSkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1snXHg3M1x4NjhceDYxXHg2NFx4NmZceDc3XHg1Mlx4NmZceDZmXHg3NCddWydceDcxXHg3NVx4NjVceDcyXHg3" # 4.33.0 - self.stcfp = "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzk0NzcpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc2MDIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU1MjIw" # 4.33.0 + self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.33.1 + self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzkwMDkpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc5MDQpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU0NzY4" # 4.33.1 self.ckwa = True def to_url_encoded_json(self): diff --git a/request_sender_test.py b/request_sender_test.py index daab207..277e864 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -104,6 +104,6 @@ if __name__ == '__main__': # '~/Desktop/contact_list_2024-05-21.xlsx', # '~/Desktop/15_05_to_test.xlsx'] # file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx'] - file_list = ['~/Desktop/contact_list_2024-08-22_aol.xlsx'] - send_request_for_file_list(file_list=file_list, thread_number=5, - data_queue_name=MORNING_DATA_CACHE_2, stop_at_hour=18, stop_at_mins=30) + file_list = ['~/Desktop/gmx_ch_100_2024-06-13.xlsx'] + send_request_for_file_list(file_list=file_list, thread_number=10, + data_queue_name=MORNING_DATA_CACHE, stop_at_hour=18, stop_at_mins=30) diff --git a/workers/captcha_result_getter.py b/workers/captcha_result_getter.py index 828dfe8..424ac3e 100644 --- a/workers/captcha_result_getter.py +++ b/workers/captcha_result_getter.py @@ -105,7 +105,7 @@ class CaptchaResultGetter: return None def get_ch_raw_data_from_js_data(self, js_data: JsDataPojo, old_valid_cookie) -> str: - _tag_version = "4.33.0" + _tag_version = "4.33.1" _raw_data = "jsData={}&eventCounters=%5B%5D&jsType=ch&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv={}".format( js_data.to_url_encoded_json(), old_valid_cookie, _tag_version) print("raw ch data is " + _raw_data) @@ -172,7 +172,7 @@ class CaptchaResultGetter: # _le_js_raw_data = self.get_le_raw_data_from_js_data(js_le_type_data=js_le_type_data, # old_valid_cookie=old_valid_cookie) _cid = get_datadome_cookies(old_valid_cookie) - _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.33.0".format( + _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.33.1".format( js_le_type_data.to_url_encoded_json(), mousemove_count, click_count, scroll_count, touch_count, touch_count, touch_move, key_count, From acb10dc751bd71976b4e2654b7debd30993b8317 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Tue, 27 Aug 2024 19:03:23 +0200 Subject: [PATCH 14/16] support 4.34.0 --- docs/4_34_0/jsdata_ch_4_34_0.json | 126 ++++++++++++++++ docs/4_34_0/jsdata_ch_4_34_0_generated.json | 126 ++++++++++++++++ docs/4_34_0/jsdata_le_4_34_0.json | 151 ++++++++++++++++++++ docs/4_34_0/jsdata_le_4_34_0_generated.json | 151 ++++++++++++++++++++ models/jsdata_le_pojo.py | 4 +- models/jsdata_pojo.py | 4 +- request_sender_test.py | 6 +- workers/captcha_result_getter.py | 4 +- 8 files changed, 563 insertions(+), 9 deletions(-) create mode 100644 docs/4_34_0/jsdata_ch_4_34_0.json create mode 100644 docs/4_34_0/jsdata_ch_4_34_0_generated.json create mode 100644 docs/4_34_0/jsdata_le_4_34_0.json create mode 100644 docs/4_34_0/jsdata_le_4_34_0_generated.json diff --git a/docs/4_34_0/jsdata_ch_4_34_0.json b/docs/4_34_0/jsdata_ch_4_34_0.json new file mode 100644 index 0000000..a70b8a2 --- /dev/null +++ b/docs/4_34_0/jsdata_ch_4_34_0.json @@ -0,0 +1,126 @@ +{ + "ttst": 73, + "ifov": false, + "hc": 7, + "br_oh": 755, + "br_ow": 407, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 5.225924117686218, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 755, + "br_w": 407, + "isf": true, + "nddc": 1, + "rs_h": 904, + "rs_w": 407, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 904, + "ars_w": 407, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": false, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "t157dt15,9HqVp78", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 8, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "Z2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4MjM4OCkKICAgIGF0IDxjb21wdXRlZD4gW2FzIGRkX2FjXSAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDAzOTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU3NDM3", + "ckwa": true, + "glvd": "ARM", + "glrd": "Mali-G610 MC6", + "wwl": false, + "jset": 1724777457 +} \ No newline at end of file diff --git a/docs/4_34_0/jsdata_ch_4_34_0_generated.json b/docs/4_34_0/jsdata_ch_4_34_0_generated.json new file mode 100644 index 0000000..d3307fb --- /dev/null +++ b/docs/4_34_0/jsdata_ch_4_34_0_generated.json @@ -0,0 +1,126 @@ +{ + "ttst": 72, + "ifov": false, + "hc": 5, + "br_oh": 620, + "br_ow": 360, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 6.369887388737959, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 620, + "br_w": 360, + "isf": true, + "nddc": 1, + "rs_h": 804, + "rs_w": 360, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 804, + "ars_w": 360, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "eTJjZUp,iZMOHLs1", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 4, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "Z2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4MjM4OCkKICAgIGF0IDxjb21wdXRlZD4gW2FzIGRkX2FjXSAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDAzOTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU3NDM3", + "ckwa": true, + "glvd": "ARM", + "glrd": "Mali-G57 MC2", + "wwl": false, + "jset": 1724778146 +} diff --git a/docs/4_34_0/jsdata_le_4_34_0.json b/docs/4_34_0/jsdata_le_4_34_0.json new file mode 100644 index 0000000..67ed58c --- /dev/null +++ b/docs/4_34_0/jsdata_le_4_34_0.json @@ -0,0 +1,151 @@ +{ + "ttst": 73, + "ifov": false, + "hc": 7, + "br_oh": 755, + "br_ow": 407, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 5.225924117686218, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 755, + "br_w": 407, + "isf": true, + "nddc": 1, + "rs_h": 904, + "rs_w": 407, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 904, + "ars_w": 407, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": false, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "t157dt15,9HqVp78", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 8, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "Z2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4MjM4OCkKICAgIGF0IDxjb21wdXRlZD4gW2FzIGRkX2FjXSAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDAzOTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU3NDM3", + "ckwa": true, + "glvd": "ARM", + "glrd": "Mali-G610 MC6", + "wwl": false, + "jset": 1724777475, + "emd": "k:ai,vi,ao", + "dcok": ".hermes.com", + "log2": true, + "m_fmi": false, + "tbce": 0, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "k_hA": null, + "k_hSD": null, + "k_pA": null, + "k_pSD": null, + "k_rA": null, + "k_rSD": null, + "k_ikA": null, + "k_ikSD": null, + "k_kdc": 0, + "k_kuc": 0, + "m_s_c": 23, + "m_m_c": 5, + "m_c_c": 7, + "m_cm_r": 1.4, + "m_ms_r": 0.21739130434782608 +} \ No newline at end of file diff --git a/docs/4_34_0/jsdata_le_4_34_0_generated.json b/docs/4_34_0/jsdata_le_4_34_0_generated.json new file mode 100644 index 0000000..d24c016 --- /dev/null +++ b/docs/4_34_0/jsdata_le_4_34_0_generated.json @@ -0,0 +1,151 @@ +{ + "ttst": 148, + "ifov": false, + "hc": 5, + "br_oh": 620, + "br_ow": 360, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 7.793792317196083, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 620, + "br_w": 360, + "isf": true, + "nddc": 1, + "rs_h": 804, + "rs_w": 360, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 804, + "ars_w": 360, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "eTJjZUp,iZMOHLs1", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 4, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "Z2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4MjM4OCkKICAgIGF0IDxjb21wdXRlZD4gW2FzIGRkX2FjXSAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDAzOTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU3NDM3", + "ckwa": true, + "glvd": "ARM", + "glrd": "Mali-G57 MC2", + "wwl": false, + "jset": 1724777863, + "dcok": ".hermes.com", + "log2": true, + "emd": "k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801", + "m_fmi": false, + "tbce": 0, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "k_hA": null, + "k_hSD": null, + "k_pA": null, + "k_pSD": null, + "k_rA": null, + "k_rSD": null, + "k_ikA": null, + "k_ikSD": null, + "k_kdc": 0, + "k_kuc": 0, + "m_s_c": 0, + "m_m_c": 3, + "m_c_c": 3, + "m_cm_r": 1, + "m_ms_r": -1 +} diff --git a/models/jsdata_le_pojo.py b/models/jsdata_le_pojo.py index 24f2997..dd40cf4 100644 --- a/models/jsdata_le_pojo.py +++ b/models/jsdata_le_pojo.py @@ -48,8 +48,8 @@ class JsDataLeTypePojo(JsDataPojo): self.m_c_c = m_c_c self.m_cm_r = m_cm_r self.m_ms_r = m_ms_r - self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.33.1 - self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzkwMDkpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc5MDQpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU0NzY4" # 4.33.1 + self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.34.0 + self.stcfp = "Z2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4MjM4OCkKICAgIGF0IDxjb21wdXRlZD4gW2FzIGRkX2FjXSAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDAzOTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU3NDM3" # 4.34.0 test_data_json = """{"glvd":"ARM","glrd":"Mali-G57 MC2","hc":5,"br_oh":620,"br_ow":360,"br_h":620,"br_w":360,"rs_h":804,"rs_w":360,"rs_cd":24,"ars_h":804,"ars_w":360,"plg":2,"eva":33,"vnd":"Google Inc.","plu":["eTJjZUp","iZMOHLs1"],"ua":"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36","dvm":4,"acw":"probably","pr":3,"ts_mtp":5,"tz":-120,"emd":"k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801","cookiesStr":"app.sig=_FqS3_SPN2D40Z4Qlq-_ybxzRvQ;datadome=VSGCNMYkY6L81typRJUpurlyd39dHxdfxIYc6DzRWDT7_rtZ8UodGcbow1okQR2zg_t4d_NcP0rC1ZHNz3XSDAGFZCJhFI49lai~x2Nb2unkhOlY8o4E04Wjmde~auDZ;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiLW9jbktLVXAzMjVTOEFEdFpLUU9kSXlMIn0=;policy=accepted;lang=fr;"}""" diff --git a/models/jsdata_pojo.py b/models/jsdata_pojo.py index 7498c8e..fa33f06 100644 --- a/models/jsdata_pojo.py +++ b/models/jsdata_pojo.py @@ -265,8 +265,8 @@ class JsDataPojo: self.emt = False self.bfr = False self.dbov = False - self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.33.1 - self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzkwMDkpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTc5MDQpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU0NzY4" # 4.33.1 + self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.34.0 + self.stcfp = "Z2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4MjM4OCkKICAgIGF0IDxjb21wdXRlZD4gW2FzIGRkX2FjXSAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDAzOTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU3NDM3" # 4.34.0 self.ckwa = True def to_url_encoded_json(self): diff --git a/request_sender_test.py b/request_sender_test.py index 277e864..aab51eb 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -104,6 +104,6 @@ if __name__ == '__main__': # '~/Desktop/contact_list_2024-05-21.xlsx', # '~/Desktop/15_05_to_test.xlsx'] # file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx'] - file_list = ['~/Desktop/gmx_ch_100_2024-06-13.xlsx'] - send_request_for_file_list(file_list=file_list, thread_number=10, - data_queue_name=MORNING_DATA_CACHE, stop_at_hour=18, stop_at_mins=30) + file_list = ['~/Desktop/contact_list_all.xlsx'] + send_request_for_file_list(file_list=file_list, thread_number=1, + data_queue_name=MORNING_DATA_CACHE_2, stop_at_hour=18, stop_at_mins=30) diff --git a/workers/captcha_result_getter.py b/workers/captcha_result_getter.py index 424ac3e..f990e28 100644 --- a/workers/captcha_result_getter.py +++ b/workers/captcha_result_getter.py @@ -105,7 +105,7 @@ class CaptchaResultGetter: return None def get_ch_raw_data_from_js_data(self, js_data: JsDataPojo, old_valid_cookie) -> str: - _tag_version = "4.33.1" + _tag_version = "4.34.0" _raw_data = "jsData={}&eventCounters=%5B%5D&jsType=ch&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv={}".format( js_data.to_url_encoded_json(), old_valid_cookie, _tag_version) print("raw ch data is " + _raw_data) @@ -172,7 +172,7 @@ class CaptchaResultGetter: # _le_js_raw_data = self.get_le_raw_data_from_js_data(js_le_type_data=js_le_type_data, # old_valid_cookie=old_valid_cookie) _cid = get_datadome_cookies(old_valid_cookie) - _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.33.1".format( + _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.34.0".format( js_le_type_data.to_url_encoded_json(), mousemove_count, click_count, scroll_count, touch_count, touch_count, touch_move, key_count, From 442020d74a92f4fc2d1e6d01c94633c738eb500b Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Thu, 29 Aug 2024 16:42:33 +0200 Subject: [PATCH 15/16] support urlEncoded subject --- mail/mail_reader_all_contacts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mail/mail_reader_all_contacts.py b/mail/mail_reader_all_contacts.py index adead85..75bc7ac 100755 --- a/mail/mail_reader_all_contacts.py +++ b/mail/mail_reader_all_contacts.py @@ -179,12 +179,14 @@ class MailReader(): body = body + part.get_payload(decode=True).decode("utf-8") elif part.get_content_type() == "text/plain": body = body + part.get_payload() - if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject: + if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject or "Validation=20de=20votre=20demande=20de=20rendez-vous" in subject: mail = MailPojo(subject=subject, body=body, from_address=from_address) mail.isImapClient = True print("email is {}".format(self.login)) print("body is {}".format(body)) print("subject is {}".format(subject)) + if len(mail.to_address) == 0: + mail.to_address = self.login mail_messages.append(mail) except Exception as error: print(error) From 0312b5bd7ef7a24cc8540b38df213b5ba8267586 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Thu, 29 Aug 2024 23:26:19 +0200 Subject: [PATCH 16/16] support urlEncoded subject --- request_sender_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request_sender_test.py b/request_sender_test.py index aab51eb..fcdb411 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -105,5 +105,5 @@ if __name__ == '__main__': # '~/Desktop/15_05_to_test.xlsx'] # file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx'] file_list = ['~/Desktop/contact_list_all.xlsx'] - send_request_for_file_list(file_list=file_list, thread_number=1, + send_request_for_file_list(file_list=file_list, thread_number=20, data_queue_name=MORNING_DATA_CACHE_2, stop_at_hour=18, stop_at_mins=30)