do not read email for successful requests

docs/generate_jsdata.json

@@ -0,0 +1,127 @@
+{
+  "opts": "endpoint,ajaxListenerPath",
+  "ttst": 93,
+  "ifov": false,
+  "hc": 2,
+  "br_oh": 663,
+  "br_ow": 384,
+  "ua": "Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36",
+  "wbd": false,
+  "tagpu": 9.185432887936322,
+  "wdif": false,
+  "wdifrm": false,
+  "npmtm": false,
+  "br_h": 663,
+  "br_w": 384,
+  "isf": true,
+  "nddc": 1,
+  "rs_h": 811,
+  "rs_w": 384,
+  "rs_cd": 24,
+  "phe": false,
+  "nm": false,
+  "jsf": false,
+  "lg": "fr-FR",
+  "pr": 2.8125,
+  "ars_h": 811,
+  "ars_w": 384,
+  "tz": -60,
+  "str_ss": true,
+  "str_ls": true,
+  "str_idb": true,
+  "str_odb": true,
+  "plgod": false,
+  "plg": 2,
+  "plgne": "err",
+  "plgre": "err",
+  "plgof": "err",
+  "plggt": "err",
+  "pltod": false,
+  "hcovdr": false,
+  "hcovdr2": false,
+  "plovdr": false,
+  "plovdr2": false,
+  "ftsovdr": false,
+  "ftsovdr2": false,
+  "lb": false,
+  "eva": 33,
+  "lo": false,
+  "ts_mtp": 5,
+  "ts_tec": true,
+  "ts_tsa": true,
+  "vnd": "Google Inc.",
+  "bid": "NA",
+  "mmt": "empty",
+  "plu": "5gQIEKN,Iv26GLka",
+  "hdn": false,
+  "awe": false,
+  "geb": false,
+  "dat": false,
+  "med": "defined",
+  "aco": "probably",
+  "acots": false,
+  "acmp": "probably",
+  "acmpts": true,
+  "acw": "probably",
+  "acwts": false,
+  "acma": "maybe",
+  "acmats": false,
+  "acaa": "probably",
+  "acaats": true,
+  "ac3": "",
+  "ac3ts": false,
+  "acf": "probably",
+  "acfts": false,
+  "acmp4": "maybe",
+  "acmp4ts": false,
+  "acmp3": "probably",
+  "acmp3ts": false,
+  "acwm": "maybe",
+  "acwmts": false,
+  "ocpt": false,
+  "vco": "",
+  "vcots": false,
+  "vch": "probably",
+  "vchts": true,
+  "vcw": "probably",
+  "vcwts": true,
+  "vc3": "maybe",
+  "vc3ts": false,
+  "vcmp": "",
+  "vcmpts": false,
+  "vcq": "",
+  "vcqts": false,
+  "vc1": "probably",
+  "vc1ts": true,
+  "dvm": 4,
+  "sqt": false,
+  "so": "portrait-primary",
+  "wdw": true,
+  "cokys": "bG9hZFRpbWVzY3NpL=",
+  "ecpc": false,
+  "lgs": true,
+  "lgsod": false,
+  "psn": true,
+  "edp": false,
+  "addt": true,
+  "wsdc": true,
+  "ccsr": true,
+  "nuad": true,
+  "bcda": true,
+  "idn": true,
+  "capi": false,
+  "svde": false,
+  "vpbq": true,
+  "ucdv": false,
+  "spwn": false,
+  "emt": false,
+  "bfr": false,
+  "dbov": false,
+  "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2",
+  "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzMDIpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWIgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTA2MjIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxNzMz",
+  "ckwa": true,
+  "glrd": "Adreno (TM) 630",
+  "glvd": "Qualcomm",
+  "wwl": false,
+  "jset": 1710598314
+}

excel_reader.py

@@ -30,6 +30,7 @@ def read_contacts(file_name) -> list:
                                   first_name=first_name,
                                   passport_number=contact_dict['passport'],
                                   mail=contact_dict['email'])
+            contact.store = contact_dict['store']
             contact_list.append(contact)
     return contact_list
 

models/contact_pojo.py

@@ -10,6 +10,7 @@ class ContactPojo:
     mail: str
     ccid: str
     position: int
+    store: str
     note: str
 
     def __init__(self, phone_number: str, passport_number: str, last_name: str, first_name: str, mail: str,
@@ -31,6 +32,7 @@ class ContactPojo:
             u'last_name': self.last_name,
             u'first_name': self.first_name,
             u'mail': self.mail,
+            u'store': self.store,
             u'ccid': self.ccid,
             u'position': self.position
         }

models/jsdata_pojo.py

@@ -9,7 +9,7 @@ import jsonpickle
 
 @dataclass
 class JsDataPojo:
-    opts: str
+    # opts: str
     ttst: int
     ifov: bool
     hc: int
@@ -137,7 +137,7 @@ class JsDataPojo:
 
     def __init__(self, glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu,
                  vnd, pr, ts_mtp, dvm):
-        self.opts = "endpoint,ajaxListenerPath"
+        # self.opts = "endpoint,ajaxListenerPath"
         self.ttst = random.randint(38, 148)
         self.ifov = False
         self.hc = hc
@@ -265,8 +265,8 @@ class JsDataPojo:
         self.emt = False
         self.bfr = False
         self.dbov = False
-        self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigzODYpXSgnXHg2Mlx4NzJceDZmXHg3N1x4NzNceDY1XHg3Mlx4NjZceDZjXHg2Zlx4NzdceDJkXHg2M1x4NmZceDZlXHg3NFx4NjFceDY5XHg2ZVx4NjVceDcyJyk7aWYobyl7IWZ1bmN0aW9uIHQoKXt2YXIg"  # 4.24.3
+        self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2"  # 4.25.0
-        self.stcfp = "dy5oZXJtZXMuY29tL2ZyL2ZyL3BvbHlmaWxscy43ZTFjYzMyM2Y3OWMwNjZlLmpzOjE6ODI2MykKICAgIGF0IG0uYXJncy48Y29tcHV0ZWQ+IChodHRwczovL3d3dy5oZXJtZXMuY29tL2ZyL2ZyL3BvbHlmaWxscy43ZTFjYzMyM2Y3OWMwNjZlLmpzOjE6Mjc5NTEp"  # 4.24.3
+        self.stcfp = "dy5oZXJtZXMuY29tL2ZyL2ZyL3BvbHlmaWxscy43ZTFjYzMyM2Y3OWMwNjZlLmpzOjE6ODI2MykKICAgIGF0IG0uYXJncy48Y29tcHV0ZWQ+IChodHRwczovL3d3dy5oZXJtZXMuY29tL2ZyL2ZyL3BvbHlmaWxscy43ZTFjYzMyM2Y3OWMwNjZlLmpzOjE6Mjc5NTEp"  # 4.25.0
         self.ckwa = True
 
     def to_url_encoded_json(self):

queue_message/appointmentrequestsender.py

@@ -34,6 +34,7 @@ def is_already_sent(contact: ContactPojo) -> bool:
 def filter_contacts(_contact_list: list, provided_list=[]) -> list:
     already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
     already_sent_contacts.extend(provided_list)
+    _link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
     _contact_list_to_book = []
     for contact in _contact_list:
         _to_add = True
@@ -44,6 +45,11 @@ def filter_contacts(_contact_list: list, provided_list=[]) -> list:
             else:
                 if contact.mail == booked.mail:
                     _to_add = False
+        # 如果已经收到链接了，就不要再请求
+        for link_to_validate in _link_to_validate_list:
+            if contact.mail == link_to_validate.email:
+                print("{}: link already received".format(contact.mail))
+                _to_add = False
         if _to_add:
             _contact_list_to_book.append(contact)
 
@@ -127,7 +133,7 @@ class AppointmentRequestSender(threading.Thread):
                     _new_cookies = captchaResultGetter.get_valid_ch_cookie(sender.proxy_to_use, js_data,
                                                                            old_valid_cookie=_received_cookies)
                     if _new_cookies is not None:
-                        # self.logger.info("new cookie is " + _new_cookies)
+                        self.logger.info("new cookie is " + _new_cookies)
                         time.sleep(random.randint(1, 3))
                         # m_s_c = f.scroll
                         m_s_c = random.randint(0, 3)
@@ -160,6 +166,14 @@ class AppointmentRequestSender(threading.Thread):
                             time.sleep(random.randint(1, 3))
                             self.already_tried_contact_list.append(con)
                             can_continue = sender.send_request(HERMES_REGISTER, js_data, con, csrf=self.valid_csrf)
+                            if can_continue == RequestResult.SUCCESS:
+                                # 让服务器读取成功的约会
+                                try:
+                                    self.logger.info("try to remove success contact from list to retrieve mails")
+                                    self.list_to_retrieve_mails.remove(con)
+                                except Exception as e:
+                                    self.logger.info("exception while remove success contact from list to retrieve mails")
+                                    print(e)
                         else:
                             can_continue = RequestResult.COOKIES_ERROR
                     else:
@@ -196,6 +210,7 @@ class AppointmentRequestSender(threading.Thread):
     def retreive_invalidate_urls(self):
         if not self.already_read_emails:
             self.logger.info("will retreive_invalidate_urls")
+            time.sleep(30)
             _mail_list = MONGO_STORE_MANAGER.get_destination_emails()
             _mail_list_filtred = []
             for mail in _mail_list:

request_sender.py

@@ -26,12 +26,18 @@ def is_already_sent(contact: ContactPojo) -> bool:
 
 def filter_contacts(_contact_list: list) -> list:
     already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+    _link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
     _contact_list_to_book = []
     for contact in _contact_list:
         _to_add = True
         for booked in already_sent_contacts:
             if contact.mail == booked.email:
                 _to_add = False
+        #如果已经收到链接了，就不要再请求
+        for link_to_validate in _link_to_validate_list:
+            if contact.mail == link_to_validate.email:
+                logger.info("{}: link already received".format(contact.mail))
+                _to_add = False
         if _to_add:
             _contact_list_to_book.append(contact)
 
@@ -64,10 +70,10 @@ def send_appointment_request(message_queue_name, _contact_list):
 
 
 if __name__ == '__main__':
-    contacts_file_path = '~/Desktop/contact_list_2024-03-26-2.xlsx'
+    contacts_file_path = '~/Desktop/contact_list_2024-03-25.xlsx'
     _contact_list = read_contacts(contacts_file_path)
     _contact_list_to_book = filter_contacts(_contact_list)
-    _segment_number = 5
+    _segment_number = 2
     logger.info("{} contacts to book".format(len(_contact_list_to_book)))
     last_thread = None
     for i in range(0, _segment_number):

workers/captcha_result_getter.py

@@ -101,7 +101,7 @@ class CaptchaResultGetter:
             return None
 
     def get_ch_raw_data_from_js_data(self, js_data: JsDataPojo, old_valid_cookie) -> str:
-        _tag_version = "4.24.3"
+        _tag_version = "4.25.0"
         _raw_data = "jsData={}&eventCounters=%5B%5D&jsType=ch&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv={}".format(
             js_data.to_url_encoded_json(), old_valid_cookie, _tag_version)
         print("raw data is " + _raw_data)
@@ -168,7 +168,7 @@ class CaptchaResultGetter:
         # _le_js_raw_data = self.get_le_raw_data_from_js_data(js_le_type_data=js_le_type_data,
         #                                                     old_valid_cookie=old_valid_cookie)
         _cid = get_datadome_cookies(old_valid_cookie)
-        _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.24.3".format(
+        _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.25.0".format(
             js_le_type_data.to_url_encoded_json(), mousemove_count, click_count, scroll_count, touch_count, touch_count,
             touch_move,
             key_count,

workers/proxies_constants.py

@@ -72,7 +72,4 @@ DE_PROXY_RES = {
 # PROXY_LIST_FR = [FR_PROXY_MOBILE_2, FR_PROXY_MOBILE, FR_PROXY_RES, FR_PROXY_RES_4, FR_PROXY_RES_2]
 # PROXY_LIST_FR = [FR_PROXY_RES_OXY, FR_PROXY_MOBILE_3]
 PROXY_LIST_FR = [FR_PROXY_MOB_OXY]
-MOBILE_PROXY_LIST_FR = [FR_PROXY_MOB_OXY_STICKY]
+MOBILE_PROXY_LIST_FR = [FR_PROXY_MOB_OXY]
-# PROXY_LIST = [FR_PROXY_MOBILE, FR_
-# PROXY_RES]
-PROXY_LIST_DE = [DE_PROXY_RES, DE_PROXY_MOBILE]

workers/sender.py

@@ -28,7 +28,7 @@ class Sender:
         self.proxy_to_use = proxy_to_use
         self.cookie.load(self.cookie_str)
 
-    def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str):
+    def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str, store_type: str):
         # create the message
         if url == "https://rendezvousparis.hermes.com/client/welcome":
             return
@@ -37,7 +37,7 @@ class Sender:
                                    firstName=contact.first_name, lastName=contact.last_name, email=contact.mail,
                                    passport=contact.passport, ccid=contact.ccid)
         result.id = id
-        result.store_type = self.store_type
+        result.store_type = store_type
         result.created_at = time.strftime("%H:%M:%S", time.localtime())
         collection_name = str(datetime.date.today())
         MONGO_STORE_MANAGER.insert_reserve_result(collection_name=collection_name, reserve=result)
@@ -57,6 +57,11 @@ class Sender:
         else:
             self._csrf = csrf
             _csrf = csrf
+        # 默认为空，也就是随机
+        _selected_store = ''
+        if contact.store is not None and contact.store != "random":
+            _selected_store = contact.store
+        self.logger.info("store is:{}".format(_selected_store))
         headers = {'Content-Type': 'application/x-www-form-urlencoded',
                    'User-Agent': js_data.ua,
                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
@@ -64,7 +69,7 @@ class Sender:
                    'Sec-Fetch-Mode': 'navigate',
                    'Sec-Fetch-Dest': 'document',
                    'Accept-Language': 'fr-FR,fr;q=0.6'}
-        data = {'check': '', '_csrf': _csrf, 'prefer': '',
+        data = {'check': '', '_csrf': _csrf, 'prefer': _selected_store,
                 'surname': contact.last_name.capitalize(), 'name': contact.first_name.capitalize(),
                 'phone_country': "FR", 'phone_number': "+33 " + contact.phone, 'email': contact.mail,
                 'passport_id': contact.passport, 'processing': 'on', 'cgu': 'on'}
@@ -81,7 +86,8 @@ class Sender:
                 # add to mongodb
                 self.logger.info(response.text)
                 self.logger.info("{}:{}".format(contact.mail, response.url))
-                self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=response.url)
+                self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=response.url,
+                                              store_type=_selected_store)
                 cookies_to_set = response.headers['set-cookie']
                 self.cookie.load(cookies_to_set)
                 new_cookies = {k: v.value for k, v in self.cookie.items()}