Merge branch 'develop' into feature/proxy_manager

db/mongo_manager.py

@@ -7,6 +7,7 @@ from pymongo import MongoClient
 from models.LinkPojo import LinkPojo
 from models.ReserveResultPojo import ReserveResultPojo
 from models.contact_pojo import ContactPojo
+from models.mail_pojo import MailAddress
 
 MONGO_DB_URL = "mongo.lpaconsulting.fr"
 CAPTCHA_ERROR_COLLECTION_PREFIX = "CAPTCHA_ERROR_"
@@ -72,6 +73,43 @@ class MongoDbManager:
             result_list.append(ContactPojo.from_firestore_dict(document))
         return result_list
 
+    def save_links_to_validate(self, link: str, mail_address: str, _all_contact_list: list):
+        collection_to_use = self.db[LINKS_TO_VALIDATE]
+        updated_at = time.strftime("%H:%M:%S", time.localtime())
+        _ip_country = "FR"
+        # find ip_country info
+        for _contact in _all_contact_list:
+            if _contact.mail == mail_address:
+                _ip_country = _contact.ip_country
+
+        if len(mail_address) > 0:
+            collection_to_use.replace_one(filter={'_id': mail_address, }, replacement={
+                u'url': link,
+                u'email': mail_address,
+                u'ip_country': _ip_country,
+                "updated_at": updated_at
+            },
+                                          upsert=True)
+        else:
+            collection_to_use.replace_one(filter={'_id': link, }, replacement={
+                u'url': link,
+                u'ip_country': _ip_country,
+                "updated_at": updated_at
+            },
+                                          upsert=True)
+
+    def get_destination_emails(self) -> list:
+        collection_name = DESTINATION_EMAIL_LIST
+        email_list = []
+        try:
+            collection_to_use = self.db[collection_name]
+            for document in collection_to_use.find():
+                email_list.append(MailAddress.from_firestore_dict(document))
+        except Exception as error:
+            self.logger.info(error)
+            print(error)
+        return email_list
+
     def get_links_to_validate(self) -> list:
         collection_name = LINKS_TO_VALIDATE
         link_list = []

docs/generate_jsdata.json

@@ -0,0 +1,127 @@
+{
+  "opts": "endpoint,ajaxListenerPath",
+  "ttst": 93,
+  "ifov": false,
+  "hc": 2,
+  "br_oh": 663,
+  "br_ow": 384,
+  "ua": "Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36",
+  "wbd": false,
+  "tagpu": 9.185432887936322,
+  "wdif": false,
+  "wdifrm": false,
+  "npmtm": false,
+  "br_h": 663,
+  "br_w": 384,
+  "isf": true,
+  "nddc": 1,
+  "rs_h": 811,
+  "rs_w": 384,
+  "rs_cd": 24,
+  "phe": false,
+  "nm": false,
+  "jsf": false,
+  "lg": "fr-FR",
+  "pr": 2.8125,
+  "ars_h": 811,
+  "ars_w": 384,
+  "tz": -60,
+  "str_ss": true,
+  "str_ls": true,
+  "str_idb": true,
+  "str_odb": true,
+  "plgod": false,
+  "plg": 2,
+  "plgne": "err",
+  "plgre": "err",
+  "plgof": "err",
+  "plggt": "err",
+  "pltod": false,
+  "hcovdr": false,
+  "hcovdr2": false,
+  "plovdr": false,
+  "plovdr2": false,
+  "ftsovdr": false,
+  "ftsovdr2": false,
+  "lb": false,
+  "eva": 33,
+  "lo": false,
+  "ts_mtp": 5,
+  "ts_tec": true,
+  "ts_tsa": true,
+  "vnd": "Google Inc.",
+  "bid": "NA",
+  "mmt": "empty",
+  "plu": "5gQIEKN,Iv26GLka",
+  "hdn": false,
+  "awe": false,
+  "geb": false,
+  "dat": false,
+  "med": "defined",
+  "aco": "probably",
+  "acots": false,
+  "acmp": "probably",
+  "acmpts": true,
+  "acw": "probably",
+  "acwts": false,
+  "acma": "maybe",
+  "acmats": false,
+  "acaa": "probably",
+  "acaats": true,
+  "ac3": "",
+  "ac3ts": false,
+  "acf": "probably",
+  "acfts": false,
+  "acmp4": "maybe",
+  "acmp4ts": false,
+  "acmp3": "probably",
+  "acmp3ts": false,
+  "acwm": "maybe",
+  "acwmts": false,
+  "ocpt": false,
+  "vco": "",
+  "vcots": false,
+  "vch": "probably",
+  "vchts": true,
+  "vcw": "probably",
+  "vcwts": true,
+  "vc3": "maybe",
+  "vc3ts": false,
+  "vcmp": "",
+  "vcmpts": false,
+  "vcq": "",
+  "vcqts": false,
+  "vc1": "probably",
+  "vc1ts": true,
+  "dvm": 4,
+  "sqt": false,
+  "so": "portrait-primary",
+  "wdw": true,
+  "cokys": "bG9hZFRpbWVzY3NpL=",
+  "ecpc": false,
+  "lgs": true,
+  "lgsod": false,
+  "psn": true,
+  "edp": false,
+  "addt": true,
+  "wsdc": true,
+  "ccsr": true,
+  "nuad": true,
+  "bcda": true,
+  "idn": true,
+  "capi": false,
+  "svde": false,
+  "vpbq": true,
+  "ucdv": false,
+  "spwn": false,
+  "emt": false,
+  "bfr": false,
+  "dbov": false,
+  "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2",
+  "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzMDIpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWIgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTA2MjIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxNzMz",
+  "ckwa": true,
+  "glrd": "Adreno (TM) 630",
+  "glvd": "Qualcomm",
+  "wwl": false,
+  "jset": 1710598314
+}

excel_reader.py

@@ -30,6 +30,7 @@ def read_contacts(file_name) -> list:
                                   first_name=first_name,
                                   passport_number=contact_dict['passport'],
                                   mail=contact_dict['email'])
+            contact.store = contact_dict['store']
             contact_list.append(contact)
     return contact_list
 

mail/mail_constants.py

@@ -0,0 +1,106 @@
+import imaplib
+
+from imapclient import IMAPClient
+
+DOMAIN_YAHOO = "yahoo.com"
+DOMAIN_SINA = "sina.com"
+DOMAIN_HOTMAIL = "hotmail.com"
+DOMAIN_TIM_IT = "tim.it"
+DOMAIN_163 = "163.com"
+DOMAIN_RAMBLER_RU = "rambler.ru"
+DOMAIN_ALICE_IT = "alice.it"
+DOMAIN_MARS_DTI_NE_JP = "mars.dti.ne.jp"
+DOMAN_BTVM_NE_JP = "btvm.ne.jp"
+DOMAN_AURORA_DTI_NE_JP = "aurora.dti.ne.jp"
+DOMAN_GMAIL = "gmail.com"
+DOMAIN_GMX = "gmx.com"
+DOMAIN_GMX_FR = "gmx.fr"
+DOMAIN_GMX_US = "gmx.us"
+DOMAIN_GMX_CH = "gmx.ch"
+DOMAIN_ONET = "onet.pl"
+DOMAIN_NAVER = "naver.com"
+DOMAIN_INBOX_LV = "inbox.lv"
+
+DOMAIN_PISS_MAIL = "pissmail.com"
+DOMAIN_INCEL_EMAIL = "incel.email"
+DOMAIN_SHITPOSTING_EXPERT = "shitposting.expert"
+DOMAIN_HATESJE_WS = "hatesje.ws"
+DOMAIN_CHILD_PIZZA = "child.pizza"
+DOMAIN_GENOCIDE_FUN = "genocide.fun"
+DOMAIN_DMC_CHAT = "dmc.chat"
+
+AOL_IMAP_SERVER = "imap.aol.com"
+IMAP_SERVER_163 = "imap.163.com"
+IMAP_SERVER_SINA = "imap.sina.com"
+YAHOO_IMAP_SERVER = "imap.mail.yahoo.com"
+HOTMAIL_IMAP_SERVER = "outlook.office365.com"
+RAMBLER_IMAP_SERVER = "imap.rambler.ru"
+ALICE_IMAP_SERVER = "in.alice.it"
+TIME_IT_SERVER = "imap.tim.it"
+MARS_DTI_NE_JP_SERVER = "imap.cm.dream.jp"
+NAVER_SERVER = "imap.naver.com"
+BTVM_NE_JP = "imap.btvm.ne.jp"
+SEREVER_GMAIL = "imap.gmail.com"
+SERVER_IMAGE_ONET = "imap.poczta.onet.pl"
+SERVER_GMX = "imap.gmx.com"
+SERVER_PISS_MAIL = "mail.pissmail.com"
+INBOX_LV = "mail.inbox.lv"
+
+
+def show_folders(imap) -> list:
+    folders = []
+    isImapClient = isinstance(imap, IMAPClient)
+    if not isImapClient:
+        for i in imap.list()[1]:
+            l = i.decode().split(' "/" ')
+            if len(l) > 1:
+                folders.append(l[1])
+        if len(folders) == 0:
+            folders.append('INBOX')
+        return folders
+    else:
+        list = imap.list_folders()
+        for i in list:
+            name = i[-1]
+            folders.append(name)
+        return folders
+
+
+def create_imap(login: str):
+    # create an IMAP4 class with SSL
+    if DOMAIN_163 in login:
+        imap = IMAPClient(IMAP_SERVER_163, use_uid=True)
+    elif DOMAIN_YAHOO in login:
+        imap = imaplib.IMAP4_SSL(YAHOO_IMAP_SERVER)
+    elif DOMAIN_GMX in login or DOMAIN_GMX_FR in login or DOMAIN_GMX_US in login or DOMAIN_GMX_CH in login:
+        imap = imaplib.IMAP4_SSL(SERVER_GMX)
+    elif DOMAIN_SINA in login:
+        # imap = imaplib.IMAP4_SSL(IMAP_SERVER_SINA)
+        imap = IMAPClient(IMAP_SERVER_SINA, use_uid=True)
+    elif DOMAIN_HOTMAIL in login:
+        imap = imaplib.IMAP4_SSL(HOTMAIL_IMAP_SERVER)
+    elif DOMAIN_RAMBLER_RU in login:
+        imap = imaplib.IMAP4_SSL(RAMBLER_IMAP_SERVER)
+    elif DOMAN_BTVM_NE_JP in login:
+        imap = imaplib.IMAP4_SSL(BTVM_NE_JP)
+    elif DOMAN_GMAIL in login:
+        imap = imaplib.IMAP4_SSL(SEREVER_GMAIL, port=993)
+    elif DOMAIN_ONET in login:
+        imap = IMAPClient(SERVER_IMAGE_ONET, use_uid=True)
+    elif DOMAIN_TIM_IT in login:
+        imap = imaplib.IMAP4(TIME_IT_SERVER)
+    elif DOMAIN_ALICE_IT in login:
+        imap = imaplib.IMAP4(ALICE_IMAP_SERVER, port=143)
+    elif DOMAIN_MARS_DTI_NE_JP in login:
+        imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143)
+    elif DOMAN_AURORA_DTI_NE_JP in login:
+        imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143)
+    elif DOMAIN_NAVER in login:
+        imap = imaplib.IMAP4_SSL(NAVER_SERVER, port=993)
+    elif DOMAIN_INBOX_LV in login:
+        imap = imaplib.IMAP4_SSL(INBOX_LV, port=993)
+    elif DOMAIN_PISS_MAIL in login or DOMAIN_CHILD_PIZZA in login or DOMAIN_DMC_CHAT in login or DOMAIN_GENOCIDE_FUN in login or DOMAIN_HATESJE_WS in login or DOMAIN_INCEL_EMAIL in login or DOMAIN_SHITPOSTING_EXPERT in login:
+        imap = imaplib.IMAP4_SSL(SERVER_PISS_MAIL, port=993)
+    else:
+        imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER)
+    return imap

mail/mail_reader_all_contacts.py

@@ -0,0 +1,265 @@
+import datetime
+import email
+import re
+from builtins import list
+from concurrent.futures import ThreadPoolExecutor
+from datetime import time
+from email.header import decode_header
+from email.message import Message
+
+from imapclient import IMAPClient
+from db.mongo_manager import MONGO_STORE_MANAGER
+from mail.mail_constants import DOMAIN_HOTMAIL, create_imap
+from models.mail_pojo import MailPojo
+
+VALIDATION_URL_SUBJECT_fr = 'Validation de votre demande de rendez-vous'
+VALIDATION_URL_SUBJECT_EN = 'Please confirm your appointment request'
+VALIDATION_URL_REGEX = """https:\/\/rendezvousparis.hermes.com\/client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+"""
+PART_VALIDATION_URL_REGEX = """client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+"""
+HERMES_EMAIL = "no-reply@hermes.com"
+EMAIL_ADDRESS_REGEX = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b'
+
+date_format = "%d-%b-%Y"  # DD-Mon-YYYY e.g., 3-Mar-2014
+REDIRECTION_MAILS = "appointment2022@aol.com, chenpeijun@aol.com,hongjiang176@aol.com,ciyuexie@aol.com,rutger.62@aol.com,ciccidaniel@aol.com,armasgoodman@aol.com,wknd.gemerine@aol.com,rafmail1981@aol.com,tonovichivanenaki@aol.com,hetland.ari@aol.com,mateusiversen@aol.com,lacerdaraffaello@aol.com,anasida76@aol.com,liamolinari@aol.com,sen70zib@aol.com,mezeiderrick@aol.com,stanisl49avchic@aol.com,damcvrobaneuron@aol.com,suyzanna_fleona@aol.com,dxealing.dissa@aol.com,hogg.karen@aol.com,obocharovamarina@aol.com,buchholzjohann@aol.com,orn.cecchini@aol.com,percivaltorgersen@aol.com,candalgudrun@aol.com,filimonis.76@aol.com,bengann_100@aol.com,axelhanne@aol.com,tiffanylarochelle@aol.com,nicoleta.r@aol.com,eichenbaum.1963@aol.com,kotensasharev@aol.com,samognat32@aol.com,edem_headshot@aol.com,kozmakuzmich1960@aol.com,damonsvensson@aol.com,anders.riva@aol.com,caiminwei123@gmail.com,yulingguo086@gmail.com,yingxiaolu086@gmail.com,lijiazhen0035@gmail.com,fangp370@gmail.com,huangyayu10086@gmail.com,fuziyuan110@gmail.com,xinyingdu886@gmail.com,yasiaforever.1971@aol.com,lukaszfidalgo@aol.com,zaichi29@aol.com,prostotakitak.1974@aol.com,mo90nroe@aol.com,blonde.87@aol.com,dimidrol.1969@aol.com"
+
+
+def check_email_address(email):
+    # pass the regular expression
+    # and the string into the fullmatch() method
+    if (re.fullmatch(EMAIL_ADDRESS_REGEX, email)):
+        print("Valid Email")
+        return True
+    else:
+        print("Invalid Email")
+        return False
+
+
+def find_from_mail(param):
+    from_address, encoded_algo = param[0]
+    if isinstance(from_address, bytes):
+        from_address = from_address.decode(encoded_algo)
+        if not check_email_address(from_address) and len(param) == 2:
+            from_address, new_encode = param[1]
+            if new_encode is None:
+                new_encode = encoded_algo
+            if isinstance(from_address, bytes):
+                from_address = from_address.decode(new_encode)
+                return from_address.strip(" ").strip(">").strip("<")
+    return from_address.strip(" ").strip(">").strip("<")
+
+
+class MailReader():
+    def __init__(self, login, password):
+        self.login = login
+        self.password = password
+
+    @staticmethod
+    def show_folders(imap) -> list:
+        folders = []
+        isImapClient = isinstance(imap, IMAPClient)
+        if not isImapClient:
+            for i in imap.list()[1]:
+                l = i.decode().split(' "/" ')
+                folders.append(l[1])
+            return folders
+        else:
+            list = imap.list_folders()
+            for i in list:
+                name = i[-1]
+                folders.append(name)
+            return folders
+
+    def read_emails(self, mails_messages: list) -> list:
+        imap = create_imap(self.login)
+        isImapClient = isinstance(imap, IMAPClient)
+        print("isImapClient is " + str(isImapClient))
+        if isImapClient:
+            # authenticate
+            dat = imap.login(self.login, str(self.password))
+            print("type is {} for {}".format(dat, self.login))
+        else:
+            responseType, dat = imap.login(self.login, str(self.password))
+            print("type is {} for {}".format(responseType, self.login))
+
+        mail_list = []
+        print("read mails from {}".format(self.login))
+        if not isImapClient:
+            folder_list = self.show_folders(imap)
+            for folder in folder_list:
+                print("folder is {}".format(folder))
+                mail_list.extend(self._get_messages_from_folder(imap, subject=VALIDATION_URL_SUBJECT_fr,
+                                                                folder=folder))
+                mail_list.extend(self._get_messages_from_folder(imap, subject=VALIDATION_URL_SUBJECT_EN,
+                                                                folder=folder))
+        else:
+            folder_list = self.show_folders(imap)
+            for folder in folder_list:
+                print("folder is " + folder)
+                mail_list.extend(self._get_messages_from_folder_for_imapclient(imap, folder=folder))
+        if DOMAIN_HOTMAIL in self.login:
+            mail_list.extend(
+                self._get_messages_from_folder_for_imapclient(imap, folder="Junk"))
+        if not isImapClient:
+            imap.close()
+        imap.logout()
+        mails_messages.extend(mail_list)
+        return mail_list
+
+    def _get_messages_from_folder(self, imap, subject, folder="INBOX") -> list:
+        imap.select(folder)
+        mail_messages = []
+        typ, data = imap.search(None, '(SUBJECT "{}" SINCE "{}")'.format(subject,
+                                                                         datetime.datetime.today().strftime(
+                                                                             date_format)))
+        for i in data[0].split():
+            # fetch the email message by ID
+            res, msg = imap.fetch(i.decode("utf-8"), "(RFC822)")
+            body = ''
+            for response in msg:
+                if isinstance(response, tuple):
+                    # parse a bytes email into a message object
+                    msg = email.message_from_bytes(response[1])
+                    # decode the email subject
+                    subject, subject_encoded = decode_header(msg["Subject"])[0]
+                    received_date = msg["Date"]
+                    if isinstance(subject, bytes):
+                        # if it's a bytes, decode to str
+                        subject = subject.decode(subject_encoded)
+                    # decode email sender
+                    from_address = find_from_mail(decode_header(msg.get("From")))
+                    to_email = find_from_mail(decode_header(msg.get("To")))
+                    print("Email:", self.login)
+                    print("From:", from_address)
+                    print("To:", to_email)
+                    print("Subject:", subject)
+                    # if the email message is multipart
+                    if msg.is_multipart():
+                        # iterate over email parts
+                        for part in msg.walk():
+                            try:
+                                # get the email body
+                                payloads = part.get_payload()
+                                if isinstance(payloads, list):
+                                    for payload in payloads:
+                                        if isinstance(payload, Message):
+                                            body = body + payload.get_payload(decode=True).decode("iso-8859-1")
+                                # print(body)
+                            except Exception as Error:
+                                print(Error)
+                    else:
+                        body = msg.get_payload(decode=True).decode()
+                        print(body)
+                    if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject:
+                        mail = MailPojo(subject=subject, body=body, from_address=from_address)
+                        if to_email is None:
+                            mail.to_address = self.login
+                        else:
+                            mail.to_address = to_email
+                        mail.mail_address = self.login
+                        mail_messages.append(mail)
+        return mail_messages
+
+    def _get_messages_from_folder_for_imapclient(self, imap, folder="INBOX") -> list:
+        mail_messages = []
+        search_terms = 'SINCE "{}"'.format(
+            datetime.datetime.today().strftime(
+                date_format))
+        print("search terms is " + search_terms)
+        imap.select_folder(folder)
+        messages = imap.search(['SINCE', datetime.datetime.today()])
+        print("%d messages from our best friend" % len(messages))
+        for uid, message_data in imap.fetch(messages, 'RFC822').items():
+            try:
+                email_message = email.message_from_bytes(message_data[b'RFC822'])
+                from_address = email_message.get('FROM')
+                subject = email_message.get('subject')
+                # print("{}, {},{}".format(from_address, subject, email_message))
+                body = ""
+                if "no-reply@hermes.com" in from_address or "appointment2022@aol.com":
+                    for part in email_message.walk():
+                        print(part.get_content_type())
+                        if part.get_content_type() == "text/html":
+                            body = body + part.get_payload(decode=True).decode("utf-8")
+                        elif part.get_content_type() == "text/plain":
+                            body = body + part.get_payload()
+                    if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject:
+                        mail = MailPojo(subject=subject, body=body, from_address=from_address)
+                        mail.isImapClient = True
+                        print("email is {}".format(self.login))
+                        print("body is {}".format(body))
+                        print("subject is {}".format(subject))
+                        mail_messages.append(mail)
+            except Exception as error:
+                print(error)
+                print("error trying to read email_Message for {}".format(self.login))
+        return mail_messages
+
+
+def need_to_valid_url(url: str, successful_items) -> bool:
+    # return True
+    # if len(successful_items) == 0:
+    #     return False
+    print("url is :" + url)
+    parts = url.split('/')
+    id = parts[5]
+    if len(id) == 6:
+        if id == "CS93VB":
+            print("found")
+        for item in successful_items:
+            if item.id == id:
+                if item.url_validated is not None:
+                    return not item.url_validated
+                else:
+                    # if url_validated is None
+                    if item.url_validated is not None:
+                        return not item.url_validated
+                    return True
+        return True
+    else:
+        print("id not valid:{}".format(id))
+        return False
+
+
+def need_to_check_email(mail: str, successful_items) -> bool:
+    print("successful_items size is " + str(len(successful_items)))
+    filtered_items = list(filter(lambda item: item.email == mail, successful_items))
+    # has validated value
+    if len(filtered_items) > 0:
+        validated_items = list(filter(
+            lambda filtered_item: filtered_item.url_validated is not None and filtered_item.url_validated is True,
+            filtered_items))
+        if len(validated_items) > 0:
+            return False
+        else:
+            return True
+    else:
+        return True
+
+
+def find_links_to_validate_from_mail_list(mail_list: list):
+    # check time before start checking emails
+    contact_to_book_list = MONGO_STORE_MANAGER.get_all_contact_to_book_list()
+    successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+    mails_messages = []
+    with ThreadPoolExecutor(max_workers=len(mail_list)) as executor:
+        for mail in mail_list:
+            # check whether we need to read mail
+            if need_to_check_email(mail.mail, successful_items):
+                mail_reader = MailReader(mail.mail, mail.password)
+                executor.submit(mail_reader.read_emails, mails_messages)
+    # for mail in mail_list:
+    #     # check whether we need to read mail
+    #     if need_to_check_email(mail.mail, successful_items):
+    #         mail_reader = MailReader(mail.mail, mail.password)
+    #         mail_reader.read_emails(mails_messages)
+
+    for mail in mails_messages:
+        match = re.search(VALIDATION_URL_REGEX, mail.body)
+        if match:
+            url = match.group(0)
+            if need_to_valid_url(url, successful_items):
+                print("need to validate url: " + url)
+                MONGO_STORE_MANAGER.save_links_to_validate(url, mail.to_address,
+                                                           _all_contact_list=contact_to_book_list)
+            else:
+                print("do not need to click url --> {}".format(mail.mail_address))

models/contact_pojo.py

@@ -10,6 +10,7 @@ class ContactPojo:
     mail: str
     ccid: str
     position: int
+    store: str
     note: str
 
     def __init__(self, phone_number: str, passport_number: str, last_name: str, first_name: str, mail: str,
@@ -31,6 +32,7 @@ class ContactPojo:
             u'last_name': self.last_name,
             u'first_name': self.first_name,
             u'mail': self.mail,
+            u'store': self.store,
             u'ccid': self.ccid,
             u'position': self.position
         }
@@ -39,13 +41,15 @@ class ContactPojo:
 
     @staticmethod
     def from_firestore_dict(source):
-        ccid = source['ccid']
         phone = source['phone']
-        position = source['position']
         passport = source['passport']
         email = source['mail']
         last_name = source['last_name']
         first_name = source['first_name']
-        result = ContactPojo(ccid=ccid, phone_number=phone, passport_number=passport, position=position, mail=email,
+        ip_country = "FR"
+        if source.get('ip_country'):
+            ip_country = source['ip_country']
+        result = ContactPojo(phone_number=phone, passport_number=passport, mail=email,
                              last_name=last_name, first_name=first_name)
+        result.ip_country = ip_country
         return result

models/jsdata_le_pojo.py

@@ -0,0 +1,63 @@
+import json
+
+from models.jsdata_pojo import JsDataPojo
+
+
+# "log2": true,
+# "dcok": ".hermes.com",
+# "m_fmi": false,
+# "tbce": 0,
+# "es_sigmdn": null,
+# "es_mumdn": null,
+# "es_distmdn": null,
+# "es_angsmdn": null,
+# "es_angemdn": null,
+# "m_s_c": 0,
+# "m_m_c": 3,
+# "m_c_c": 3,
+# "m_cm_r": 1,
+# "m_ms_r": -1
+class JsDataLeTypePojo(JsDataPojo):
+
+    def __init__(self, glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu,
+                 vnd, pr, ts_mtp, dvm, m_s_c, m_m_c, m_c_c, m_cm_r, m_ms_r):
+        super().__init__(glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu,
+                         vnd, pr, ts_mtp, dvm)
+        self.log2 = True
+        self.dcok = ".hermes.com"
+        self.m_fmi = False
+        self.tbce = 0
+        self.es_sigmdn = None
+        self.es_mumdn = None
+        self.es_distmdn = None
+        self.es_angsmdn = None
+        self.es_angemdn = None
+        self.m_s_c = m_s_c
+        self.m_m_c = m_m_c
+        self.m_c_c = m_c_c
+        self.m_cm_r = m_cm_r
+        self.m_ms_r = m_ms_r
+        self.cfpfe = "RXJyb3I6IENhbm5vdCByZWFkIHByb3BlcnRpZXMgb2YgbnVsbA=="  # 4.25.1
+        self.stcfp = "ckFsbCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo3NjQ3MykKICAgIGF0IEhUTUxEb2N1bWVudC5yIChodHRwczovL3JlbmRlenZvdXNwYXJpcy5oZXJtZXMuY29tL2Rpc3QvdmVuZG9yLWM2Mjk1ZTljMjg3ZGExNzYwODllLmpzOjI6ODg2Nzkp"  # 4.25.1
+
+
+
+test_data_json = """{"glvd": "Qualcomm", "glrd": "Adreno (TM) 630", "hc": 2, "br_oh": 663, "br_ow": 384, "br_h": 663,
+                  "br_w": 384, "rs_h": 811, "rs_w": 384, "rs_cd": 24, "ars_h": 811, "ars_w": 384, "plg": 2, "eva": 33,
+                  "vnd": "Google Inc.", "plu": ["5gQIEKN", "Iv26GLka"],
+                  "ua": "Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36",
+                  "dvm": 4, "acw": "probably", "pr": 2.8125, "ts_mtp": 5,
+                  "cookiesStr": "app.sig=Arp0SzXCTBF2ordHf3pPTd6cS14;datadome=o_7W8TjHcQjqvnFjFBGAZZ0OmNOd0fnJXfzgqOajAHBBNv7zNMFi2~Jn7wQbTEIZdr2qvg0IFFQQr~Zk5ifzM154V1XxZVhgDZeVkNyRKrQewLBIzVeSagWs587fBqdX;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoieVVRWmJBWnlRTnQ5bFlNbDdNN3R6eHhqIn0=;policy=accepted;lang=fr;"}"""
+
+if __name__ == '__main__':
+    test_data_dict = json.loads(test_data_json)
+    le_js_data = JsDataLeTypePojo(glrd=test_data_dict['glrd'], glvd=test_data_dict['glvd'], hc=test_data_dict['hc'],
+                                  ua=test_data_dict['ua'], br_oh=test_data_dict['br_oh'], br_ow=test_data_dict['br_ow'],
+                                  ars_h=test_data_dict['ars_h'], ars_w=test_data_dict['ars_w'], pr=test_data_dict['pr'],
+                                  plg=test_data_dict['plg'], br_h=test_data_dict['br_h'], br_w=test_data_dict['br_w'],
+                                  plu=test_data_dict['plu'], vnd=test_data_dict['vnd'], dvm=test_data_dict['dvm'],
+                                  ts_mtp=test_data_dict['ts_mtp'], eva=test_data_dict['eva'],
+                                  rs_h=test_data_dict['rs_h'],
+                                  rs_w=test_data_dict['rs_w'], rs_cd=test_data_dict['rs_cd'], m_s_c=0, m_m_c=3, m_c_c=3,
+                                  m_cm_r=1, m_ms_r=-1)
+    print(le_js_data.to_url_encoded_json())

models/jsdata_pojo.py

@@ -9,7 +9,7 @@ import jsonpickle
 
 @dataclass
 class JsDataPojo:
-    opts: str
+    # opts: str
     ttst: int
     ifov: bool
     hc: int
@@ -137,8 +137,8 @@ class JsDataPojo:
 
     def __init__(self, glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu,
                  vnd, pr, ts_mtp, dvm):
-        self.opts = "endpoint,ajaxListenerPath"
+        # self.opts = "endpoint,ajaxListenerPath"
-        self.ttst = random.randint(38, 94)
+        self.ttst = random.randint(38, 148)
         self.ifov = False
         self.hc = hc
         self.br_oh = br_oh
@@ -166,7 +166,7 @@ class JsDataPojo:
         self.pr = pr
         self.ars_h = ars_h
         self.ars_w = ars_w
-        self.tz = -60
+        self.tz = -120
         self.str_ss = True
         self.str_ls = True
         self.str_idb = True
@@ -236,8 +236,8 @@ class JsDataPojo:
         self.vc1ts = True
         self.dvm = dvm
         self.set_default_values()
-        self.glrd = glrd
         self.glvd = glvd
+        self.glrd = glrd
         self.wwl = False
         self.jset = int(time.time())
 
@@ -265,8 +265,8 @@ class JsDataPojo:
         self.emt = False
         self.bfr = False
         self.dbov = False
-        self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2"  # 4.24.1
+        self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2"  # 4.25.1
-        self.stcfp = "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzMDIpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWIgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTA2MjIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxNzMz"  # 4.24.1
+        self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzY0NzMpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hYl0gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTAxMTgpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUyMjY2"  # 4.25.1
         self.ckwa = True
 
     def to_url_encoded_json(self):

models/mail_pojo.py

@@ -0,0 +1,36 @@
+class MailAddress:
+    def __init__(self, mail, password):
+        self.mail = mail
+        self.password = password
+
+    def __repr__(self):
+        return "邮箱:{}, 密码:{}".format(self.mail, self.password)
+
+    def to_firestore_dict(self):
+        dest = {
+            u'mail': self.mail,
+            u'password': self.password
+        }
+        return dest
+
+    @staticmethod
+    def from_firestore_dict(source):
+        password = source['password']
+        mail = source['mail']
+        return MailAddress(mail=mail, password=password)
+
+
+class MailPojo:
+    from_address: str
+    to_address: str
+    body: str
+    subject: str
+    mail_address: str = ""
+    isImapClient = False
+
+    def __init__(self, from_address, body, subject):
+        self.body = body
+        self.subject = subject
+        self.from_address = from_address
+        self.isImapClient = False
+        self.to_address = ""

models/result_pojo.py

@@ -8,3 +8,4 @@ class RequestResult(Enum):
     COOKIES_ERROR = "COOKIES_ERROR"
     UNKNOWN = "UNKNOWN"
     BAD_GATEWAY = "BAD_GATEWAY"
+    CTRF_ERROR = "CTRF_ERROR"

parallel_request_sender.py

@@ -0,0 +1,87 @@
+import datetime
+import logging
+import sys
+from threading import Thread
+
+from db.mongo_manager import MONGO_STORE_MANAGER
+from excel_reader import read_contacts
+from models.contact_pojo import ContactPojo
+from queue_message.CookiesPublisher import CookiesPublisher, SHARED_OBJECT, TEST_QUEUE
+from queue_message.appointmentrequestsender import AppointmentRequestSender
+from queue_message.parallel_requestsender import ParallelRequestSender
+from utiles import is_time_between
+from utils.AppLogging import init_logger
+from workers.proxies_constants import MOBILE_PROXY_LIST_FR
+
+IPFIY = 'http://api.ipify.org'
+NGROK_TEST = "https://bcc6-193-164-156-53.ngrok-free.app"
+
+
+def is_already_sent(contact: ContactPojo) -> bool:
+    already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+    for required_contact in already_sent_contacts:
+        if contact.mail == required_contact.email:
+            return True
+    return False
+
+
+def filter_contacts(_contact_list: list) -> list:
+    already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+    _link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
+    _contact_list_to_book = []
+    for contact in _contact_list:
+        _to_add = True
+        for booked in already_sent_contacts:
+            if contact.mail == booked.email:
+                _to_add = False
+        # 如果已经收到链接了，就不要再请求
+        for link_to_validate in _link_to_validate_list:
+            if contact.mail == link_to_validate.email:
+                logger.info("{}: link already received".format(contact.mail))
+                _to_add = False
+        if _to_add:
+            _contact_list_to_book.append(contact)
+
+    return _contact_list_to_book
+
+
+def is_open():
+    return is_time_between(datetime.time(10, 30), datetime.time(19, 00))
+
+
+count = 0
+init_logger()
+logger = logging.getLogger()
+
+logger.addHandler(logging.StreamHandler(stream=sys.stdout))
+
+
+def send_appointment_request(message_queue_name, _contact_list):
+    global count
+    count = count + 1
+    for _contact in _contact_list:
+        logger.info(_contact)
+    _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name)
+    _cookiesPublisher.set_up_connection()
+    receiver = ParallelRequestSender(sub_contact_list=_contact_list, proxy_to_use_list=MOBILE_PROXY_LIST_FR,
+                                     queue_name=message_queue_name, just_send=True,
+                                     cookiesPublisher=_cookiesPublisher, logger=logger)
+    print("count is " + str(count))
+    receiver.run()
+
+
+if __name__ == '__main__':
+    contacts_file_path = '~/Desktop/31_03_to_test.xlsx'
+    _contact_list = read_contacts(contacts_file_path)[0:20]
+    _contact_list_to_book = filter_contacts(_contact_list)
+    _segment_number = 1
+    logger.info("{} contacts to book".format(len(_contact_list_to_book)))
+    last_thread = None
+    for i in range(0, _segment_number):
+        logger.info("segment is {}".format(i))
+        _step = int(len(_contact_list_to_book) / _segment_number)
+        _sublist = _contact_list_to_book[i * _step:_step * (i + 1)]
+        _thread1 = Thread(target=send_appointment_request, args=(TEST_QUEUE, _sublist))
+        last_thread = _thread1
+        _thread1.start()
+    last_thread.join()

queue_message/CookiesPublisher.py

@@ -5,8 +5,11 @@ REQUEST_DATA_QUEUE = 'REQUEST_DATA'
 # REQUEST_DATA_QUEUE_TEST = 'REQUEST_DATA_TEST'
 REQUEST_DATA_QUEUE_TEST = 'REQUEST_DATA_TEST'
 TEST_QUEUE = 'TEST_QUEUE'
+SHARED_OBJECT = 'SHARED_OBJECT'
+MORNING_DATA_CACHE = 'MORNING_DATA_CACHE'
 REQUEST_DATA_QUEUE_DE = 'REQUEST_DATA_DE'
 REQUEST_DATA_OBJECT = 'REQUEST_DATA_OBJECT'
+REGISTER_QUEUE = 'REGISTER_QUEUE'
 credentials = pika.PlainCredentials('appointment', 'ZyuhJZ2xEYWhElhpJjy7YEpZGZwNYJz2fHIu')
 
 

queue_message/appointmentrequestsender.py

@@ -3,12 +3,14 @@ import json
 import random
 import threading
 import time
-from http.cookies import SimpleCookie
 
 import pika
 
 from db.mongo_manager import MONGO_STORE_MANAGER
+from mail.mail_reader_all_contacts import find_links_to_validate_from_mail_list
+from models.ReserveResultPojo import ReserveResultPojo
 from models.contact_pojo import ContactPojo
+from models.jsdata_le_pojo import JsDataLeTypePojo
 from models.jsdata_pojo import JsDataPojo
 from models.result_pojo import RequestResult
 from queue_message.CookiesPublisher import CookiesPublisher
@@ -30,63 +32,53 @@ def is_already_sent(contact: ContactPojo) -> bool:
     return False
 
 
-def filter_contacts(_contact_list: list) -> list:
+def filter_contacts(_contact_list: list, provided_list=[]) -> list:
     already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+    already_sent_contacts.extend(provided_list)
+    _link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
     _contact_list_to_book = []
     for contact in _contact_list:
         _to_add = True
         for booked in already_sent_contacts:
+            if isinstance(booked, ReserveResultPojo):
                 if contact.mail == booked.email:
                     _to_add = False
+            else:
+                if contact.mail == booked.mail:
+                    _to_add = False
+        # 如果已经收到链接了，就不要再请求
+        for link_to_validate in _link_to_validate_list:
+            if contact.mail == link_to_validate.email:
+                print("{}: link already received".format(contact.mail))
+                _to_add = False
         if _to_add:
             _contact_list_to_book.append(contact)
 
     return _contact_list_to_book
 
 
-def get_valid_csrf() -> str:
-    captchaResultGetter = CaptchaResultGetter()
-    _valid_cookie = captchaResultGetter.get_valid_cookie()
-    # while _valid_cookie is None:
-    #     _valid_cookie = captchaResultGetter.get_valid_cookie()
-    new_csrf = None
-    while new_csrf is None:
-        valid_cookie = None
-        if _valid_cookie is not None:
-            simple_cookie = SimpleCookie()
-            simple_cookie.load(_valid_cookie)
-            new_cookies = {k: v.value for k, v in simple_cookie.items()}
-            new_coolies_str = ""
-            for key in new_cookies:
-                print(key)
-                new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
-            print(new_coolies_str)
-            valid_cookie = new_coolies_str + "app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;policy=accepted;lang=fr;"
-        print(valid_cookie)
-        new_csrf = captchaResultGetter.get_csrf(valid_cookie)
-        if new_csrf is None:
-            _valid_cookie = None
-            while _valid_cookie is None:
-                _valid_cookie = captchaResultGetter.get_valid_cookie()
-        time.sleep(2)
-    return new_csrf
-
-
 def is_open():
     return is_time_between(datetime.time(10, 30), datetime.time(19, 00))
 
 
 class AppointmentRequestSender(threading.Thread):
-    def __init__(self, sub_contact_list: list, proxy_to_use_list, cookiesPublisher: CookiesPublisher,
+    def __init__(self, sub_contact_list: list, proxy_to_use_list, logger, cookiesPublisher: CookiesPublisher,
+                 just_send=False,
                  queue_name=REQUEST_DATA_QUEUE):
         super().__init__()
         self.connection = None
+        self.just_send = just_send
+        self.logger = logger
+        self.already_tried_contact_list = []
         self.cookiesPublisher = cookiesPublisher
         self.channel = None
         self.valid_csrf = None
+        self.list_to_retrieve_mails = sub_contact_list
+        self.initial_contact_list = sub_contact_list
         self.contact_list = sub_contact_list
         self.queue_name = queue_name
         self.proxy_to_use_list = proxy_to_use_list
+        self.already_read_emails = False
 
     def set_up_connection(self):
         self.connection = pika.BlockingConnection(
@@ -94,16 +86,22 @@ class AppointmentRequestSender(threading.Thread):
         self.channel = self.connection.channel()
 
     def listen_to_queue(self, callback):
-        print("listen to queue {}".format(self.queue_name))
+        self.logger.info("listen to queue {}".format(self.queue_name))
         self.channel.basic_qos(prefetch_count=1)
         self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback)
         self.channel.start_consuming()
 
     def on_message(self, ch, method, properties, body):
         _message_count = self.cookiesPublisher.message_count()
-        print("message count in queue is {}".format(_message_count))
+        self.logger.info("message count in queue is {}".format(_message_count))
         _received_object = body.decode("UTF-8")
-        print(f" [x] Received {_received_object}")
+        self.logger.info(f" [x] Received {_received_object}")
+        _port = random.randint(40001, 49999)
+        self.logger.info("generated port is {}".format(_port))
+        _chosen_proxy = random.choice(self.proxy_to_use_list)
+        _proxy_to_use = {}
+        _proxy_to_use["http"] = _chosen_proxy["http"].format(_port)
+        _proxy_to_use["https"] = _chosen_proxy["https"].format(_port)
         if "glrd" in _received_object:
             _received_dict = json.loads(_received_object)
             js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'],
@@ -116,55 +114,128 @@ class AppointmentRequestSender(threading.Thread):
                                  rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'])
             _received_cookies = _received_dict["cookiesStr"]
             sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict,
-                            proxy_to_use=random.choice(self.proxy_to_use_list))
+                            proxy_to_use=_proxy_to_use, logger=self.logger)
+            # remove already sent contacts
+            if self.just_send:
+                self.contact_list = filter_contacts(self.contact_list, self.already_tried_contact_list)
+            else:
                 self.contact_list = filter_contacts(self.contact_list)
             # remove already booked contacts
             random.shuffle(self.contact_list)
             if len(self.contact_list) > 0 and is_open():
                 captchaResultGetter = CaptchaResultGetter()
-                print("contact number is {}".format(len(self.contact_list)))
+                self.logger.info("contact number is {}".format(len(self.contact_list)))
-                self.contact_list = filter_contacts(self.contact_list)
+                # self.contact_list = filter_contacts(self.contact_list)
                 for con in self.contact_list:
-                    # if not is_already_sent(con):
+                    self.logger.info(con.mail)
-                    print(con.mail)
-                    # time.sleep(random.randint(1, 5))
                     if self.valid_csrf is None:
-                        self.valid_csrf = captchaResultGetter.get_csrf(
+                        csrf_result = captchaResultGetter.get_csrf(
-                            proxy_to_use=random.choice(self.proxy_to_use_list),
+                            proxy_to_use=_proxy_to_use, js_data=js_data,
-                            cookie=body.decode("UTF-8"))
+                            cookie=_received_cookies)
+                        if isinstance(csrf_result, str):
+                            self.valid_csrf = csrf_result
+                        else:
+                            self.logger.info("csrf is {}".format(csrf_result))
+                            if csrf_result == RequestResult.BLOCKED:
+                                break
                     _new_cookies = captchaResultGetter.get_valid_ch_cookie(sender.proxy_to_use, js_data,
                                                                            old_valid_cookie=_received_cookies)
-                    if _received_cookies is not None:
+                    if _new_cookies is not None:
-                        print("new cookie is " + _received_cookies)
+                        self.logger.info("new cookie is " + _new_cookies)
-                        sender.cookie_str = _received_cookies
+                        time.sleep(random.randint(1, 3))
-                        time.sleep(random.randint(1, 5))
+                        # m_s_c = f.scroll
-                        can_continue = sender.send_request(HERMES_REGISTER, con, csrf=self.valid_csrf)
+                        m_s_c = random.randint(0, 3)
+                        m_c_c = random.randint(3, 5)  # click count
+                        m_m_c = random.randint(3, 5)  # move count
+                        m_cm_r = m_c_c / m_m_c
+                        m_ms_r = random.randint(-1, 1)
+
+                        js_le_data = JsDataLeTypePojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'],
+                                                      hc=_received_dict['hc'],
+                                                      ua=_received_dict['ua'], br_oh=_received_dict['br_oh'],
+                                                      br_ow=_received_dict['br_ow'],
+                                                      ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'],
+                                                      pr=_received_dict['pr'],
+                                                      plg=_received_dict['plg'], br_h=_received_dict['br_h'],
+                                                      br_w=_received_dict['br_w'],
+                                                      plu=_received_dict['plu'], vnd=_received_dict['vnd'],
+                                                      dvm=_received_dict['dvm'],
+                                                      ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'],
+                                                      rs_h=_received_dict['rs_h'],
+                                                      rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'],
+                                                      m_s_c=m_s_c, m_m_c=m_m_c, m_c_c=m_c_c,
+                                                      m_cm_r=m_cm_r, m_ms_r=m_ms_r)
+                        time.sleep(random.randint(1, 4))
+                        _new_le_cookies = captchaResultGetter.get_le_valid_cookie(proxy_to_use=_proxy_to_use,
+                                                                                  js_le_type_data=js_le_data,
+                                                                                  old_valid_cookie=_new_cookies)
+                        if _new_le_cookies is not None:
+                            # self.logger.info("new le type cookie is " + _new_le_cookies)
+                            sender.cookie_str = _new_le_cookies
+                            time.sleep(random.randint(1, 3))
+                            self.already_tried_contact_list.append(con)
+                            can_continue = sender.send_request(HERMES_REGISTER, js_data, con, csrf=self.valid_csrf)
+                            if can_continue == RequestResult.SUCCESS:
+                                # 让服务器读取成功的约会
+                                try:
+                                    self.logger.info("try to remove success contact from list to retrieve mails")
+                                    self.list_to_retrieve_mails.remove(con)
+                                except Exception as e:
+                                    self.logger.info(
+                                        "exception while remove success contact from list to retrieve mails")
+                                    print(e)
+                        else:
+                            can_continue = RequestResult.COOKIES_ERROR
                     else:
                         can_continue = RequestResult.COOKIES_ERROR
                     if can_continue == RequestResult.BLOCKED:
-                        print("cannot continue, valid_csrf is " + str(self.valid_csrf))
+                        self.logger.info("BLOCKED, valid_csrf is " + str(self.valid_csrf))
                         break
                     elif can_continue == RequestResult.PROXY_ERROR:
-                        print("PROXY_ERROR, will not reset valid_csrf")
+                        self.logger.info("PROXY_ERROR, will not reset valid_csrf")
                     elif can_continue == RequestResult.COOKIES_ERROR:
-                        print("COOKIES_ERROR, will not reset valid_csrf")
+                        self.logger.info("COOKIES_ERROR, will not reset valid_csrf")
                     else:
-                        print("can continue, will reset valid_csrf")
+                        self.logger.info("can continue, will reset valid_csrf")
                         self.valid_csrf = None
                 time.sleep(random.randint(1, 2))
-                print("will ack method.delivery_tag: " + str(method.delivery_tag))
+                self.logger.info("will ack method.delivery_tag: " + str(method.delivery_tag))
                 ch.basic_ack(delivery_tag=method.delivery_tag)
             else:
-                print("empty list")
+                self.retrieve_invalidate_urls()
-                time.sleep(120)
+                self.logger.info("empty list")
-                print("will basic_reject method.delivery_tag: " + str(method.delivery_tag))
+                time.sleep(60)
+                self.logger.info("will basic_reject method.delivery_tag: " + str(method.delivery_tag))
                 ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
         else:
-            print("not a valid object")
+            self.logger.info("not a valid object")
             ch.basic_ack(delivery_tag=method.delivery_tag)
 
     def run(self):
-        print(threading.currentThread().name + " starts")
+        self.logger.info(threading.currentThread().name + " starts")
         self.set_up_connection()
         self.listen_to_queue(self.on_message)
         self.channel.start_consuming()
+
+    def retrieve_invalidate_urls(self):
+        if not self.already_read_emails and len(self.list_to_retrieve_mails) > 0:
+            self.logger.info("will retrieve validate urls")
+            time.sleep(30)
+            _mail_list = MONGO_STORE_MANAGER.get_destination_emails()
+            _mail_list_filtered = []
+            for mail in _mail_list:
+                for _contact in self.list_to_retrieve_mails:
+                    if _contact.mail == mail.mail:
+                        _mail_list_filtered.append(mail)
+            self.logger.info("will call find_links_to_validate_from_mail_list, size = " + str(len(_mail_list_filtered)))
+            find_links_to_validate_from_mail_list(_mail_list_filtered)
+            self.already_read_emails = True
+        else:
+            self.logger.info("already read emails, is there any contacts to use")
+            self.logger.info("reset already_tried_contact_list")
+            self.already_tried_contact_list = []
+            self.contact_list = filter_contacts(self.initial_contact_list, self.already_tried_contact_list)
+            self.logger.info("contact_list size is " + str(len(self.contact_list)))
+            if len(self.contact_list) > 0:
+                self.logger.info("set already_read_emails to False")
+                self.already_read_emails = False

queue_message/link_validator_receiver.py

@@ -36,34 +36,6 @@ def filter_contacts(_contact_list: list) -> list:
     return _contact_list_to_book
 
 
-def get_valid_csrf() -> str:
-    captchaResultGetter = CaptchaResultGetter()
-    _valid_cookie = captchaResultGetter.get_valid_cookie()
-    # while _valid_cookie is None:
-    #     _valid_cookie = captchaResultGetter.get_valid_cookie()
-    new_csrf = None
-    while new_csrf is None:
-        valid_cookie = None
-        if _valid_cookie is not None:
-            simple_cookie = SimpleCookie()
-            simple_cookie.load(_valid_cookie)
-            new_cookies = {k: v.value for k, v in simple_cookie.items()}
-            new_coolies_str = ""
-            for key in new_cookies:
-                print(key)
-                new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
-            print(new_coolies_str)
-            valid_cookie = new_coolies_str + "app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;policy=accepted;lang=fr;"
-        print(valid_cookie)
-        new_csrf = captchaResultGetter.get_csrf(valid_cookie)
-        if new_csrf is None:
-            _valid_cookie = None
-            while _valid_cookie is None:
-                _valid_cookie = captchaResultGetter.get_valid_cookie()
-        time.sleep(2)
-    return new_csrf
-
-
 class LinkValidatorReceiver(threading.Thread):
     def __init__(self, linkpojo_list: list):
         self.connection = None

queue_message/parallel_requestsender.py

@@ -0,0 +1,221 @@
+import datetime
+import json
+import random
+import threading
+import time
+from concurrent.futures.thread import ThreadPoolExecutor
+
+import pika
+
+from db.mongo_manager import MONGO_STORE_MANAGER
+from mail.mail_reader_all_contacts import find_links_to_validate_from_mail_list
+from models.ReserveResultPojo import ReserveResultPojo
+from models.contact_pojo import ContactPojo
+from models.jsdata_le_pojo import JsDataLeTypePojo
+from models.jsdata_pojo import JsDataPojo
+from models.result_pojo import RequestResult
+from queue_message.CookiesPublisher import CookiesPublisher
+from queue_message.appointmentrequestsender import filter_contacts, is_open
+from utiles import is_time_between
+from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
+from workers.sender import Sender
+
+QUEUE_HOST = "appointment.lpaconsulting.fr"
+REQUEST_DATA_QUEUE = 'REQUEST_DATA'
+credentials = pika.PlainCredentials('appointment', 'ZyuhJZ2xEYWhElhpJjy7YEpZGZwNYJz2fHIu')
+
+
+def is_already_sent(contact: ContactPojo) -> bool:
+    already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+    for required_contact in already_sent_contacts:
+        if contact.mail == required_contact.email:
+            return True
+    return False
+
+
+class ParallelRequestSender(threading.Thread):
+    def __init__(self, sub_contact_list: list, proxy_to_use_list, logger, cookiesPublisher: CookiesPublisher,
+                 just_send=False,
+                 queue_name=REQUEST_DATA_QUEUE):
+        super().__init__()
+        self.connection = None
+        self.just_send = just_send
+        self.logger = logger
+        self.already_tried_contact_list = []
+        self.cookiesPublisher = cookiesPublisher
+        self.channel = None
+        self.valid_csrf = None
+        self.list_to_retrieve_mails = sub_contact_list
+        self.contact_list = sub_contact_list
+        self.queue_name = queue_name
+        self.proxy_to_use_list = proxy_to_use_list
+        self.already_read_emails = False
+
+    def set_up_connection(self):
+        self.connection = pika.BlockingConnection(
+            pika.ConnectionParameters(host=QUEUE_HOST, port=5672, credentials=credentials))
+        self.channel = self.connection.channel()
+
+    def listen_to_queue(self, callback):
+        self.logger.info("listen to queue {}".format(self.queue_name))
+        self.channel.basic_qos(prefetch_count=1)
+        self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback)
+        self.channel.start_consuming()
+
+    def send_request(self, _received_cookies, _received_dict, js_data: JsDataPojo, logger,
+                     _contact) -> RequestResult:
+        _proxy_to_use = self.generate_proxy()
+        logger.info("send_request for contact: {}, cookies: {}".format(_contact.mail, _received_cookies))
+        logger.info("proxy to use is {}".format(_proxy_to_use))
+        sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict,
+                        proxy_to_use=_proxy_to_use, logger=logger)
+        # remove already sent contacts
+        if is_open():
+            captchaResultGetter = CaptchaResultGetter()
+            _new_cookies = captchaResultGetter.get_valid_ch_cookie(sender.proxy_to_use, js_data,
+                                                                   old_valid_cookie=_received_cookies)
+            # self.contact_list = filter_contacts(self.contact_list)
+            logger.info(_contact.mail)
+            valid_csrf = captchaResultGetter.get_csrf(
+                proxy_to_use=_proxy_to_use, js_data=js_data,
+                cookie=_new_cookies)
+            if isinstance(valid_csrf, str):
+                if _new_cookies is not None:
+                    logger.info("new cookie is " + _new_cookies)
+                    # m_s_c = f.scroll
+                    m_s_c = random.randint(0, 3)
+                    m_c_c = random.randint(3, 5)  # click count
+                    m_m_c = random.randint(3, 5)  # move count
+                    m_cm_r = m_c_c / m_m_c
+                    m_ms_r = random.randint(-1, 1)
+
+                    js_le_data = JsDataLeTypePojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'],
+                                                  hc=_received_dict['hc'],
+                                                  ua=_received_dict['ua'], br_oh=_received_dict['br_oh'],
+                                                  br_ow=_received_dict['br_ow'],
+                                                  ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'],
+                                                  pr=_received_dict['pr'],
+                                                  plg=_received_dict['plg'], br_h=_received_dict['br_h'],
+                                                  br_w=_received_dict['br_w'],
+                                                  plu=_received_dict['plu'], vnd=_received_dict['vnd'],
+                                                  dvm=_received_dict['dvm'],
+                                                  ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'],
+                                                  rs_h=_received_dict['rs_h'],
+                                                  rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'],
+                                                  m_s_c=m_s_c, m_m_c=m_m_c, m_c_c=m_c_c,
+                                                  m_cm_r=m_cm_r, m_ms_r=m_ms_r)
+                    time.sleep(random.randint(1, 4))
+                    _new_le_cookies = captchaResultGetter.get_le_valid_cookie(proxy_to_use=_proxy_to_use,
+                                                                              js_le_type_data=js_le_data,
+                                                                              old_valid_cookie=_new_cookies)
+                    if _new_le_cookies is not None:
+                        # self.logger.info("new le type cookie is " + _new_le_cookies)
+                        sender.cookie_str = _new_le_cookies
+                        time.sleep(random.randint(1, 3))
+                        self.already_tried_contact_list.append(_contact)
+                        can_continue = sender.send_request(HERMES_REGISTER, js_data, _contact, csrf=valid_csrf)
+                        if can_continue == RequestResult.SUCCESS:
+                            # 让服务器读取成功的约会
+                            try:
+                                self.logger.info("try to remove success contact from list to retrieve mails")
+                                self.list_to_retrieve_mails.remove(_contact)
+                            except Exception as e:
+                                self.logger.info(
+                                    "exception while remove success contact from list to retrieve mails")
+                                print(e)
+                    else:
+                        can_continue = RequestResult.COOKIES_ERROR
+                else:
+                    can_continue = RequestResult.COOKIES_ERROR
+                if can_continue == RequestResult.BLOCKED:
+                    self.logger.info("cannot continue, we are blocked " + str(self.valid_csrf))
+                elif can_continue == RequestResult.PROXY_ERROR:
+                    self.logger.info("PROXY_ERROR, will not reset valid_csrf")
+                elif can_continue == RequestResult.COOKIES_ERROR:
+                    self.logger.info("COOKIES_ERROR, will not reset valid_csrf")
+                else:
+                    self.logger.info("can continue, will reset valid_csrf")
+                    self.valid_csrf = None
+                return can_continue
+            else:
+                return valid_csrf
+                # return RequestResult.CTRF_ERROR
+
+    def getChTypeJsDataFromDict(self, _received_dict) -> JsDataPojo:
+        return JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'],
+                          ua=_received_dict['ua'], br_oh=_received_dict['br_oh'], br_ow=_received_dict['br_ow'],
+                          ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'], pr=_received_dict['pr'],
+                          plg=_received_dict['plg'], br_h=_received_dict['br_h'], br_w=_received_dict['br_w'],
+                          plu=_received_dict['plu'], vnd=_received_dict['vnd'], dvm=_received_dict['dvm'],
+                          ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'],
+                          rs_h=_received_dict['rs_h'],
+                          rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'])
+
+    def generate_proxy(self):
+        _port = random.randint(40001, 49999)
+        _chosen_proxy = random.choice(self.proxy_to_use_list)
+        self.logger.info("generated port is {}".format(_port))
+        _proxy_to_use = {}
+        _proxy_to_use["http"] = _chosen_proxy["http"].format(_port)
+        _proxy_to_use["https"] = _chosen_proxy["https"].format(_port)
+        return _proxy_to_use
+
+    def on_message(self, ch, method, properties, body):
+        _message_count = self.cookiesPublisher.message_count()
+        self.logger.info("message count in queue is {}".format(_message_count))
+        # prepare the contact list
+        if self.just_send:
+            self.contact_list = filter_contacts(self.contact_list, self.already_tried_contact_list)
+        else:
+            self.contact_list = filter_contacts(self.contact_list)
+        # remove already booked contacts
+        random.shuffle(self.contact_list)
+        _received_object = body.decode("UTF-8")
+        self.logger.info(f" [x] Received {_received_object}")
+        step = 5
+        _received_dict = json.loads(_received_object)
+        js_data = self.getChTypeJsDataFromDict(_received_dict)
+        _received_cookies = _received_dict["cookiesStr"]
+        if len(self.contact_list) > step:
+            _sub_list = self.contact_list[0:step]
+            result = None
+            for con in _sub_list:
+                with ThreadPoolExecutor(max_workers=step) as executor:
+                    result = executor.submit(self.send_request, _received_cookies, _received_dict, js_data, self.logger,
+                                             con)
+                    self.logger.info("result is: " + str(result.result()))
+                    if result.result() == RequestResult.SUCCESS:
+                        self.logger.info("Success for {}, with cookies{}".format(con.mail, _received_cookies))
+            if result.result() == RequestResult.BLOCKED or result.result() == RequestResult.CTRF_ERROR:
+                ch.basic_ack(delivery_tag=method.delivery_tag)
+
+            else:
+                ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
+        else:
+            self.retrieve_invalidate_urls()
+            self.logger.info("empty list")
+            time.sleep(120)
+            self.logger.info("will basic_reject method.delivery_tag: " + str(method.delivery_tag))
+            ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
+
+    def run(self):
+        self.logger.info(threading.currentThread().name + " starts")
+        self.set_up_connection()
+        self.listen_to_queue(self.on_message)
+        self.channel.start_consuming()
+
+    def retrieve_invalidate_urls(self):
+        if not self.already_read_emails and len(self.list_to_retrieve_mails) > 0:
+            self.logger.info("will retrieve validate urls")
+            time.sleep(30)
+            _mail_list = MONGO_STORE_MANAGER.get_destination_emails()
+            _mail_list_filtered = []
+            for mail in _mail_list:
+                for _contact in self.list_to_retrieve_mails:
+                    if _contact.mail == mail.mail:
+                        _mail_list_filtered.append(mail)
+            self.logger.info("will call find_links_to_validate_from_mail_list, size = " + str(len(_mail_list_filtered)))
+            find_links_to_validate_from_mail_list(_mail_list_filtered)
+            self.already_read_emails = True
+        else:
+            self.logger.info("already read emails, will not retrieve validate urls")

request_sender.py

@@ -1,19 +1,16 @@
 import datetime
-import random
+import logging
-import time
+import sys
-from http.cookies import SimpleCookie
 from threading import Thread
 
 from db.mongo_manager import MONGO_STORE_MANAGER
 from excel_reader import read_contacts
 from models.contact_pojo import ContactPojo
-from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_DE, REQUEST_DATA_QUEUE_TEST, \
+from queue_message.CookiesPublisher import CookiesPublisher, SHARED_OBJECT, TEST_QUEUE
-    REQUEST_DATA_OBJECT, TEST_QUEUE
+from queue_message.appointmentrequestsender import AppointmentRequestSender
-from queue_message.appointmentrequestsender import AppointmentRequestSender, REQUEST_DATA_QUEUE
 from utiles import is_time_between
-from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
+from utils.AppLogging import init_logger
-from workers.proxies_constants import PROXY_LIST_DE, PROXY_LIST_FR, MOBILE_PROXY_LIST_FR
+from workers.proxies_constants import MOBILE_PROXY_LIST_FR
-from workers.sender import Sender
 
 IPFIY = 'http://api.ipify.org'
 NGROK_TEST = "https://bcc6-193-164-156-53.ngrok-free.app"
@@ -29,74 +26,66 @@ def is_already_sent(contact: ContactPojo) -> bool:
 
 def filter_contacts(_contact_list: list) -> list:
     already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+    _link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
     _contact_list_to_book = []
     for contact in _contact_list:
         _to_add = True
         for booked in already_sent_contacts:
             if contact.mail == booked.email:
                 _to_add = False
+        # 如果已经收到链接了，就不要再请求
+        for link_to_validate in _link_to_validate_list:
+            if contact.mail == link_to_validate.email:
+                logger.info("{}: link already received".format(contact.mail))
+                _to_add = False
         if _to_add:
             _contact_list_to_book.append(contact)
 
     return _contact_list_to_book
 
 
-def get_valid_csrf() -> str:
-    captchaResultGetter = CaptchaResultGetter()
-    _valid_cookie = captchaResultGetter.get_valid_cookie()
-    # while _valid_cookie is None:
-    #     _valid_cookie = captchaResultGetter.get_valid_cookie()
-    new_csrf = None
-    while new_csrf is None and is_open():
-        valid_cookie = None
-        if _valid_cookie is not None:
-            simple_cookie = SimpleCookie()
-            simple_cookie.load(_valid_cookie)
-            new_cookies = {k: v.value for k, v in simple_cookie.items()}
-            new_coolies_str = ""
-            for key in new_cookies:
-                print(key)
-                new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
-            print(new_coolies_str)
-            valid_cookie = new_coolies_str + "app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;policy=accepted;lang=fr;"
-        print(valid_cookie)
-        new_csrf = captchaResultGetter.get_csrf(valid_cookie)
-        if new_csrf is None:
-            _valid_cookie = None
-            while _valid_cookie is None:
-                _valid_cookie = captchaResultGetter.get_valid_cookie()
-        time.sleep(2)
-    return new_csrf
-
-
 def is_open():
     return is_time_between(datetime.time(10, 30), datetime.time(19, 00))
 
 
 count = 0
+init_logger()
+logger = logging.getLogger()
+
+logger.addHandler(logging.StreamHandler(stream=sys.stdout))
 
 
 def send_appointment_request(message_queue_name, _contact_list):
     global count
     count = count + 1
+    for _contact in _contact_list:
+        logger.info(_contact)
     _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name)
     _cookiesPublisher.set_up_connection()
     receiver = AppointmentRequestSender(sub_contact_list=_contact_list, proxy_to_use_list=MOBILE_PROXY_LIST_FR,
-                                        queue_name=message_queue_name,
+                                        queue_name=message_queue_name, just_send=True,
-                                        cookiesPublisher=_cookiesPublisher)
+                                        cookiesPublisher=_cookiesPublisher, logger=logger)
     print("count is " + str(count))
     receiver.run()
 
 
-if __name__ == '__main__':
+def start_send_requests():
-    contacts_file_path = '~/Desktop/contact_list_2024-03-09.xlsx'
+    print("start send requests")
+    contacts_file_path = '~/Desktop/contact_list_2024-04-16_2.xlsx'
     _contact_list = read_contacts(contacts_file_path)
+    _contact_list_to_book = filter_contacts(_contact_list)
     _segment_number = 10
+    logger.info("{} contacts to book".format(len(_contact_list_to_book)))
+    last_thread = None
     for i in range(0, _segment_number):
-        print(i)
+        logger.info("segment is {}".format(i))
-        _step = int(len(_contact_list) / _segment_number)
+        _step = int(len(_contact_list_to_book) / _segment_number)
-        _sublist = _contact_list[i:_step * (i + 1)]
+        _sublist = _contact_list_to_book[i * _step:_step * (i + 1)]
-        print(_sublist[0])
+        _thread1 = Thread(target=send_appointment_request, args=(SHARED_OBJECT, _sublist))
-        print(_sublist[-1])
+        last_thread = _thread1
-        _thread1 = Thread(target=send_appointment_request, args=(REQUEST_DATA_OBJECT, _sublist))
         _thread1.start()
+    last_thread.join()
+
+
+if __name__ == '__main__':
+    start_send_requests()

request_sender_test.py

@@ -0,0 +1,91 @@
+import datetime
+import logging
+import sys
+from threading import Thread
+
+from db.mongo_manager import MONGO_STORE_MANAGER
+from excel_reader import read_contacts
+from models.contact_pojo import ContactPojo
+from queue_message.CookiesPublisher import CookiesPublisher, SHARED_OBJECT, TEST_QUEUE, MORNING_DATA_CACHE
+from queue_message.appointmentrequestsender import AppointmentRequestSender
+from utiles import is_time_between
+from utils.AppLogging import init_logger
+from workers.proxies_constants import MOBILE_PROXY_LIST_FR
+
+IPFIY = 'http://api.ipify.org'
+NGROK_TEST = "https://bcc6-193-164-156-53.ngrok-free.app"
+
+
+def is_already_sent(contact: ContactPojo) -> bool:
+    already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+    for required_contact in already_sent_contacts:
+        if contact.mail == required_contact.email:
+            return True
+    return False
+
+
+def filter_contacts(_contact_list: list) -> list:
+    already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+    _link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
+    _contact_list_to_book = []
+    for contact in _contact_list:
+        _to_add = True
+        for booked in already_sent_contacts:
+            if contact.mail == booked.email:
+                _to_add = False
+        # 如果已经收到链接了，就不要再请求
+        for link_to_validate in _link_to_validate_list:
+            if contact.mail == link_to_validate.email:
+                logger.info("{}: link already received".format(contact.mail))
+                _to_add = False
+        if _to_add:
+            _contact_list_to_book.append(contact)
+
+    return _contact_list_to_book
+
+
+def is_open():
+    return is_time_between(datetime.time(10, 30), datetime.time(19, 00))
+
+
+count = 0
+init_logger()
+logger = logging.getLogger()
+
+logger.addHandler(logging.StreamHandler(stream=sys.stdout))
+
+
+def send_appointment_request(message_queue_name, _contact_list):
+    global count
+    count = count + 1
+    for _contact in _contact_list:
+        logger.info(_contact)
+    _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name)
+    _cookiesPublisher.set_up_connection()
+    receiver = AppointmentRequestSender(sub_contact_list=_contact_list, proxy_to_use_list=MOBILE_PROXY_LIST_FR,
+                                        queue_name=message_queue_name, just_send=True,
+                                        cookiesPublisher=_cookiesPublisher, logger=logger)
+    print("count is " + str(count))
+    receiver.run()
+
+
+def start_send_requests():
+    print("start send requests")
+    contacts_file_path = '~/Desktop/contact_list_merged.xlsx'
+    _contact_list = read_contacts(contacts_file_path)
+    _contact_list_to_book = filter_contacts(_contact_list)
+    _segment_number = 20
+    logger.info("{} contacts to book".format(len(_contact_list_to_book)))
+    last_thread = None
+    for i in range(0, _segment_number):
+        logger.info("segment is {}".format(i))
+        _step = int(len(_contact_list_to_book) / _segment_number)
+        _sublist = _contact_list_to_book[i * _step:_step * (i + 1)]
+        _thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE, _sublist))
+        last_thread = _thread1
+        _thread1.start()
+    last_thread.join()
+
+
+if __name__ == '__main__':
+    start_send_requests()

scheduler.py

@@ -0,0 +1,30 @@
+from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor
+from apscheduler.schedulers.blocking import BlockingScheduler
+
+from request_sender import start_send_requests
+
+
+def start_book_appointment():
+    start_send_requests()
+
+
+def start_check_results_job(sched):
+    sched.add_job(start_book_appointment, 'cron', day_of_week='mon-sat', hour='10',
+                  minute='30',
+                  misfire_grace_time=10,
+                  second='0', timezone='Europe/Paris', max_instances=1, args=[])
+
+
+def config_and_start_jobs():
+    executors = {
+        'default': ThreadPoolExecutor(30),
+        'processpool': ProcessPoolExecutor(12)
+    }
+    sched = BlockingScheduler(executors=executors)
+    start_check_results_job(sched)
+    sched.print_jobs()
+    sched.start()
+
+
+if __name__ == '__main__':
+    config_and_start_jobs()

scheduler_test.py

@@ -0,0 +1,30 @@
+from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor
+from apscheduler.schedulers.blocking import BlockingScheduler
+
+from request_sender_test import start_send_requests
+
+
+def start_book_appointment():
+    start_send_requests()
+
+
+def start_check_results_job(sched):
+    sched.add_job(start_book_appointment, 'cron', day_of_week='mon-sat', hour='10',
+                  minute='30',
+                  misfire_grace_time=10,
+                  second='0', timezone='Europe/Paris', max_instances=1, args=[])
+
+
+def config_and_start_jobs():
+    executors = {
+        'default': ThreadPoolExecutor(30),
+        'processpool': ProcessPoolExecutor(12)
+    }
+    sched = BlockingScheduler(executors=executors)
+    start_check_results_job(sched)
+    sched.print_jobs()
+    sched.start()
+
+
+if __name__ == '__main__':
+    config_and_start_jobs()

utils/AppLogging.py

@@ -0,0 +1,13 @@
+import datetime
+import logging
+from pathlib import Path
+
+LOGS_DIR = str(Path.home())
+
+
+def init_logger():
+    logging.basicConfig(filename=LOGS_DIR + "/request_{}.log".format(str(datetime.date.today())),
+                        filemode='a',
+                        format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
+                        datefmt='%Y-%m-%d %H:%M:%S',
+                        level=logging.INFO)

utils/get_only_datadome_cookies.py

@@ -0,0 +1,70 @@
+from http.cookies import SimpleCookie
+
+
+def get_datadome_cookies(cookiesStr: str):
+    _cookies = SimpleCookie()
+    _cookies.load(cookiesStr)
+    new_cookies = {k: v.value for k, v in _cookies.items()}
+    _new_coolies_str = None
+    for key in new_cookies:
+        if key == "datadome":
+            if _new_coolies_str is None:
+                _new_coolies_str = ""
+            _new_coolies_str = _new_coolies_str + new_cookies[key]
+    return _new_coolies_str
+
+
+def retain_only_dataome_cookies(cookiesStr: str):
+    _cookies = SimpleCookie()
+    _cookies.load(cookiesStr)
+    new_cookies = {k: v.value for k, v in _cookies.items()}
+    new_coolies_str = None
+    for key in new_cookies:
+        if key == "datadome":
+            if new_coolies_str is None:
+                new_coolies_str = ""
+            new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
+    return new_coolies_str
+
+
+def get_app_cookies(cookiesStr: str):
+    _cookies = SimpleCookie()
+    _cookies.load(cookiesStr)
+    new_cookies = {k: v.value for k, v in _cookies.items()}
+    new_coolies_str = None
+    for key in new_cookies:
+        if key == "app.sig":
+            if new_coolies_str is None:
+                new_coolies_str = ""
+            new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
+    return new_coolies_str
+
+
+def get_lang_cookies(cookiesStr: str):
+    _cookies = SimpleCookie()
+    _cookies.load(cookiesStr)
+    new_cookies = {k: v.value for k, v in _cookies.items()}
+    new_coolies_str = None
+    for key in new_cookies:
+        if key == "app":
+            if new_coolies_str is None:
+                new_coolies_str = ""
+            new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
+        if key == "policy":
+            if new_coolies_str is None:
+                new_coolies_str = ""
+            new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
+        if key == "lang":
+            if new_coolies_str is None:
+                new_coolies_str = ""
+            new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
+    return new_coolies_str
+
+
+if __name__ == '__main__':
+    print(get_datadome_cookies(
+        "datadome=HS7HPdbsFTYfxD60ogUwxjplOuqrveIjcTaucXVIWCko6pRayhXuKxC~44Ky5eNR2wnx4PEd2P2AadewmeXgxi4vn3x_QKdxLb6H1a9MbaMLNwmxfMEjQQ~CK8Y_nspJ;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiNXhvX3ZFaWc0aURla0dYMHZGTHNJX2p4In0=;policy=accepted;lang=fr;"))
+    print(get_app_cookies(
+        "datadome=HS7HPdbsFTYfxD60ogUwxjplOuqrveIjcTaucXVIWCko6pRayhXuKxC~44Ky5eNR2wnx4PEd2P2AadewmeXgxi4vn3x_QKdxLb6H1a9MbaMLNwmxfMEjQQ~CK8Y_nspJ;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiNXhvX3ZFaWc0aURla0dYMHZGTHNJX2p4In0=;policy=accepted;lang=fr;"))
+    print(get_lang_cookies(
+        "datadome=HS7HPdbsFTYfxD60ogUwxjplOuqrveIjcTaucXVIWCko6pRayhXuKxC~44Ky5eNR2wnx4PEd2P2AadewmeXgxi4vn3x_QKdxLb6H1a9MbaMLNwmxfMEjQQ~CK8Y_nspJ;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiNXhvX3ZFaWc0aURla0dYMHZGTHNJX2p4In0=;policy=accepted;lang=fr;"))

workers/captcha_result_getter.py

@@ -1,11 +1,14 @@
 import json
 import random
 import re
-from time import time
 from typing import Union
 import requests
 
+from models.jsdata_le_pojo import JsDataLeTypePojo
 from models.jsdata_pojo import JsDataPojo
+from models.result_pojo import RequestResult
+from utils.get_only_datadome_cookies import get_datadome_cookies, get_app_cookies, get_lang_cookies, \
+    retain_only_dataome_cookies
 from workers.proxies_constants import PROXY_LIST_FR
 
 API_KEY = "d66aaf490d8aa424a5175e1fbd1aadea"
@@ -22,10 +25,10 @@ class CaptchaResultGetter:
         self.cookie_str = 'datadome=5Nq~NEP_qQSHC0g_lZNnZmEv36J8gVV~rpZ329xmCkTq2~H3meIoXr4h_b988qB2XW5Te7iEGsvq8BzA5KeFupyrZFh4kgrDyl8hT2UymSByKHzAcDaNIBPDsRu2g_KG; Max-Age=31536000; Domain=.hermes.com; Path=/; Secure; SameSite=None'
         pass
 
-    def get_csrf(self, proxy_to_use, cookie: str = None) -> Union[str, None]:
+    def get_csrf(self, proxy_to_use, js_data: JsDataPojo, cookie: str = None) -> Union[str, RequestResult]:
         if cookie is not None:
             headers = {'Content-Type': 'application/x-www-form-urlencoded',
-                       'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
+                       'User-Agent': js_data.ua,
                        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
                        'Referer': 'https://www.google.fr',
                        'Cookie': cookie,
@@ -34,7 +37,7 @@ class CaptchaResultGetter:
                        'Accept-Language': 'fr-FR,fr;q=0.6'}
         else:
             headers = {'Content-Type': 'application/x-www-form-urlencoded',
-                       'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
+                       'User-Agent': js_data.ua,
                        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
                        'Referer': 'https://www.google.fr',
                        'Sec-Fetch-Mode': 'navigate',
@@ -47,13 +50,16 @@ class CaptchaResultGetter:
                                     timeout=15)
             print(response.status_code)
             if response.status_code == 200:
-                # print(response.text)
+                print(response.text)
-                print(response.url)
                 return self.extract_csrf_from_html(response.text)
+            elif response.status_code == 403:
+                return RequestResult.BLOCKED
             else:
-                return None
+                print(response.text)
+                return RequestResult.UNKNOWN
         except Exception as error:
             print(error)
+            return RequestResult.PROXY_ERROR
 
     def extract_csrf_from_html(self, html: str) -> Union[str, None]:
         result = re.findall(r'_csrf" value="[A-Za-z0-9-_]+', html)
@@ -63,9 +69,10 @@ class CaptchaResultGetter:
             return result_list[-1]
         return None
 
-    def get_valid_cookie(self, proxy_to_use, old_valid_cookie: str, dvm=4, hc=6) -> Union[str, None]:
+    def get_le_valid_cookie(self, proxy_to_use, js_le_type_data: JsDataLeTypePojo, old_valid_cookie: str) -> Union[
+        str, None]:
         headers = {'content-Type': 'application/x-www-form-urlencoded',
-                   'user-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
+                   'user-Agent': js_le_type_data.ua,
                    'accept': '*/*',
                    'referer': 'https://rendezvousparis.hermes.com/',
                    'origin': 'https://rendezvousparis.hermes.com/',
@@ -76,16 +83,15 @@ class CaptchaResultGetter:
                    'sec-ch-ua-platform': 'Android',
                    'accept-encoding': 'gzip, deflate, br',
                    'accept-language': 'fr-FR,fr;q=0.6'}
-        print("send request to get new cookie")
+        print("send request to get le type cookie")
         print(proxy_to_use)
         print(headers)
         try:
-            # tag_pu = 10 * Math.random()
+            raw_data = self.get_le_type_raw_data(old_valid_cookie=old_valid_cookie, js_le_type_data=js_le_type_data)
-            raw_data = self.get_raw_data(old_valid_cookie=old_valid_cookie, dvm=dvm, hc=hc)
             response = requests.post(url="https://d.digital.hermes/js/", headers=headers, verify=False,
                                      data=raw_data,
                                      proxies=proxy_to_use, timeout=15)
-            print(response.status_code)
+            # print(response.status_code)
             if response.status_code == 200:
                 print(response.text)
                 returned_result = json.loads(response.text)
@@ -99,20 +105,32 @@ class CaptchaResultGetter:
             return None
 
     def get_ch_raw_data_from_js_data(self, js_data: JsDataPojo, old_valid_cookie) -> str:
-        _tag_version = "4.24.1"
+        _tag_version = "4.25.1"
         _raw_data = "jsData={}&eventCounters=%5B%5D&jsType=ch&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv={}".format(
             js_data.to_url_encoded_json(), old_valid_cookie, _tag_version)
-        print("raw data is " + _raw_data)
+        print("raw ch data is " + _raw_data)
         return _raw_data
 
     def get_valid_ch_cookie(self, proxy_to_use, js_data: JsDataPojo, old_valid_cookie: str = None) -> Union[str, None]:
+        # extract datadome cookie
+        # get new cookie
+        _cookies_to_use = get_datadome_cookies(old_valid_cookie)
+        _app_cookies = get_app_cookies(old_valid_cookie)
+        _lang_cookies = get_lang_cookies(old_valid_cookie)
+        _origin = "https://rendezvousparis.hermes.com/"
+        # _origin = "https://www.hermes.com"
+        _referer = "https://rendezvousparis.hermes.com/"
+        # _referer = "https://www.hermes.com"
+
         headers = {'content-Type': 'application/x-www-form-urlencoded',
                    'user-Agent': js_data.ua,
                    'accept': '*/*',
-                   'referer': 'https://rendezvousparis.hermes.com/',
+                   'referer': _referer,
-                   'origin': 'https://rendezvousparis.hermes.com/',
+                   'origin': _origin,
+                   'request': _origin,
                    'Sec-Fetch-Mode': 'cors',
                    'sec-fetch-site': 'cross-site',
+                   'sec-ch-ua-mobile': '?1',
                    'sec-gpc': "1",
                    'sec-fetch-dest': 'empty',
                    'accept-encoding': 'gzip, deflate, br',
@@ -121,10 +139,9 @@ class CaptchaResultGetter:
         print("send request to get new cookie")
         print(headers)
         try:
-            # tag_pu = 10 * Math.random()
             response = requests.post(url="https://d.digital.hermes/js/", headers=headers, verify=False,
                                      data=self.get_ch_raw_data_from_js_data(js_data=js_data,
-                                                                            old_valid_cookie=old_valid_cookie),
+                                                                            old_valid_cookie=_cookies_to_use),
                                      proxies=proxy_to_use, timeout=15)
             print(response.status_code)
             if response.status_code == 200:
@@ -132,6 +149,9 @@ class CaptchaResultGetter:
                 returned_result = json.loads(response.text)
                 new_cookie = returned_result['cookie']
                 print(response.url)
+                new_cookie = retain_only_dataome_cookies(new_cookie)
+                if _app_cookies is not None:
+                    new_cookie = _app_cookies + new_cookie + _lang_cookies
                 print(new_cookie)
                 return new_cookie
         except Exception as error:
@@ -139,27 +159,25 @@ class CaptchaResultGetter:
             print(error)
             return None
 
-    def get_raw_data(self, old_valid_cookie: str = None, dvm=4, hc=6):
+    def get_le_type_raw_data(self, old_valid_cookie: str, js_le_type_data: JsDataLeTypePojo):
-        _raw_data = None
+        # m_m_c = f.mousemove
-
+        mousemove_count = js_le_type_data.m_m_c
-        mousemove_count = random.randint(5, 10)
+        # m_c_c = f.click
-        click_count = random.randint(5, 10)
+        click_count = js_le_type_data.m_c_c
-        click_count = mousemove_count
+        # m_s_c = f.scroll
-        key_count = random.randint(5, 10)
+        scroll_count = js_le_type_data.m_s_c
+        key_count = random.randint(5, 12)
         touch_count = mousemove_count
-        js_set = int(time())
+        touch_move = touch_count + 2
-        tag_pu = random.uniform(1, 10)
+        # _le_js_raw_data = self.get_le_raw_data_from_js_data(js_le_type_data=js_le_type_data,
-        while len(str(tag_pu)) != 17:
+        #                                                     old_valid_cookie=old_valid_cookie)
-            tag_pu = random.uniform(1, 10)
+        _cid = get_datadome_cookies(old_valid_cookie)
-
+        _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.25.1".format(
-        if old_valid_cookie is None:
+            js_le_type_data.to_url_encoded_json(), mousemove_count, click_count, scroll_count, touch_count, touch_count,
-            _raw_data = "jsData=%7B%22log1%22%3Afalse%2C%22opts%22%3A%22endpoint%2CajaxListenerPath%22%2C%22ttst%22%3A38%2C%22ifov%22%3Afalse%2C%22hc%22%3A{}%2C%22br_oh%22%3A646%2C%22br_ow%22%3A360%2C%22ua%22%3A%22Mozilla%2F5.0%20(Linux%3B%20Android%2010%3B%20K)%20AppleWebKit%2F537.36%20(KHTML%2C%20like%20Gecko)%20Chrome%2F115.0.0.0%20Mobile%20Safari%2F537.36%22%2C%22wbd%22%3Afalse%2C%22tagpu%22%3A{}%2C%22wdif%22%3Afalse%2C%22wdifrm%22%3Afalse%2C%22npmtm%22%3Afalse%2C%22br_h%22%3A646%2C%22br_w%22%3A360%2C%22nddc%22%3A1%2C%22rs_h%22%3A800%2C%22rs_w%22%3A360%2C%22rs_cd%22%3A24%2C%22phe%22%3Afalse%2C%22nm%22%3Afalse%2C%22jsf%22%3Afalse%2C%22lg%22%3A%22fr-FR%22%2C%22pr%22%3A3%2C%22ars_h%22%3A800%2C%22ars_w%22%3A360%2C%22tz%22%3A-60%2C%22str_ss%22%3Atrue%2C%22str_ls%22%3Atrue%2C%22str_idb%22%3Atrue%2C%22str_odb%22%3Atrue%2C%22plgod%22%3Afalse%2C%22plg%22%3A2%2C%22plgne%22%3A%22err%22%2C%22plgre%22%3A%22err%22%2C%22plgof%22%3A%22err%22%2C%22plggt%22%3A%22err%22%2C%22pltod%22%3Afalse%2C%22hcovdr%22%3Afalse%2C%22hcovdr2%22%3Afalse%2C%22plovdr%22%3Afalse%2C%22plovdr2%22%3Afalse%2C%22ftsovdr%22%3Afalse%2C%22ftsovdr2%22%3Afalse%2C%22lb%22%3Afalse%2C%22eva%22%3A33%2C%22lo%22%3Afalse%2C%22ts_mtp%22%3A5%2C%22ts_tec%22%3Atrue%2C%22ts_tsa%22%3Atrue%2C%22vnd%22%3A%22Google%20Inc.%22%2C%22bid%22%3A%22NA%22%2C%22mmt%22%3A%22empty%22%2C%22plu%22%3A%226GLkaVK%2CjRvAfXq0%22%2C%22hdn%22%3Afalse%2C%22awe%22%3Afalse%2C%22geb%22%3Afalse%2C%22dat%22%3Afalse%2C%22med%22%3A%22defined%22%2C%22aco%22%3A%22probably%22%2C%22acots%22%3Afalse%2C%22acmp%22%3A%22probably%22%2C%22acmpts%22%3Atrue%2C%22acw%22%3A%22probably%22%2C%22acwts%22%3Afalse%2C%22acma%22%3A%22maybe%22%2C%22acmats%22%3Afalse%2C%22acaa%22%3A%22probably%22%2C%22acaats%22%3Atrue%2C%22ac3%22%3A%22%22%2C%22ac3ts%22%3Afalse%2C%22acf%22%3A%22probably%22%2C%22acfts%22%3Afalse%2C%22acmp4%22%3A%22maybe%22%2C%22acmp4ts%22%3Afalse%2C%22acmp3%22%3A%22probably%22%2C%22acmp3ts%22%3Afalse%2C%22acwm%22%3A%22maybe%22%2C%22acwmts%22%3Afalse%2C%22ocpt%22%3Afalse%2C%22vco%22%3A%22%22%2C%22vcots%22%3Afalse%2C%22vch%22%3A%22probably%22%2C%22vchts%22%3Atrue%2C%22vcw%22%3A%22probably%22%2C%22vcwts%22%3Atrue%2C%22vc3%22%3A%22maybe%22%2C%22vc3ts%22%3Afalse%2C%22vcmp%22%3A%22%22%2C%22vcmpts%22%3Afalse%2C%22vcq%22%3A%22%22%2C%22vcqts%22%3Afalse%2C%22vc1%22%3A%22probably%22%2C%22vc1ts%22%3Atrue%2C%22dvm%22%3A{}%2C%22sqt%22%3Afalse%2C%22so%22%3A%22portrait-primary%22%2C%22wdw%22%3Atrue%2C%22cokys%22%3A%22bG9hZFRpbWVzY3NpL%3D%22%2C%22ecpc%22%3Afalse%2C%22lgs%22%3Atrue%2C%22lgsod%22%3Afalse%2C%22psn%22%3Atrue%2C%22edp%22%3Afalse%2C%22addt%22%3Atrue%2C%22wsdc%22%3Atrue%2C%22ccsr%22%3Atrue%2C%22nuad%22%3Atrue%2C%22bcda%22%3Atrue%2C%22idn%22%3Atrue%2C%22capi%22%3Afalse%2C%22svde%22%3Afalse%2C%22vpbq%22%3Atrue%2C%22ucdv%22%3Afalse%2C%22spwn%22%3Afalse%2C%22emt%22%3Afalse%2C%22bfr%22%3Afalse%2C%22dbov%22%3Afalse%2C%22cfpfe%22%3A%22ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ3MCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1snXHg3M1x4NjhceDYxXHg2%22%2C%22stcfp%22%3A%22Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzUyMzcpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWEgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODg0NDkpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMTY1%22%2C%22ckwa%22%3Atrue%2C%22glvd%22%3A%22Google%20Inc.%20(Qualcomm)%22%2C%22glrd%22%3A%22ANGLE%20(Qualcomm%2C%20Adreno%20(TM)%20650%2C%20OpenGL%20ES%203.2)%22%2C%22wwl%22%3Afalse%2C%22jset%22%3A{}%2C%22dcok%22%3A%22.hermes.com%22%2C%22log2%22%3Atrue%2C%22tbce%22%3A0%2C%22es_sigmdn%22%3Anull%2C%22es_mumdn%22%3Anull%2C%22es_distmdn%22%3Anull%2C%22es_angsmdn%22%3Anull%2C%22es_angemdn%22%3Anull%2C%22m_s_c%22%3A3%2C%22m_m_c%22%3A5%2C%22m_c_c%22%3A5%2C%22m_cm_r%22%3A1%2C%22m_ms_r%22%3A1.6666666666666667%7D&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A3%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A0%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid=S2ilUoO80StZ2OfedRZnvyMiJ9hmD_DyIWj11O2eak~8TOek7JTTeJeKWDLzGgMZaH2IOix3BX1ocB4TwnLQFx~76pAQAD0JnVZzoxwbxOPL2SR834BfbiK0Nu4c0Pxh&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.19.3".format(
+            touch_move,
-                hc, tag_pu, dvm, js_set, mousemove_count, click_count, touch_count, touch_count, key_count, key_count)
+            key_count,
-        else:
+            key_count, _cid)
-            _raw_data = "jsData=%7B%22log1%22%3Afalse%2C%22opts%22%3A%22endpoint%2CajaxListenerPath%22%2C%22ttst%22%3A38%2C%22ifov%22%3Afalse%2C%22hc%22%3A{}%2C%22br_oh%22%3A646%2C%22br_ow%22%3A360%2C%22ua%22%3A%22Mozilla%2F5.0%20(Linux%3B%20Android%2010%3B%20K)%20AppleWebKit%2F537.36%20(KHTML%2C%20like%20Gecko)%20Chrome%2F115.0.0.0%20Mobile%20Safari%2F537.36%22%2C%22wbd%22%3Afalse%2C%22tagpu%22%3A{}%2C%22wdif%22%3Afalse%2C%22wdifrm%22%3Afalse%2C%22npmtm%22%3Afalse%2C%22br_h%22%3A646%2C%22br_w%22%3A360%2C%22nddc%22%3A1%2C%22rs_h%22%3A800%2C%22rs_w%22%3A360%2C%22rs_cd%22%3A24%2C%22phe%22%3Afalse%2C%22nm%22%3Afalse%2C%22jsf%22%3Afalse%2C%22lg%22%3A%22fr-FR%22%2C%22pr%22%3A3%2C%22ars_h%22%3A800%2C%22ars_w%22%3A360%2C%22tz%22%3A-60%2C%22str_ss%22%3Atrue%2C%22str_ls%22%3Atrue%2C%22str_idb%22%3Atrue%2C%22str_odb%22%3Atrue%2C%22plgod%22%3Afalse%2C%22plg%22%3A2%2C%22plgne%22%3A%22err%22%2C%22plgre%22%3A%22err%22%2C%22plgof%22%3A%22err%22%2C%22plggt%22%3A%22err%22%2C%22pltod%22%3Afalse%2C%22hcovdr%22%3Afalse%2C%22hcovdr2%22%3Afalse%2C%22plovdr%22%3Afalse%2C%22plovdr2%22%3Afalse%2C%22ftsovdr%22%3Afalse%2C%22ftsovdr2%22%3Afalse%2C%22lb%22%3Afalse%2C%22eva%22%3A33%2C%22lo%22%3Afalse%2C%22ts_mtp%22%3A5%2C%22ts_tec%22%3Atrue%2C%22ts_tsa%22%3Atrue%2C%22vnd%22%3A%22Google%20Inc.%22%2C%22bid%22%3A%22NA%22%2C%22mmt%22%3A%22empty%22%2C%22plu%22%3A%226GLkaVK%2CjRvAfXq0%22%2C%22hdn%22%3Afalse%2C%22awe%22%3Afalse%2C%22geb%22%3Afalse%2C%22dat%22%3Afalse%2C%22med%22%3A%22defined%22%2C%22aco%22%3A%22probably%22%2C%22acots%22%3Afalse%2C%22acmp%22%3A%22probably%22%2C%22acmpts%22%3Atrue%2C%22acw%22%3A%22probably%22%2C%22acwts%22%3Afalse%2C%22acma%22%3A%22maybe%22%2C%22acmats%22%3Afalse%2C%22acaa%22%3A%22probably%22%2C%22acaats%22%3Atrue%2C%22ac3%22%3A%22%22%2C%22ac3ts%22%3Afalse%2C%22acf%22%3A%22probably%22%2C%22acfts%22%3Afalse%2C%22acmp4%22%3A%22maybe%22%2C%22acmp4ts%22%3Afalse%2C%22acmp3%22%3A%22probably%22%2C%22acmp3ts%22%3Afalse%2C%22acwm%22%3A%22maybe%22%2C%22acwmts%22%3Afalse%2C%22ocpt%22%3Afalse%2C%22vco%22%3A%22%22%2C%22vcots%22%3Afalse%2C%22vch%22%3A%22probably%22%2C%22vchts%22%3Atrue%2C%22vcw%22%3A%22probably%22%2C%22vcwts%22%3Atrue%2C%22vc3%22%3A%22maybe%22%2C%22vc3ts%22%3Afalse%2C%22vcmp%22%3A%22%22%2C%22vcmpts%22%3Afalse%2C%22vcq%22%3A%22%22%2C%22vcqts%22%3Afalse%2C%22vc1%22%3A%22probably%22%2C%22vc1ts%22%3Atrue%2C%22dvm%22%3A{}%2C%22sqt%22%3Afalse%2C%22so%22%3A%22portrait-primary%22%2C%22wdw%22%3Atrue%2C%22cokys%22%3A%22bG9hZFRpbWVzY3NpL%3D%22%2C%22ecpc%22%3Afalse%2C%22lgs%22%3Atrue%2C%22lgsod%22%3Afalse%2C%22psn%22%3Atrue%2C%22edp%22%3Afalse%2C%22addt%22%3Atrue%2C%22wsdc%22%3Atrue%2C%22ccsr%22%3Atrue%2C%22nuad%22%3Atrue%2C%22bcda%22%3Atrue%2C%22idn%22%3Atrue%2C%22capi%22%3Afalse%2C%22svde%22%3Afalse%2C%22vpbq%22%3Atrue%2C%22ucdv%22%3Afalse%2C%22spwn%22%3Afalse%2C%22emt%22%3Afalse%2C%22bfr%22%3Afalse%2C%22dbov%22%3Afalse%2C%22cfpfe%22%3A%22ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ3MCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1snXHg3M1x4NjhceDYxXHg2%22%2C%22stcfp%22%3A%22Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzUyMzcpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWEgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODg0NDkpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMTY1%22%2C%22ckwa%22%3Atrue%2C%22glvd%22%3A%22Google%20Inc.%20(Qualcomm)%22%2C%22glrd%22%3A%22ANGLE%20(Qualcomm%2C%20Adreno%20(TM)%20650%2C%20OpenGL%20ES%203.2)%22%2C%22wwl%22%3Afalse%2C%22jset%22%3A{}%2C%22dcok%22%3A%22.hermes.com%22%2C%22log2%22%3Atrue%2C%22tbce%22%3A0%2C%22es_sigmdn%22%3Anull%2C%22es_mumdn%22%3Anull%2C%22es_distmdn%22%3Anull%2C%22es_angsmdn%22%3Anull%2C%22es_angemdn%22%3Anull%2C%22m_s_c%22%3A3%2C%22m_m_c%22%3A5%2C%22m_c_c%22%3A5%2C%22m_cm_r%22%3A1%2C%22m_ms_r%22%3A1.6666666666666667%7D&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A3%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A0%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.19.3".format(
+        print("raw le data is " + _raw_data)
-                hc, tag_pu, dvm, js_set, mousemove_count, click_count, touch_count, touch_count, key_count,
-                key_count, old_valid_cookie)
-        print("raw data is " + _raw_data)
         return _raw_data
 
     def extract_cid_from_html(self, html_text):
@@ -182,48 +200,6 @@ class CaptchaResultGetter:
 
 #
 if __name__ == '__main__':
-    # print(int(time()))
     captchaResultGetter = CaptchaResultGetter()
     _url_encoded_data = captchaResultGetter.get_valid_cookie()
     print(_url_encoded_data)
-
-    # print(urllib.parse.parse_qs((_url_encoded_data)))
-    # valid_cookie = captchaResultGetter.get_valid_cookie()
-    # simple_cookie = SimpleCookie()
-    # simple_cookie.load(valid_cookie)
-    # new_cookies = {k: v.value for k, v in simple_cookie.items()}
-    # new_coolies_str = ""
-    # for key in new_cookies:
-    #     print(key)
-    #     new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
-    # print(new_coolies_str)
-    # valid_cookie = new_coolies_str + "app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;policy=accepted;lang=fr;"
-    # print(valid_cookie)
-#     # csrf = captchaResultGetter.extract_csrf_from_html(
-#     #     """strong>Français</strong> / <a href="?lang=en">English</a> / <a href="?lang=cn">中文</a> / <a href="?lang=jp">日本語</a> / <a href="?lang=kr">한국어</a> / <a href="?lang=ru">Русский</a></div><div class="row"><div class="col-sm-6 offset-sm-3"></div></div><div class="row"><div class="col-md-6 offset-md-3"><div class="text-center"><h1>Bienvenue chez Hermès</h1><div class="client-nav register-for text-center">Demande de rendez-vous pour le vendredi 12 janvier</div><hr/></div><form id="client-registration-form" method="post" action="/client/register" class=""><input type="hidden" name="check" id="check" /><input type="hidden" name="_csrf" value="Qw5FkdRf--yiWD6S527G7Qt40DCLAzk1q21Y" /><div class="mb-3"><label for="prefer" class="control-label">Magasin préféré</label><select id="prefer" name="prefer" class="form-control" data-value=""><option value="">Sans préférence</option><option value="faubourg">Hermès Faubourg Saint-Honoré</option><option value="georgev">Hermès George V</option><option value="sevres">Hermès Sèvres</option></select></div><div class="mb-3">""")
-#     # print(csrf)
-#     contact_list = read_contacts('/Users/panlei/Desktop/yahoo_aol_valid_26-2.xlsx')
-#     # remove already booked contacts
-#     sub_contact_list = filter_contacts(contact_list)
-#     random.shuffle(sub_contact_list)
-#     valid_cookie = captchaResultGetter.get_valid_cookie()
-#     contact = random.choice(sub_contact_list)
-#     if valid_cookie is not None:
-#         # new_csrf = captchaResultGetter.get_csrf(valid_cookie)
-#         new_csrf = None
-#         # if new_csrf is not None:
-#         #     print(new_csrf)
-#         simple_cookie = SimpleCookie()
-#         simple_cookie.load(valid_cookie)
-#         new_cookies = {k: v.value for k, v in simple_cookie.items()}
-#         new_coolies_str = ""
-#         for key in new_cookies:
-#             print(key)
-#             new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
-#         print(new_coolies_str)
-#         valid_cookie = new_coolies_str + "app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;policy=accepted;lang=fr;"
-#         print(valid_cookie)
-#         new_csrf = captchaResultGetter.get_csrf(valid_cookie)
-#         if new_csrf is not None:
-#             sender = Sender(valid_cookie)
-#             sender.send_request(HERMES_REGISTER, contact=contact, csrf=new_csrf)

workers/link_validator.py

@@ -1,3 +1,4 @@
+import json
 import random
 import threading
 import time
@@ -10,8 +11,8 @@ from db.mongo_manager import MONGO_STORE_MANAGER
 from models.LinkPojo import LinkPojo
 from models.result_pojo import RequestResult
 from proxy_manager.proxy_manager import ProxyManager
-from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST
+from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST, TEST_QUEUE, SHARED_OBJECT
-from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials, REQUEST_DATA_DE
+from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
 from workers.proxies_constants import PROXY_LIST_FR
 
 DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ces coordonnées"
@@ -35,7 +36,6 @@ class LinkValidator(threading.Thread):
         self.filter_with_ip_country()
         self.proxy_manager = proxy_manager
         self.limit = limit
-        # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
 
     def set_up_connection(self):
         self.connection = pika.BlockingConnection(
@@ -47,10 +47,13 @@ class LinkValidator(threading.Thread):
         self.channel.basic_consume(queue=self.queue_to_listen, auto_ack=False, on_message_callback=callback)
         self.channel.start_consuming()
 
-    def send_request(self, linkPojo: LinkPojo) -> RequestResult:
+    def send_request(self, linkPojo: LinkPojo, _received_dict=None) -> RequestResult:
+        _ua = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36'
+        if _received_dict is not None:
+            _ua = _received_dict['ua']
         self.cookie.load(self.cookie_str)
         headers = {
-            'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
+            'User-Agent': _ua,
             'Accept': '*/*',
             'Accept-Encoding': 'gzip, deflate, br',
             'Cache-Control': 'max-age=0',
@@ -66,7 +69,9 @@ class LinkValidator(threading.Thread):
         print("received cookie is " + str(self.cookie_str))
         print("send request for link: " + linkPojo.url)
         try:
-            response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use,
+            print("will send request with ua {}".format(_ua))
+            print("will send request with cookie {}".format(self.cookie_str))
+            response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
                                     timeout=60)
             print(response.status_code)
             if response.status_code == 200:
@@ -85,6 +90,11 @@ class LinkValidator(threading.Thread):
                         new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
                     print("will publish to queue {}".format(new_coolies_str))
                     # upload the cookie to queue
+                    if _received_dict is not None:
+                        _received_dict['cookiesStr'] = new_coolies_str
+                        print("body in json:{}".format(json.dumps(_received_dict)))
+                        self.cookiesPublisher.publish_body(json.dumps(_received_dict))
+                    else:
                         self.cookiesPublisher.publish_body(new_coolies_str)
                     self.cookie_str = new_coolies_str
                     return RequestResult.SUCCESS
@@ -151,16 +161,24 @@ class LinkValidator(threading.Thread):
         print("message count in queue is {}".format(_message_in_queue_count))
         self.update_validate_list()
         self.filter_with_ip_country()
-        self.cookie_str = body.decode("UTF-8")
+        _received_object = body.decode("UTF-8")
+        js_data = None
+        _received_dict = None
+        if "glrd" in _received_object:
+            _received_dict = json.loads(_received_object)
+            _received_cookies = _received_dict["cookiesStr"]
+        else:
+            _received_cookies = _received_object
+        self.cookie_str = _received_cookies
         random.shuffle(self.link_to_validate_list)
-        if len(self.link_to_validate_list) > 0 and _message_in_queue_count > self.limit:
+        if len(self.link_to_validate_list) > 0 and _message_in_queue_count >= self.limit:
             print("links number is {}".format(len(self.link_to_validate_list)))
             can_continue = None
             for con in self.link_to_validate_list:
                 # if not is_already_sent(con):
                 print(con.email)
                 self.proxy_to_use = random.choice(PROXY_LIST_FR)
-                can_continue = self.send_request(con)
+                can_continue = self.send_request(con, _received_dict)
                 if can_continue == RequestResult.BLOCKED:
                     print("cannot continue, blocked, then skip")
                     break
@@ -190,13 +208,12 @@ class LinkValidator(threading.Thread):
 
 
 def validate_with_FR_ip(segment_position=1):
-    _queue_name = REQUEST_DATA_QUEUE
+    _queue_name = TEST_QUEUE
-    cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
+    cookiesPublisher = CookiesPublisher(queue_name=TEST_QUEUE)
     cookiesPublisher.set_up_connection()
     print("filter links with ip_country")
     _proxy_manager = ProxyManager()
-    receiver = LinkValidator(cookiesPublisher=cookiesPublisher,
+    receiver = LinkValidator(cookiesPublisher=cookiesPublisher, proxy_manager=_proxy_manager,
-                             proxy_manager=_proxy_manager,
                              queue_to_listen=_queue_name, ip_country="FR", segment_position=segment_position, limit=0)
     print("will connect to queue")
     receiver.set_up_connection()

workers/proxies_constants.py

@@ -42,12 +42,21 @@ FR_PROXY_RES_OXY = {
     'https': 'http://customer-rendezvous-cc-FR:Rdv202220212023@pr.oxylabs.io:7777'
 }
 
-
 FR_PROXY_MOB_OXY = {
     'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@pr.oxylabs.io:7777',
     'https': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@pr.oxylabs.io:7777'
 }
 
+FR_PROXY_MOB_OXY_STICKY = {
+    'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}',
+    'https': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}'
+}
+
+FR_PROXY_RES_OXY_STICKY = {
+    'http': 'http://customer-rendezvous-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}',
+    'https': 'http://customer-rendezvous-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}'
+}
+
 DE_PROXY_MOBILE = {
     'http': 'http://8153587-mobile-country-DE:jp50x9jmo@194.88.106.169:11842',
     'https': 'http://8153587-mobile-country-DE:jp50x9jmo@194.88.106.169:11842'
@@ -57,7 +66,6 @@ DE_PROXY_RES = {
     'https': 'http://8254565-res-country-DE:1st57izbhe@185.21.60.181:14648'
 }
 
-
 # FR_PROXY_MOBILE
 # FR_PROXY_RES
 # PROXY_LIST = [FR_PROXY_MOBILE, FR_PROXY_RES, DE_PROXY_RES, DE_PROXY_MOBILE, ES_PROXY_MOBILE, IT_PROXY_MOBILE]
@@ -65,7 +73,4 @@ DE_PROXY_RES = {
 # PROXY_LIST_FR = [FR_PROXY_RES_OXY, FR_PROXY_MOBILE_3]
 # PROXY_LIST_FR = [ FR_PROXY_MOBILE_3]
 PROXY_LIST_FR = [FR_PROXY_RES_OXY]
-MOBILE_PROXY_LIST_FR = [FR_PROXY_MOB_OXY]
+MOBILE_PROXY_LIST_FR = [FR_PROXY_MOB_OXY_STICKY]
-# PROXY_LIST = [FR_PROXY_MOBILE, FR_
-# PROXY_RES]
-PROXY_LIST_DE = [DE_PROXY_RES, DE_PROXY_MOBILE]

workers/sender.py

@@ -8,6 +8,7 @@ import requests
 
 from db.mongo_manager import MONGO_STORE_MANAGER
 from models.ReserveResultPojo import ReserveResultPojo, PublishType
+from models.jsdata_pojo import JsDataPojo
 from models.result_pojo import RequestResult
 from queue_message.CookiesPublisher import CookiesPublisher
 from workers.proxies_constants import PROXY_LIST_FR
@@ -15,18 +16,19 @@ from workers.proxies_constants import PROXY_LIST_FR
 
 class Sender:
 
-    def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher, received_dict, proxy_to_use):
+    def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher, received_dict, proxy_to_use, logger):
         self.store_type = "random"
         self.cookie = SimpleCookie()
         self.cookiesPublisher = cookiesPublisher
         # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
         self.cookie_str = cookie_str
+        self.logger = logger
         self._csrf = None
         self.received_dict = received_dict
         self.proxy_to_use = proxy_to_use
         self.cookie.load(self.cookie_str)
 
-    def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str):
+    def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str, store_type: str):
         # create the message
         if url == "https://rendezvousparis.hermes.com/client/welcome":
             return
@@ -35,7 +37,7 @@ class Sender:
                                    firstName=contact.first_name, lastName=contact.last_name, email=contact.mail,
                                    passport=contact.passport, ccid=contact.ccid)
         result.id = id
-        result.store_type = self.store_type
+        result.store_type = store_type
         result.created_at = time.strftime("%H:%M:%S", time.localtime())
         collection_name = str(datetime.date.today())
         MONGO_STORE_MANAGER.insert_reserve_result(collection_name=collection_name, reserve=result)
@@ -46,7 +48,7 @@ class Sender:
     def get_csrf(self):
         pass
 
-    def send_request(self, url, contact: ContactPojo, csrf: str = None) -> RequestResult:
+    def send_request(self, url, js_data: JsDataPojo, contact: ContactPojo, csrf: str = None) -> RequestResult:
         if csrf is None:
             if self._csrf is None:
                 _csrf = '8Bs2dBwb-nHONOzo9Tei2CcMZglEfsRqUz8E'
@@ -55,44 +57,58 @@ class Sender:
         else:
             self._csrf = csrf
             _csrf = csrf
+        # 默认为空，也就是随机
+        _selected_store = ''
+        if contact.store is not None and contact.store != "random":
+            _selected_store = contact.store
+        self.logger.info("store is:{}".format(_selected_store))
         headers = {'Content-Type': 'application/x-www-form-urlencoded',
-                   'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
+                   'User-Agent': js_data.ua,
                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
                    'Cookie': self.cookie_str, 'Referer': 'https://rendezvousparis.hermes.com/client/register',
                    'Sec-Fetch-Mode': 'navigate',
                    'Sec-Fetch-Dest': 'document',
                    'Accept-Language': 'fr-FR,fr;q=0.6'}
-        data = {'check': '', '_csrf': _csrf, 'prefer': '',
+        data = {'check': '', '_csrf': _csrf, 'prefer': _selected_store,
                 'surname': contact.last_name.capitalize(), 'name': contact.first_name.capitalize(),
                 'phone_country': "FR", 'phone_number': "+33 " + contact.phone, 'email': contact.mail,
                 'passport_id': contact.passport, 'processing': 'on', 'cgu': 'on'}
-        print(data)
+        self.logger.info(data)
+        self.logger.info("try to request for mail:{}".format(contact.mail))
+        print("send request with cookie:{}".format(self.cookie_str))
         try:
             proxy_to_use = self.proxy_to_use
-            # proxy_to_use = PROXY_LIST[0]
+            # print(proxy_to_use)
-            print(proxy_to_use)
             response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data,
                                      timeout=15)
-            print(response.status_code)
+            self.logger.info(response.status_code)
             if response.status_code == 200:
                 # add to mongodb
-                print(response.text)
+                self.logger.info(response.text)
-                print(response.url)
+                self.logger.info("{}:{}".format(contact.mail, response.url))
-                self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=response.url)
+                self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=response.url,
+                                              store_type=_selected_store)
                 cookies_to_set = response.headers['set-cookie']
                 self.cookie.load(cookies_to_set)
                 new_cookies = {k: v.value for k, v in self.cookie.items()}
                 new_coolies_str = ""
                 for key in new_cookies:
                     new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
-                print("will publish to queue {}".format(new_coolies_str))
+                self.logger.info("will publish to queue {}".format(new_coolies_str))
                 # upload the cookie to queue
                 self.received_dict['cookiesStr'] = new_coolies_str
-                print("body in json:{}".format(json.dumps(self.received_dict)))
+                self.logger.info("body in json:{}".format(json.dumps(self.received_dict)))
                 self.cookiesPublisher.publish_body(json.dumps(self.received_dict))
                 # self.cookie_str = new_coolies_str
+                self.logger.info("successful request, mail:{}".format(contact.mail))
                 return RequestResult.SUCCESS
             else:
+                print(response.text)
+                if response.status_code == 403:
+                    # try to load seesion
+                    # session = HTMLSession()
+                    self.logger.info("blocked by hermes, mail:{}".format(contact.mail))
+                    return RequestResult.BLOCKED
                 return RequestResult.BLOCKED
         except Exception as Error:
             print(Error)