check always saigecong1990@pissmail.com

# Conflicts:
#	mail/mail_reader_all_contacts.py
#	queue_message/appointmentrequestsender.py

db/mongo_manager.py

@@ -83,10 +83,11 @@ class MongoDbManager:
             result_list.append(ContactPojo.from_firestore_dict(document))
         return result_list
 
-    def save_links_to_validate(self, link: str, mail_address: str, _all_contact_list: list):
+    def save_links_to_validate(self, link: str, mail_address: str, model: str, _all_contact_list: list):
         collection_to_use = self.db[LINKS_TO_VALIDATE]
         updated_at = time.strftime("%H:%M:%S", time.localtime())
         _ip_country = "FR"
+        serial = "requests"
         # find ip_country info
         for _contact in _all_contact_list:
             if _contact.mail == mail_address:
@@ -96,6 +97,8 @@ class MongoDbManager:
             collection_to_use.replace_one(filter={'_id': mail_address, }, replacement={
                 u'url': link,
                 u'email': mail_address,
+                u'serial': serial,
+                u'model': model,
                 u'ip_country': _ip_country,
                 "updated_at": updated_at
             },
@@ -103,6 +106,8 @@ class MongoDbManager:
         else:
             collection_to_use.replace_one(filter={'_id': link, }, replacement={
                 u'url': link,
+                u'serial': serial,
+                u'model': model,
                 u'ip_country': _ip_country,
                 "updated_at": updated_at
             },

link_validator_executor.py

@@ -2,9 +2,6 @@ import time
 
 from workers.link_validator_with_provided_list import validate_all_links
 
-# if __name__ == '__main__':
-#     # link_list = MONGO_STORE_MANAGER.get_links_to_validate()
-#     validate_with_FR_ip(segment_position=2)
 
 if __name__ == '__main__':
     # generate test data
@@ -12,4 +9,4 @@ if __name__ == '__main__':
         print("call validate_all_links()")
         validate_all_links([])
         print("wait for 30 seconds")
-        time.sleep(30)
+        time.sleep(10)

mail/lan_mail_helper.py

@@ -0,0 +1,123 @@
+import logging
+import random
+import time
+
+import requests
+
+from db.mongo_manager import MONGO_STORE_MANAGER
+from mail.mail_reader_all_contacts import find_links_to_validate_from_mail_list
+from models.mail_pojo import MailAddress
+
+host = "https://authhk.bhdata.com:30015/bhmailer?uid=482391396&sign=32d7748da00047b9a1054c81a5750365"
+
+HERMES_EMAIL = "no-reply@hermes.com"
+
+
+def get_api_info():
+    _time = str(int(time.time() * 1000))
+    _api_info = host + "&act=getApiInfo&t=" + _time
+    print(_api_info)
+    res = requests.get(_api_info, verify=False)
+    print(res.text)
+
+
+def get_mail(mail: str):
+    _time = str(int(time.time() * 1000))
+    _hermes_mail = "Votre demande de rendez-vous"
+    _api_info = host + "&act=getMail&email={}&title={}&t={}".format(mail, _hermes_mail, _time)
+    print(_api_info)
+    res = requests.get(_api_info, verify=False)
+    print(res.text)
+
+
+def check_mail(mail: str):
+    print("check_mail called for {}".format(mail))
+    _time = str(int(time.time() * 1000))
+    _hermes_mail = HERMES_EMAIL
+    _subjet = "Votre rendez-vous est confirmé"
+    _api_info = host + "&act=checkMail&email={}&from={}&title={}&t={}".format(mail, _hermes_mail, _subjet, _time)
+    print(_api_info)
+    res = requests.get(_api_info, verify=False)
+    print(res.text)
+
+
+def check_appointment_link_mail(mail: str):
+    print("check_mail called for {}".format(mail))
+    _time = str(int(time.time() * 1000))
+    _hermes_mail = HERMES_EMAIL
+    _subjet = "Votre demande de rendez-vous"
+    _api_info = host + "&act=checkMail&email={}&from={}&title={}&t={}".format(mail, _hermes_mail, _subjet, _time)
+    print(_api_info)
+    res = requests.get(_api_info, verify=False)
+    print(res.text)
+
+
+def get_account(mail: str):
+    _time = str(int(time.time() * 1000))
+    _api_info = host + "&act=getAccount&email={}&t={}".format(mail, _time)
+    print(_api_info)
+    res = requests.get(_api_info, verify=False)
+    print(res.text)
+
+
+def filter_mail_with_links(_mail_list_to_filter):
+    _new_mail_list = []
+    _link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
+    for _mail in _mail_list_to_filter:
+        _to_add = True
+        for _link in _link_to_validate_list:
+            if _link.email == _mail:
+                _to_add = False
+        if _to_add:
+            _new_mail_list.append(_mail)
+    return _new_mail_list
+
+
+def get_mail_list_to_check():
+    successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+    _mail_list = []
+    for _item in successful_items:
+        if _item.url_validated is None or _item.url_validated != True:
+            _mail_list.append(_item.email)
+    return _mail_list
+
+
+def check_confirmed_mails():
+    successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+    # _mail_list = []
+    for _item in successful_items:
+        if "outlook" in _item.email or "hotmail" in _item.email:
+            check_mail(_item.email)
+            time.sleep(random.randint(1, 5))
+
+
+def check_all_need_to_check_emails():
+    logger = logging.getLogger()
+    _mail_list_before_filter = get_mail_list_to_check()
+    _mails = filter_mail_with_links(_mail_list_before_filter)
+    for _mail in _mails:
+        if "outlook.com" in _mail or "hotmail.com" in _mail:
+            check_mail(_mail)
+            time.sleep(2)
+
+    _mail_list = [MailAddress("saigecong1990@pissmail.com", "cvExXKOP8oY1D@")]
+    find_links_to_validate_from_mail_list(_mail_list, logger)
+
+
+def try_to_check_all_mails():
+    logger = logging.getLogger()
+    _mail_list_before_filter = get_mail_list_to_check()
+    _mails = filter_mail_with_links(_mail_list_before_filter)
+    for _mail in _mails:
+        if "outlook.com" in _mail or "hotmail.com" in _mail:
+            check_appointment_link_mail(_mail)
+            time.sleep(2)
+    _mail_list = [MailAddress("saigecong1990@pissmail.com", "cvExXKOP8oY1D@")]
+    find_links_to_validate_from_mail_list(_mail_list, logger)
+
+
+if __name__ == '__main__':
+    # check_all_need_to_check_emails()
+    try_to_check_all_mails()
+    # check_confirmed_mails()
+    # check_appointment_link_mail("hcunlvi533@outlook.com")

mail/mail_constants.py

@@ -86,47 +86,65 @@ def create_imap(login: str):
     if DOMAIN_163 in login:
         imap = IMAPClient(IMAP_SERVER_163, use_uid=True)
     elif DOMAIN_YAHOO in login:
-        imap = imaplib.IMAP4_SSL(YAHOO_IMAP_SERVER)
+        # imap = imaplib.IMAP4_SSL(YAHOO_IMAP_SERVER)
+        imap = IMAPClient(YAHOO_IMAP_SERVER, use_uid=True)
     elif DOMAIN_FIREMAIL_DE in login:
         # imap = imaplib.IMAP4_SSL(SERVER_FIREMAIL_DE)
         imap = IMAPClient(SERVER_FIREMAIL_DE, use_uid=True)
     elif DOMAIN_GMX in login or DOMAIN_GMX_FR in login or DOMAIN_GMX_US in login or DOMAIN_GMX_CH in login or DOMAIN_GMX_PT in login or DOMAIN_GMX_SG in login:
-        imap = imaplib.IMAP4_SSL(SERVER_GMX)
+        # imap = imaplib.IMAP4_SSL(SERVER_GMX)
+        imap = IMAPClient(SERVER_GMX, use_uid=True)
     elif DOMAIN_SINA in login:
         # imap = imaplib.IMAP4_SSL(IMAP_SERVER_SINA)
         imap = IMAPClient(IMAP_SERVER_SINA, use_uid=True)
     elif DOMAIN_HOTMAIL in login or DOMAIN_OUTLOOK_COM in login:
-        imap = imaplib.IMAP4_SSL(HOTMAIL_IMAP_SERVER)
+        # imap = imaplib.IMAP4_SSL(HOTMAIL_IMAP_SERVER)
+        imap = IMAPClient(HOTMAIL_IMAP_SERVER, use_uid=True)
     elif DOMAIN_RAMBLER_RU in login:
-        imap = imaplib.IMAP4_SSL(RAMBLER_IMAP_SERVER)
+        # imap = imaplib.IMAP4_SSL(RAMBLER_IMAP_SERVER)
+        imap = IMAPClient(RAMBLER_IMAP_SERVER, use_uid=True)
     elif DOMAN_BTVM_NE_JP in login:
-        imap = imaplib.IMAP4_SSL(BTVM_NE_JP)
+        # imap = imaplib.IMAP4_SSL(BTVM_NE_JP)
+        imap = IMAPClient(BTVM_NE_JP, use_uid=True)
     elif DOMAN_GMAIL in login:
-        imap = imaplib.IMAP4_SSL(SEREVER_GMAIL, port=993)
+        # imap = imaplib.IMAP4_SSL(SEREVER_GMAIL, port=993)
+        imap = IMAPClient(SEREVER_GMAIL, use_uid=True)
     elif DOMAIN_ONET in login:
         imap = IMAPClient(SERVER_IMAGE_ONET, use_uid=True)
     elif DOMAIN_TIM_IT in login:
-        imap = imaplib.IMAP4(TIME_IT_SERVER)
+        # imap = imaplib.IMAP4(TIME_IT_SERVER)
+        imap = IMAPClient(TIME_IT_SERVER, use_uid=True)
     elif DOMAIN_ALICE_IT in login:
-        imap = imaplib.IMAP4(ALICE_IMAP_SERVER, port=143)
+        # imap = imaplib.IMAP4(ALICE_IMAP_SERVER, port=143)
+        imap = IMAPClient(ALICE_IMAP_SERVER, use_uid=True)
     elif DOMAIN_MARS_DTI_NE_JP in login:
-        imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143)
+        # imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143)
+        imap = IMAPClient(MARS_DTI_NE_JP_SERVER, use_uid=True)
     elif DOMAN_AURORA_DTI_NE_JP in login:
-        imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143)
+        # imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143)
+        imap = IMAPClient(MARS_DTI_NE_JP_SERVER, use_uid=True)
     elif DOMAIN_NAVER in login:
-        imap = imaplib.IMAP4_SSL(NAVER_SERVER, port=993)
+        # imap = imaplib.IMAP4_SSL(NAVER_SERVER, port=993)
+        imap = IMAPClient(NAVER_SERVER, use_uid=True)
     elif DOMAIN_GMX_DE in login or DOMAIN_GMX_NET in login:
-        imap = imaplib.IMAP4_SSL(SERVER_GMX_NET, port=993)
+        # imap = imaplib.IMAP4_SSL(SERVER_GMX_NET, port=993)
+        imap = IMAPClient(SERVER_GMX_NET, use_uid=True)
     elif DOMAIN_GMX_AT in login:
-        imap = imaplib.IMAP4_SSL(SERVER_GMX_AT, port=993)
+        # imap = imaplib.IMAP4_SSL(SERVER_GMX_AT, port=993)
+        imap = IMAPClient(SERVER_GMX_AT, use_uid=True)
     elif DOMAIN_GAZETA_PL in login:
-        imap = imaplib.IMAP4_SSL(IMAP_SERVER_DOMAIN_GAZETA_PL, port=993)
+        # imap = imaplib.IMAP4_SSL(IMAP_SERVER_DOMAIN_GAZETA_PL, port=993)
+        imap = IMAPClient(IMAP_SERVER_DOMAIN_GAZETA_PL, use_uid=True)
     elif DOMAIN_INBOX_LV in login:
-        imap = imaplib.IMAP4_SSL(INBOX_LV, port=993)
+        # imap = imaplib.IMAP4_SSL(INBOX_LV, port=993)
+        imap = IMAPClient(INBOX_LV, use_uid=True)
     elif DOMAIN_WEB_DE in login:
-        imap = imaplib.IMAP4_SSL(SERVER_WEB_DE, port=993)
+        # imap = imaplib.IMAP4_SSL(SERVER_WEB_DE, port=993)
+        imap = IMAPClient(SERVER_WEB_DE, use_uid=True)
     elif DOMAIN_PISS_MAIL in login or DOMAIN_CHILD_PIZZA in login or DOMAIN_DMC_CHAT in login or DOMAIN_GENOCIDE_FUN in login or DOMAIN_HATESJE_WS in login or DOMAIN_INCEL_EMAIL in login or DOMAIN_SHITPOSTING_EXPERT in login:
-        imap = imaplib.IMAP4_SSL(SERVER_PISS_MAIL, port=993)
+        # imap = imaplib.IMAP4_SSL(SERVER_PISS_MAIL, port=993)
+        imap = IMAPClient(SERVER_PISS_MAIL, use_uid=True)
     else:
-        imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER)
+        # imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER)
+        imap = IMAPClient(AOL_IMAP_SERVER, use_uid=True)
     return imap

mail/mail_reader_all_contacts.py

@@ -6,10 +6,13 @@ from builtins import list
 from concurrent.futures import ThreadPoolExecutor
 from email.header import decode_header
 from email.message import Message
+from typing import Union
 
 from imapclient import IMAPClient
 from db.mongo_manager import MONGO_STORE_MANAGER
+from excel_reader import read_contacts
 from mail.mail_constants import DOMAIN_HOTMAIL, create_imap
+from models.ReserveResultPojo import ReserveResultPojo
 from models.mail_pojo import MailPojo, MailAddress
 
 VALIDATION_URL_SUBJECT_fr = 'Validation de votre demande de rendez-vous'
@@ -48,6 +51,11 @@ def find_from_mail(param):
     return from_address.strip(" ").strip(">").strip("<")
 
 
+def extract_email_from_from_address(content: str):
+    _match = re.search(r'[\w.+-]+@[\w-]+\.[\w.-]+', content)
+    return _match.group(0)
+
+
 class MailReader():
     def __init__(self, login, password):
         self.login = login
@@ -95,7 +103,10 @@ class MailReader():
             folder_list = self.show_folders(imap)
             for folder in folder_list:
                 print("folder is " + folder)
-                mail_list.extend(self._get_messages_from_folder_for_imapclient(imap, folder=folder))
+                if folder == "Sent" or folder == "Drafts":
+                    pass
+                else:
+                    mail_list.extend(self._get_messages_from_folder_for_imapclient(imap, folder=folder))
         if not isImapClient:
             imap.close()
         imap.logout()
@@ -161,32 +172,38 @@ class MailReader():
         search_terms = 'SINCE "{}"'.format(
             datetime.datetime.today().strftime(
                 date_format))
-        print("search terms is " + search_terms)
+        print("{}: search terms is {}".format(self.login,search_terms))
         imap.select_folder(folder)
         messages = imap.search(['SINCE', datetime.datetime.today()])
-        print("%d messages from our best friend" % len(messages))
+        print("{}: {} messages from our best friend".format(self.login, len(messages)))
+        if len(messages) ==0:
+            return mail_messages
         for uid, message_data in imap.fetch(messages, 'RFC822').items():
             try:
                 email_message = email.message_from_bytes(message_data[b'RFC822'])
                 from_address = email_message.get('FROM')
                 subject = email_message.get('subject')
-                # print("{}, {},{}".format(from_address, subject, email_message))
                 body = ""
-                if "no-reply@hermes.com" in from_address or "appointment2022@aol.com":
+                hermes_mail_adress = "no-reply@hermes.com"
+                if hermes_mail_adress in from_address or "outlook.com" in from_address or "hotmail" in from_address:
                     for part in email_message.walk():
                         print(part.get_content_type())
                         if part.get_content_type() == "text/html":
                             body = body + part.get_payload(decode=True).decode("utf-8")
                         elif part.get_content_type() == "text/plain":
                             body = body + part.get_payload()
-                    if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject or "Validation=20de=20votre=20demande=20de=20rendez-vous" in subject:
+                    if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject or "Votre=20demande=20de=20rendez-vous" in subject or "Votre demande de rendez-vous" in body:
                         mail = MailPojo(subject=subject, body=body, from_address=from_address)
                         mail.isImapClient = True
                         print("email is {}".format(self.login))
                         print("body is {}".format(body))
                         print("subject is {}".format(subject))
                         if len(mail.to_address) == 0:
-                            mail.to_address = self.login
+                            if "outlook.com" in from_address or "hotmail.com" in from_address:
+                                # it is a transferred email
+                                mail.to_address = extract_email_from_from_address(from_address)
+                            else:
+                                mail.to_address = self.login
                         mail_messages.append(mail)
             except Exception as error:
                 print(error)
@@ -194,21 +211,31 @@ class MailReader():
         return mail_messages
 
 
-def need_to_valid_url(url: str, successful_items) -> bool:
-    # return True
-    # if len(successful_items) == 0:
-    #     return False
+#
+# Find the ReserveResultPojo object from persisted items of DB
+#
+def find_item_by_url(url: str, successful_items) -> Union[None, ReserveResultPojo]:
+    print("url is :" + url)
+    parts = url.split('/')
+    _id = parts[5]
+    if len(_id) == 6:
+        for item in successful_items:
+            if item.id == _id:
+                return item
+    return None
+
+
+def need_to_valid_url(url: str, item: Union[ReserveResultPojo, None]) -> bool:
     print("url is :" + url)
     parts = url.split('/')
     id = parts[5]
     if len(id) == 6:
-        for item in successful_items:
-            if item.id == id:
-                if item.url_validated is not None:
-                    return not item.url_validated
-                else:
-                    # if url_validated is None
-                    return True
+        if item:
+            if item.url_validated is not None:
+                return not item.url_validated
+            else:
+                # if url_validated is None
+                return True
         return True
     else:
         print("id not valid:{}".format(id))
@@ -217,6 +244,8 @@ def need_to_valid_url(url: str, successful_items) -> bool:
 
 def need_to_check_email(mail: str, successful_items) -> bool:
     print("successful_items size is " + str(len(successful_items)))
+    if mail =="saigecong1990@pissmail.com":
+        return True
     filtered_items = list(filter(lambda item: item.email == mail, successful_items))
     # has validated value
     if len(filtered_items) > 0:
@@ -232,37 +261,62 @@ def need_to_check_email(mail: str, successful_items) -> bool:
 
 
 def find_links_to_validate_from_mail_list(mail_list: list, logger):
+    if not mail_list:
+        return
     # check time before start checking emails
-    if len(mail_list) > 0:
-        contact_to_book_list = MONGO_STORE_MANAGER.get_all_contact_to_book_list()
-        successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
-        mails_messages = []
-        with ThreadPoolExecutor(max_workers=len(mail_list)) as executor:
-            for mail in mail_list:
-                # check whether we need to read mail
-                if need_to_check_email(mail.mail, successful_items):
-                    mail_reader = MailReader(mail.mail, mail.password)
-                    executor.submit(mail_reader.read_emails, mails_messages)
-        # for mail in mail_list:
-        #     # check whether we need to read mail
-        #     if need_to_check_email(mail.mail, successful_items):
-        #         mail_reader = MailReader(mail.mail, mail.password)
-        #         mail_reader.read_emails(mails_messages)
-        _refreshed_successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
-        for mail in mails_messages:
-            match = re.search(VALIDATION_URL_REGEX, mail.body)
-            if match:
-                url = match.group(0)
-                if need_to_valid_url(url, _refreshed_successful_items):
-                    logger.info("need to validate url: " + url)
-                    MONGO_STORE_MANAGER.save_links_to_validate(url, mail.to_address,
-                                                               _all_contact_list=contact_to_book_list)
-                else:
-                    logger.info("do not need to click url --> {}".format(mail.mail_address))
+    contact_to_book_list = MONGO_STORE_MANAGER.get_all_contact_to_book_list()
+    successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+    mails_messages = []
+    with ThreadPoolExecutor(max_workers=200) as executor:
+        for mail in mail_list:
+            # check whether we need to read mail
+            if need_to_check_email(mail.mail, successful_items):
+                mail_reader = MailReader(mail.mail, mail.password)
+                executor.submit(mail_reader.read_emails, mails_messages)
+    _refreshed_successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+    for mail in mails_messages:
+        match = re.search(VALIDATION_URL_REGEX, mail.body)
+        if match:
+            url = match.group(0)
+            _item = find_item_by_url(url, _refreshed_successful_items)
+            if need_to_valid_url(url, _item):
+                logger.info("need to validate url: " + url)
+                _model = ""
+                if _item:
+                    _model = _item.model
+                MONGO_STORE_MANAGER.save_links_to_validate(url, mail.to_address, model=_model,
+                                                           _all_contact_list=contact_to_book_list)
+            else:
+                logger.info("do not need to click url --> {}".format(mail.mail_address))
 
 
 if __name__ == '__main__':
-    mail_address1 = MailAddress(mail="tzfhygwwjiwnf@hotmail.com", password="rqO4Pjm7NQ")
-    mail_list = [mail_address1]
+    # mail_address1 = MailAddress(mail="tinagonzales685585@aol.com", password="yhihvdkrbxnksema")
+    # mail_list = [mail_address1]
+    contact_to_book_list = read_contacts(
+        # file_name="/Users/rdv/Desktop/contact_list_not_used_contacts.xlsx")
+        # file_name="/Users/lpan/Desktop/contact_list_not_used_contacts.xlsx")
+        # file_name="/Users/rdv/Desktop/real_name_contacts_100_hotmail.xlsx")
+    # file_name="~/Desktop/contact_list_2025-07-11.xlsx")
+    file_name="~/Desktop/contact_list_all_studio.xlsx")
+    # file_name="/Users/rdv/Desktop/contact_list_all_studo_gmx_us.xlsx")
+    # file_name="/Users/rdv/Desktop/contact_list_2025-05-24.xlsx")
+    all_mail_list = MONGO_STORE_MANAGER.get_destination_emails()
+    mail_list_to_check = []
+    for contact in contact_to_book_list:
+        for mail in all_mail_list:
+            if contact.mail == mail.mail:
+                mail_list_to_check.append(mail)
     logger = logging.getLogger()
-    find_links_to_validate_from_mail_list(mail_list, logger)
+    _all_links = MONGO_STORE_MANAGER.get_links_to_validate()
+    filter_mail = []
+    for mail_pojo in mail_list_to_check:
+        _to_add = True
+        for _link in _all_links:
+            if _link.email == mail_pojo.mail:
+                _to_add = False
+        if _to_add:
+            filter_mail.append(mail_pojo)
+    filter_mail.append(MailAddress("saigecong1990@pissmail.com", "cvExXKOP8oY1D@"))
+    # filter_mail = [MailAddress("saigecong1990@pissmail.com", "cvExXKOP8oY1D@")]
+    find_links_to_validate_from_mail_list(filter_mail, logger)

migrate_to_data_cache_2.py

@@ -3,6 +3,6 @@ from queue_message.CookiesPublisher import MORNING_DATA_CACHE_2, MORNING_DATA_CA
 from workers.MessagerTransporter import migrate_message_to_queue, empty_message_just_to
 
 if __name__ == '__main__':
-    migrate_message_to_queue(from_queue=MORNING_DATA_CACHE_BAK, to_queue=MORNING_DATA_CACHE)
+    migrate_message_to_queue(from_queue=MORNING_DATA_CACHE, to_queue=MORNING_DATA_CACHE_BAK)
     # empty_message_just_to(2000, queue_name=MORNING_DATA_CACHE_BAK)
     # migrate_message_to_queue(from_queue=MORNING_DATA_CACHE_BAK,to_queue=REQUEST_DATA_QUEUE_DE)

models/LinkPojo.py

@@ -11,7 +11,9 @@ class LinkPojo():
     def from_firestore_dict(source):
         updated_at = source['updated_at']
         email = source['email']
-        model = source['model']
+        model = ""
+        if 'model' in source:
+            model = source['model']
         serial = source['serial']
         url = source['url']
         ip_country = "FR"

models/ReserveResultPojo.py

@@ -29,6 +29,7 @@ class ReserveResultPojo:
     slot_position = None
     sim_position = None
     ccid: str = ""
+    model: str = ""
     source_from: str = socket.gethostname()
     store_type = 0
     url_validated = None
@@ -106,6 +107,9 @@ class ReserveResultPojo:
         if 'created_at' in source:
             created_at = source['created_at']
             result.created_at = created_at
+        if 'model' in source:
+            model = source['model']
+            result.model = model
         if 'validated_at' in source:
             validated_at = source['validated_at']
             result.validated_at = validated_at
@@ -122,8 +126,7 @@ class ReserveResultPojo:
             u'email': self.email,
             u'passport': self.passport,
             u'url': self.url,
-            # u'sim_position': self.sim_position,
-            # u'slot_position': self.slot_position,
+            u'model': self.model,
             u'source_from': self.source_from,
             u'hostName': self.source_from,
             u'created_at': self.created_at,

proxy_manager/proxy_manager.py

@@ -1,5 +1,8 @@
+import logging
 import random
 
+import requests
+
 FR_ASOCKS_MOBILE_PROXY = {
     'http': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046',
     'https': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046',
@@ -16,8 +19,8 @@ FR_PROXY_RES_PARIS_OXY = {
 }
 
 FR_PROXY_MOB_OXY = {
-    'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@pr.oxylabs.io:7777',
-    'https': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@pr.oxylabs.io:7777'
+    'http': 'http://customer-rendezvousmob-cc-FR:Rdv+20222021@pr.oxylabs.io:7777',
+    'https': 'http://customer-rendezvousmob-cc-FR:Rdv+20222021@pr.oxylabs.io:7777'
 }
 
 FR_PROXY_ASOCK_RES_2 = {
@@ -25,8 +28,8 @@ FR_PROXY_ASOCK_RES_2 = {
     'https': 'http://zd6fbrujot-res-country-FR-hold-query:8k8avNlnLHQaMsWg@217.23.6.161:9999'
 }
 FR_PROXY_MOB_OXY_STICKY = {
-    'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}',
-    'https': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}'
+    'http': 'http://customer-rendezvousmob-cc-fr-sessid-0{}-sesstime-2:Rdv+202220212023@pr.oxylabs.io:7777',
+    'https': 'http://customer-rendezvousmob-cc-fr-sessid-0{}-sesstime-2:Rdv+202220212023@pr.oxylabs.io:7777'
 }
 
 FR_PROXY_DATA_IMPULSE_STICKY = {
@@ -52,17 +55,16 @@ FR_MOBILE_ANY_IP_ROTATING = {
     'https': 'http://user_6a7f21,type_residential,country_FR:d5c051@portal.anyip.io:1080',
 }
 # 八分之一用data_impulse
-MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY,
-                     FR_PROXY_MOB_OXY_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY,
-                     FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY,
-                     FR_PROXY_RES_OXY_STICKY,
-                     FR_PROXY_DATA_IMPULSE_STICKY]
+# MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY,
+#                      FR_PROXY_MOB_OXY_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY,
+#                      FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY,
+#                      FR_PROXY_RES_OXY_STICKY,
+#                      FR_PROXY_DATA_IMPULSE_STICKY]
 
 
-# MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY]
-
-
-# MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY]
+# MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY, FR_PROXY_MOB_OXY_STICKY]
+# MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY]
+MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY]
 
 
 class ProxyManager:
@@ -72,7 +74,8 @@ class ProxyManager:
 
     def get_link_validate_proxy(self, links_to_validate: list) -> list:
         # return [FR_PROXY_RES_PARIS_OXY]
-        return [FR_MOBILE_ANY_IP_ROTATING]
+        # return [FR_MOBILE_ANY_IP_ROTATING]
+        return [FR_PROXY_RES_OXY]
         # if len(links_to_validate) > 15:
         #     return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
         #     # return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY]
@@ -88,7 +91,7 @@ class ProxyManager:
         _chosen_proxy = random.choice(MOBILE_PROXY_LIST)
         if "oxylabs" in _chosen_proxy["http"]:
             self.logger.info("use oxylabs proxy")
-            _port = random.randint(40001, 49999)
+            _port = random.randint(900000000, 995869818)
         elif "anyip" in _chosen_proxy["http"]:
             self.logger.info("use anyip proxy")
             _port = random.randint(40001, 49999)
@@ -100,3 +103,14 @@ class ProxyManager:
         _proxy_to_use["http"] = _chosen_proxy["http"].format(_port)
         _proxy_to_use["https"] = _chosen_proxy["https"].format(_port)
         return _proxy_to_use
+
+
+if __name__ == '__main__':
+    _logger = logging.getLogger()
+    _proxy = ProxyManager(logger=_logger)
+    _proxise = _proxy.get_proxy_for_appointment_request()
+    print(_proxise)
+    response = requests.get(
+        "https://ip.oxylabs.io/location",
+        proxies=_proxise)
+    print(response.content)

queue_message/appointmentrequestsender.py

@@ -3,10 +3,13 @@ import json
 import random
 import threading
 import time
+from http.cookies import SimpleCookie
+from typing import Optional
 
 import pika
 
 from db.mongo_manager import MONGO_STORE_MANAGER
+from mail.lan_mail_helper import check_mail, check_all_need_to_check_emails
 from mail.mail_reader_all_contacts import find_links_to_validate_from_mail_list
 from models.ReserveResultPojo import ReserveResultPojo
 from models.contact_pojo import ContactPojo
@@ -62,6 +65,24 @@ def is_open():
     return is_time_between(datetime.time(10, 30), datetime.time(19, 00))
 
 
+def check_ms_mails(_mail_list_filtered):
+    print("check_ms_mails() called.")
+    check_all_need_to_check_emails()
+    # for _mail in _mail_list_filtered:
+    #     if "outlook.com" in _mail.mail or "hotmail.com" in _mail.mail:
+    #         check_mail(_mail.mail)
+
+
+def get_xsfr_token_from_cookies(cookies_str: str) -> Optional[str]:
+    _simple_cookies = SimpleCookie()
+    _simple_cookies.load(cookies_str)
+    if "x-xsrf-token" not in _simple_cookies:
+        return None
+    else:
+        _xsfr_token = _simple_cookies["x-xsrf-token"].value
+        return _xsfr_token
+
+
 class AppointmentRequestSender(threading.Thread):
     def __init__(self, sub_contact_list: list, logger, cookiesPublisher: CookiesPublisher,
                  bakeUpCookiesPublisher: CookiesPublisher,
@@ -213,9 +234,9 @@ class AppointmentRequestSender(threading.Thread):
                     # 如果在发送请求时出现csrf被拦截的情况，那么就需要重新发布cookie以目前的队列中，因为这个cookie可能重新利用
                     self.logger.info("csrf blocked, will republish cookie")
                     self.cookiesPublisher.publish_body(_received_object)
-                    self.logger.info("csrf blocked, will wait 60 seconds")
-                    time.sleep(60)
                     ch.basic_ack(delivery_tag=method.delivery_tag)
+                    self.logger.info("csrf blocked, will wait 30 seconds")
+                    time.sleep(30)
                 elif can_continue is not None and can_continue == RequestResult.BLOCKED:
                     self.logger.info("这个cookies可以给点链接用")
                     self.bakeUpCookiesPublisher.publish_body(_received_object)
@@ -239,6 +260,7 @@ class AppointmentRequestSender(threading.Thread):
         self.channel.start_consuming()
 
     def retrieve_invalidate_urls(self):
+        self.list_to_retrieve_mails = self.initial_contact_list
         # 如果没有已读邮件，而且需要读邮件的联系人表不为空，就读取未读邮件
         if not self.already_read_emails and len(self.list_to_retrieve_mails) > 0:
             self.logger.info("will retrieve validate urls")
@@ -251,6 +273,7 @@ class AppointmentRequestSender(threading.Thread):
                         self.logger.info("will get mail from " + mail.mail)
                         _mail_list_filtered.append(mail)
             self.logger.info("will call find_links_to_validate_from_mail_list, size = " + str(len(_mail_list_filtered)))
+            check_ms_mails(_mail_list_filtered)
             find_links_to_validate_from_mail_list(_mail_list_filtered, self.logger)
             self.already_read_emails = True
         else:

request_sender.py

@@ -71,10 +71,10 @@ def send_appointment_request(message_queue_name, _contact_list):
 
 def start_send_requests():
     print("start send requests")
-    contacts_file_path = '~/Desktop/11_05_to_test.xlsx'
-    _contact_list = read_contacts(contacts_file_path)[:-1]
+    contacts_file_path = '~/Desktop/06_05_to_test.xlsx'
+    _contact_list = read_contacts(contacts_file_path)[:1]
     _contact_list_to_book = filter_contacts(_contact_list)
-    _segment_number = 5
+    _segment_number = 1
     logger.info("{} contacts to book".format(len(_contact_list_to_book)))
     last_thread = None
     for i in range(0, _segment_number):

request_sender_test.py

@@ -10,9 +10,7 @@ from models.contact_pojo import ContactPojo
 from queue_message.CookiesPublisher import CookiesPublisher, SHARED_OBJECT, TEST_QUEUE, MORNING_DATA_CACHE, \
     MORNING_DATA_CACHE_2, MORNING_DATA_CACHE_BAK
 from queue_message.appointmentrequestsender import AppointmentRequestSender
-from utiles import is_time_between
 from utils.AppLogging import init_logger
-from workers.proxies_constants import MOBILE_PROXY_LIST_FR
 
 
 def is_already_sent(contact: ContactPojo) -> bool:
@@ -103,6 +101,8 @@ if __name__ == '__main__':
     #              '~/Desktop/contact_list_2024-05-21.xlsx',
     #              '~/Desktop/15_05_to_test.xlsx']
     # file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx']
-    file_list = ['~/Desktop/contact_list_2024-10-05_gmx_net_100.xlsx']
-    send_request_for_file_list(file_list=file_list, thread_number=1,
+    file_list = ['~/Desktop/contact_list_2025-07-11.xlsx']
+    # file_list = ['~/Desktop/contact_list_all_studio.xlsx']
+    # file_list = ['~/Desktop/real_name_contacts_100_27_06.xlsx']
+    send_request_for_file_list(file_list=file_list, thread_number=20,
                                data_queue_name=MORNING_DATA_CACHE, stop_at_hour=19, stop_at_mins=50)

requirements.txt

@@ -0,0 +1,2 @@
+curl_cffi==0.7.1
+openpyxl

scheduler_test.py

@@ -6,10 +6,8 @@ from request_sender_test import send_request_for_file_list
 
 
 def start_book_appointment():
-    file_list = ['~/Desktop/contact_list_2024-09-06.xlsx',
-                 '~/Desktop/contact_list_all.xlsx',
-                 '~/Desktop/contact_list_2024-05-27.xlsx']
-    send_request_for_file_list(file_list=file_list, thread_number=80,
+    file_list = ['~/Desktop/contact_list_2025-07-11.xlsx']
+    send_request_for_file_list(file_list=file_list, thread_number=60,
                                data_queue_name=MORNING_DATA_CACHE, stop_at_hour=11, stop_at_mins=20)
 
 

workers/captcha_result_getter.py

@@ -2,14 +2,13 @@ import json
 import random
 import re
 from typing import Union
-import requests
-
+# import requests
+from curl_cffi import requests
 from models.jsdata_le_pojo import JsDataLeTypePojo
 from models.jsdata_pojo import JsDataPojo
 from models.result_pojo import RequestResult
 from utils.get_only_datadome_cookies import get_datadome_cookies, get_app_cookies, get_lang_cookies, \
     retain_only_dataome_cookies
-from workers.proxies_constants import PROXY_LIST_FR
 
 API_KEY = "d66aaf490d8aa424a5175e1fbd1aadea"
 

workers/link_validator_with_provided_list.py

@@ -19,7 +19,7 @@ from utils.AppLogging import init_logger
 from utils.user_agent_helper import generate_headers_from_request_message
 
 DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ces coordonnées"
-INVALID = "Depuis plus de 130 ans,"
+INVALID = "nos conseillers vous accueillent sur rendez-vous dans nos magasins"
 
 init_logger()
 logger = logging.getLogger()
@@ -28,7 +28,7 @@ logger = logging.getLogger()
 def filter_link_pojo_list_with_serial(_received_dict, link_to_validate_list):
     _serial = _received_dict["serial"]
     _model = _received_dict["model"]
-    _bit_browser_list = list(filter(lambda link_pojo: link_pojo.serial == "bitbrowser", link_to_validate_list))
+    _bit_browser_list = list(filter(lambda link_pojo: link_pojo.serial == "requests", link_to_validate_list))
     if len(_bit_browser_list) > 0:
         return _bit_browser_list
     else:
@@ -149,44 +149,51 @@ class LinkValidatorWithProvidedList(threading.Thread):
         _message_in_queue_count = self.cookiesPublisher.message_count()
         print("message count in queue is {}".format(_message_in_queue_count))
         _received_object = body.decode("UTF-8")
-        _received_dict = json.loads(_received_object)
-        _received_cookies = _received_dict["cookiesStr"]
-        self.cookie_str = _received_cookies
-        _links_to_validate = filter_link_pojo_list_with_serial(_received_dict, self.link_to_validate_list)
-        random.shuffle(_links_to_validate)
-        if len(_links_to_validate) > 0 and _message_in_queue_count >= self.limit:
-            print("{}:links number is {}".format(threading.currentThread().name, len(_links_to_validate)))
-            can_continue = None
-            for link_to_validate in _links_to_validate:
-                print(link_to_validate)
-                self.proxy_to_use = random.choice(
-                    self.proxy_manager.get_link_validate_proxy(_links_to_validate))
-                print("proxy to use is {}".format(self.proxy_to_use))
-                can_continue = self.send_request(link_to_validate, _received_dict)
-                # remove the tested link from link list
-                self.link_to_validate_list.remove(link_to_validate)
-                if can_continue == RequestResult.BLOCKED:
-                    print("cannot continue, blocked, then skip")
-                    break
-                else:
-                    if can_continue == RequestResult.BAD_GATEWAY:
-                        time.sleep(30)
+        try:
+            _received_dict = json.loads(_received_object)
+            _received_cookies = _received_dict["cookiesStr"]
+            self.cookie_str = _received_cookies
+            _links_to_validate = filter_link_pojo_list_with_serial(_received_dict, self.link_to_validate_list)
+            random.shuffle(_links_to_validate)
+            if len(_links_to_validate) > 0 and _message_in_queue_count >= self.limit:
+                print("{}:links number is {}".format(threading.currentThread().name, len(_links_to_validate)))
+                can_continue = None
+                for link_to_validate in _links_to_validate:
+                    print(link_to_validate)
+                    self.proxy_to_use = random.choice(
+                        self.proxy_manager.get_link_validate_proxy(_links_to_validate))
+                    print("proxy to use is {}".format(self.proxy_to_use))
+                    can_continue = self.send_request(link_to_validate, _received_dict)
+                    # remove the tested link from link list
+                    self.link_to_validate_list.remove(link_to_validate)
+                    if can_continue == RequestResult.BLOCKED:
+                        print("cannot continue, blocked, then skip")
                         break
-                    time.sleep(random.randint(2, 5))
-                    print("can continue, continue")
-            if can_continue == RequestResult.BAD_GATEWAY or can_continue == RequestResult.PROXY_ERROR or can_continue == RequestResult.SUCCESS:
-                print("will requeue the message")
-                ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
+                    else:
+                        if can_continue == RequestResult.BAD_GATEWAY:
+                            time.sleep(30)
+                            break
+                        time.sleep(random.randint(2, 5))
+                        print("can continue, continue")
+                if can_continue == RequestResult.BAD_GATEWAY or can_continue == RequestResult.PROXY_ERROR or can_continue == RequestResult.SUCCESS:
+                    print("will requeue the message")
+                    ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
+                else:
+                    print("will ack")
+                    print("will wait for 40s")
+                    time.sleep(20)
+                    ch.basic_ack(delivery_tag=method.delivery_tag)
             else:
-                print("will ack")
+                print("empty list, will republish message")
+                time.sleep(0)
+                print("body in json:{}".format(json.dumps(_received_dict)))
+                self.cookiesPublisher.publish_body(json.dumps(_received_dict))
                 ch.basic_ack(delivery_tag=method.delivery_tag)
-        else:
-            print("empty list, will republish message")
-            time.sleep(0)
-            print("body in json:{}".format(json.dumps(_received_dict)))
-            self.cookiesPublisher.publish_body(json.dumps(_received_dict))
+                self.channel.stop_consuming()
+        except Exception as error:
+            print(error)
+            print("not json format will ack")
             ch.basic_ack(delivery_tag=method.delivery_tag)
-            self.channel.stop_consuming()
 
     def filter_with_ip_country(self):
         _link_list_to_click = []
@@ -210,15 +217,21 @@ def validate_links(cookiesPublisher, queue_name: str, link_list: list, _contact_
 def validate_all_links(_contact_serial_list):
     print("will get all links")
     all_link_list = MONGO_STORE_MANAGER.get_links_to_validate()
+    link_to_validated = []
+    for _link in all_link_list:
+        # print("serial is "+_link.serial)
+        print("email is " + _link.email)
+        if _link.serial == "requests":
+            link_to_validated.append(_link)
     # get the first 50 links
-    if len(all_link_list) == 0:
+    if len(link_to_validated) == 0:
         return
-    # divided = 1
+    divided = 1
     # default_segment_number = 20
-    # _first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))]
+    _first_25_percent_links = link_to_validated[0:(int(len(all_link_list) / divided))]
     _first_25_percent_links = all_link_list
-    _queue_name = MORNING_DATA_CACHE
-    # _queue_name = MORNING_DATA_CACHE_BAK
+    # _queue_name = MORNING_DATA_CACHE
+    _queue_name = MORNING_DATA_CACHE_BAK
     # if len(all_link_list) > divided * default_segment_number:
     #     _segment_number = default_segment_number
     # else:
@@ -228,7 +241,10 @@ def validate_all_links(_contact_serial_list):
     #     else:
     #         _segment_number = 1
     _thread_list = []
-    _segment_number = 1
+    if len(_first_25_percent_links) >= 10:
+        _segment_number = 10
+    else:
+        _segment_number = len(_first_25_percent_links)
     for i in range(0, _segment_number):
         logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_25_percent_links)))
         logger.info("segment is {}".format(i))

workers/sender.py

@@ -1,7 +1,5 @@
 import datetime
 import json
-import logging
-import random
 import re
 import time
 from http.cookies import SimpleCookie
@@ -40,7 +38,8 @@ class Sender:
         self.proxy_to_use = proxy_to_use
         self.cookie.load(self.cookie_str)
 
-    def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str, store_type: str):
+    def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str, store_type: str,
+                                 model: str = ""):
         # create the message
         if url == "https://rendezvousparis.hermes.com/client/welcome":
             return
@@ -61,11 +60,12 @@ class Sender:
             result.proxy = "data_impulse"
         result.id = id
         result.store_type = store_type
+        result.model = model
         result.created_at = time.strftime("%H:%M:%S", time.localtime())
         collection_name = str(datetime.date.today())
         MONGO_STORE_MANAGER.insert_reserve_result(collection_name=collection_name, reserve=result)
 
-    def apply_redirect(self, response, old_headers, contact, js_data, selected_store):
+    def apply_redirect(self, response, old_headers, contact, js_data, selected_store, model=""):
         #     /client/register/5XD2E2
         _res_headers = response.headers
         _location = _res_headers['location']
@@ -89,7 +89,7 @@ class Sender:
         contact.current_ip = get_address_ip(proxy_to_use=self.proxy_to_use)
         _appointment_url = _redirect_url
         self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=_appointment_url,
-                                      store_type=selected_store)
+                                      store_type=selected_store, model=model)
         self.cookie.load(_cookies_to_set)
         new_cookies = {k: v.value for k, v in self.cookie.items()}
         new_coolies_str = ""
@@ -162,7 +162,7 @@ class Sender:
             headers['sec-ch-ua-full-version-list'] = _chrome_ua_version
         data = {'check': '', '_csrf': _csrf, 'prefer': _selected_store,
                 'surname': contact.last_name.capitalize(), 'name': contact.first_name.capitalize(),
-                'phone_country': "FR", 'phone_number': "+33 " + contact.phone, 'email': contact.mail,
+                'phone_country': "FR", 'phone_number': "0" + contact.phone, 'email': contact.mail,
                 'passport_id': contact.passport, 'processing': 'on', 'cgu': 'on'}
         self.logger.info(data)
         self.logger.info("try to request for mail:{}".format(contact.mail))
@@ -177,7 +177,7 @@ class Sender:
                 # add to mongodb
                 self.logger.info(response.text)
                 self.apply_redirect(response=response, old_headers=headers, contact=contact, js_data=js_data,
-                                    selected_store=_selected_store)
+                                    selected_store=_selected_store, model=model)
                 # self.logger.info("{}:{}".format(contact.mail, response.url))
                 # contact.ua = js_data.ua
                 # contact.current_ip = get_address_ip(proxy_to_use=proxy_to_use)