Merge branch 'feature/check_email_validation'

main.py

@@ -62,7 +62,7 @@ def get_proxy(proxy_type=ProxyType.RESIDENTIAL):
 
 if __name__ == '__main__':
     # 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机，传入1的时候是总店
-    start_book(2, 2600, store_choose_state=0, mode=ModeEnum.AUTOMATIC, headless=False, max_workers=8,
+    start_book(2, 2000, store_choose_state=0, mode=ModeEnum.AUTOMATIC, headless=False, max_workers=30,
                proxy_type=ProxyType.RESIDENTIAL)
     # start_book(828, 857, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=True)
     # start_book(1210, 1211, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=False, max_workers=3,

requirements.txt

@@ -2,7 +2,7 @@ dataclasses_json==0.5.7
 flask-cors==3.0.10
 firebase_admin==5.2.0
 pandas~=1.3.5
-playwright==1.27.0
+playwright==1.28.0
 dataclasses~=0.6
 SpeechRecognition==3.8.1
 pymongo==4.1.1

src/db/mongo_manager.py

@@ -19,6 +19,7 @@ ACCEPTED_APPOINTMENT_LIST = "ACCEPTED_APPOINTMENT_LIST"
 EMAIL_LIST = "EMAIL_LIST"
 DESTINATION_EMAIL_LIST = "DESTINATION_EMAIL_LIST"
 LINKS_TO_VALIDATE = "LINKS_TO_VALIDATE"
+INVALID_EMAIL_LIST = "INVALID_EMAIL_LIST"
 
 
 class MongoDbManager:
@@ -39,6 +40,15 @@ class MongoDbManager:
         except Exception as Error:
             self.logger.info(Error)
 
+    def insert_invalid_mail(self, email: MailAddress):
+        try:
+            collection_to_use = self.db[INVALID_EMAIL_LIST]
+            collection_to_use.replace_one(filter={'_id': email.mail, }, replacement=email.to_firestore_dict(),
+                                          upsert=True)
+        except Exception as Error:
+            self.logger.info(Error)
+            print(Error)
+
     def save_destinary_emails(self, email: MailAddress):
         try:
             collection_to_use = self.db[DESTINATION_EMAIL_LIST]
@@ -84,6 +94,17 @@ class MongoDbManager:
             self.logger.info(error)
         return email_list
 
+    def get_invalid_emails(self) -> list:
+        collection_name = INVALID_EMAIL_LIST
+        email_list = []
+        try:
+            collection_to_use = self.db[collection_name]
+            for document in collection_to_use.find():
+                email_list.append(MailAddress.from_firestore_dict(document))
+        except Exception as error:
+            self.logger.info(error)
+        return email_list
+
     def insert_captcha_error_contact(self, contact: ContactPojo):
         day = str(datetime.date.today())
         collection_name = CAPTCHA_ERROR_COLLECTION_PREFIX + day
@@ -183,6 +204,16 @@ class MongoDbManager:
         except Exception as error:
             self.logger.info(error)
 
+    def remove_email_from_destination_email_list(self, mail: MailAddress):
+        #     DESTINATION_EMAIL_LIST
+        collection = self.db[DESTINATION_EMAIL_LIST]
+        to_delete = {'_id': mail.mail}
+        try:
+            collection.delete_one(to_delete)
+        except Exception as error:
+            self.logger.info(error)
+            print(error)
+
     def remove_contact_from_black_list(self, contact: ContactPojo):
         collection_name = BLACK_LIST
         collection = self.db[collection_name]

src/mail/mail_address_validator.py

@@ -0,0 +1,69 @@
+import imaplib
+from concurrent.futures.thread import ThreadPoolExecutor
+
+from src.db.mongo_manager import MONGO_STORE_MANAGER
+from src.mail.mail_constants import DOMAIN_163, DOMAIN_YAHOO, DOMAIN_SINA, IMAP_SERVER_163, YAHOO_IMAP_SERVER, \
+    IMAP_SERVER_SINA, AOL_IMAP_SERVER
+from src.pojo.mail.mail_pojo import MailAddress
+
+
+class MailAddressValidator():
+    def __init__(self, login, password):
+        self.login = login
+        self.password = password
+
+    @staticmethod
+    def show_folders(imap):
+        for i in imap.list()[1]:
+            l = i.decode().split(' "/" ')
+            print(l[0] + " = " + l[1])
+
+    def create_imap(self):
+        # create an IMAP4 class with SSL
+        if DOMAIN_163 in self.login:
+            imap = imaplib.IMAP4_SSL(IMAP_SERVER_163)
+        elif DOMAIN_YAHOO in self.login:
+            imap = imaplib.IMAP4_SSL(YAHOO_IMAP_SERVER)
+        elif DOMAIN_SINA in self.login:
+            imap = imaplib.IMAP4_SSL(IMAP_SERVER_SINA)
+        else:
+            imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER)
+        return imap
+
+    def is_valid_email_address(self) -> bool:
+        # authenticate
+        imap = self.create_imap()
+        isValid = True
+        try:
+            type, dat = imap.login(self.login, self.password)
+            print("type is " + type)
+            imap.logout()
+        except Exception as error:
+            print(error)
+            isValid = False
+
+        return isValid
+
+    def check_and_save_to_db(self):
+        if not self.is_valid_email_address():
+            MONGO_STORE_MANAGER.insert_invalid_mail(MailAddress(self.login, self.password))
+            print("{} is not valid".format(self.login))
+
+
+def remove_invalid_email():
+    invalid = MONGO_STORE_MANAGER.get_invalid_emails()
+    for mail in invalid:
+        MONGO_STORE_MANAGER.remove_email_from_destination_email_list(mail)
+
+
+def find_and_update_invalid_emails():
+    mail_list = MONGO_STORE_MANAGER.get_destination_emails()
+    with ThreadPoolExecutor(max_workers=20) as executor:
+        for mail in mail_list:
+            valiator = MailAddressValidator(mail.mail, mail.password)
+            executor.submit(valiator.check_and_save_to_db)
+
+
+if __name__ == '__main__':
+    # remove_invalid_email()
+    find_and_update_invalid_emails()

src/mail/mail_confirmation.py

@@ -9,22 +9,17 @@ from builtins import list
 
 from src.db.mirgration.migration_tools import migre_accepted_appointment
 from src.db.mongo_manager import MONGO_STORE_MANAGER
+from src.mail.mail_constants import DOMAIN_163, DOMAIN_YAHOO, DOMAIN_SINA, IMAP_SERVER_163, YAHOO_IMAP_SERVER, \
+    IMAP_SERVER_SINA, AOL_IMAP_SERVER
 from src.notification.AcceptedResultPojo import get_accepted_result_from
 from src.notification.mailer import Mailer
 from src.pojo.ResultEnum import ResultEnum
 from src.pojo.mail.mail_pojo import MailPojo, MailAddress
 
-AOL_IMAP_SERVER = "imap.aol.com"
-IMAP_SERVER_163 = "imap.163.com"
-IMAP_SERVER_SINA = "imap.sina.com"
-YAHOO_IMAP_SERVER = "imap.mail.yahoo.com"
 CONFIRMATION_SUBJECT_FR = 'Votre rendez-vous est'
 CONFIRMATION_SUBJECT_EN = 'appointment is confirmed'
 HERMES_EMAIL = "no-reply@hermes.com"
-DOMAIN_YAHOO = "yahoo.com"
-DOMAIN_SINA = "sina.com"
 
-DOMAIN_163 = "163.com"
 date_format = "%d-%b-%Y"  # DD-Mon-YYYY e.g., 3-Mar-2014
 
 
@@ -154,6 +149,9 @@ def read_mails_and_find_confirmation_contacts():
                 elif "10:30" in message_body and (item.email == mail.mail_address or item.email in message_body):
                     item.message = message_body
                     accepted_appointment_list.append(item)
+                elif "11:30" in message_body and (item.email == mail.mail_address or item.email in message_body):
+                    item.message = message_body
+                    accepted_appointment_list.append(item)
             print(mail.mail_address)
             print(mail.subject)
             print(mail.body)

src/mail/mail_constants.py

@@ -0,0 +1,9 @@
+DOMAIN_YAHOO = "yahoo.com"
+DOMAIN_SINA = "sina.com"
+
+DOMAIN_163 = "163.com"
+
+AOL_IMAP_SERVER = "imap.aol.com"
+IMAP_SERVER_163 = "imap.163.com"
+IMAP_SERVER_SINA = "imap.sina.com"
+YAHOO_IMAP_SERVER = "imap.mail.yahoo.com"

src/mail/mail_reader.py

@@ -11,24 +11,19 @@ from builtins import list
 from src import params
 from src.db.mongo_manager import MONGO_STORE_MANAGER
 from src.logs.AppLogging import init_logger
+from src.mail.mail_constants import DOMAIN_163, DOMAIN_YAHOO, DOMAIN_SINA, IMAP_SERVER_163, YAHOO_IMAP_SERVER, \
+    IMAP_SERVER_SINA, AOL_IMAP_SERVER
 from src.pojo.mail.mail_pojo import MailPojo, MailAddress
 from src.proxy.proxy_type import ProxyType
 from src.utils.timeutiles import is_time_between
 from src.workers.link_validator import LinkValidator
 from datetime import time
 
-AOL_IMAP_SERVER = "imap.aol.com"
-YAHOO_IMAP_SERVER = "imap.mail.yahoo.com"
-IMAP_SERVER_163 = "imap.163.com"
-IMAP_SERVER_SINA = "imap.sina.com"
 VALIDATION_URL_SUBJECT_fr = 'Validation de votre demande de rendez-vous'
 VALIDATION_URL_SUBJECT_EN = 'Please confirm your appointment'
 VALIDATION_URL_REGEX = """https:\/\/rendezvousparis.hermes.com\/client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+"""
 HERMES_EMAIL = "no-reply@hermes.com"
 
-DOMAIN_163 = "163.com"
-DOMAIN_SINA = "sina.com"
-DOMAIN_YAHOO = "yahoo.com"
 date_format = "%d-%b-%Y"  # DD-Mon-YYYY e.g., 3-Mar-2014
 REDIRECTION_MAILS = "appointment2022@aol.com, chenpeijun@aol.com,hongjiang176@aol.com,ciyuexie@aol.com"
 

src/utils/black_list_checker.py

@@ -30,7 +30,11 @@ def is_in_accepted_list(contact: ContactPojo) -> bool:
     for accepted_contact in accepted_appointment_list:
         if contact.mail == accepted_contact.email:
             # check date
+            try:
                 return accepted_contact.accepted_at + TWO_WEEKS_IN_S > time.time()
+            except Exception as error:
+                print(error)
+                return False
 
     return False
 

src/utils/excel_reader.py

@@ -213,7 +213,7 @@ def save_mails_to_db():
 
 if __name__ == '__main__':
     excel_reader = ExcelHelper()
-    contacts = excel_reader.read_names("/Users/panlei/Documents/rdv/随机/24-11/100_yahoo.xlsx")
+    contacts = excel_reader.read_names("/Users/panlei/Documents/rdv/未注册/500.xlsx")
     print(contacts)
     write_new_contacts_to_excel(valid_contacts=contacts)
 

src/utils/generate_random_passport_id.py

@@ -56,7 +56,7 @@ def generate_titre_sejour_number(size=10) -> list:
 if __name__ == '__main__':
     # for i in range(1,200):
     #     print(get_random_id_number())
-    # for i in range(1, 501):
-    #     print(get_random_passport_id_number())
-    for id in generate_titre_sejour_number(100):
-        print(id)
+    for i in range(1, 501):
+        print(get_random_passport_id_number())
+    # for id in generate_titre_sejour_number(100):
+    #     print(id)

src/workers/commandor_page.py

@@ -226,14 +226,14 @@ class CommandorPage:
         try:
             if self.store_type == 0:
                 self.page.evaluate("""()=>{
-                //document.getElementById("phone_country").focus();
-                document.getElementById("phone_country").value = \"FR\" }""")
+                document.getElementById("phone_country").focus();
+                document.getElementById("phone_country").value = \"FR\"; }""")
             else:
                 store_to_choose = self.store_map[self.store_type]
                 self.page.evaluate("""(store_to_choose)=>{
                 document.getElementById("prefer").value = store_to_choose;
-                //document.getElementById("phone_country").focus();
-                document.getElementById("phone_country").value = \"FR\" }""", store_to_choose)
+                document.getElementById("phone_country").focus();
+                document.getElementById("phone_country").value = \"FR\"; }""", store_to_choose)
         except Exception as error:
             self.logger.error(error)