add yahoo support

mail_scheduler.py

@@ -5,22 +5,24 @@ from src.mail.mail_reader import read_mails
 
 
 def start_check_mail_job(sched):
-    sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='11',
+    sched.add_job(read_mails, 'cron', day_of_week='mon-sat', minute='*/10', jitter=240, timezone='Europe/Paris',
-                  minute='30',
+                  max_instances=1)
-                  misfire_grace_time=10,
+    # sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='11',
-                  second='0', timezone='Europe/Paris', max_instances=1)
+    #               minute='30',
-    sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='13',
+    #               misfire_grace_time=10,
-                  minute='30',
+    #               second='0', timezone='Europe/Paris', max_instances=1)
-                  misfire_grace_time=10,
+    # sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='13',
-                  second='0', timezone='Europe/Paris', max_instances=1)
+    #               minute='30',
-    sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='15',
+    #               misfire_grace_time=10,
-                  minute='30',
+    #               second='0', timezone='Europe/Paris', max_instances=1)
-                  misfire_grace_time=10,
+    # sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='15',
-                  second='0', timezone='Europe/Paris', max_instances=1)
+    #               minute='30',
-    sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='17',
+    #               misfire_grace_time=10,
-                  minute='30',
+    #               second='0', timezone='Europe/Paris', max_instances=1)
-                  misfire_grace_time=10,
+    # sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='17',
-                  second='0', timezone='Europe/Paris', max_instances=1)
+    #               minute='30',
+    #               misfire_grace_time=10,
+    #               second='0', timezone='Europe/Paris', max_instances=1)
 
 
 def config_and_start_jobs():

src/mail/mail_reader.py

@@ -15,6 +15,7 @@ from src.proxy.proxy_type import ProxyType
 from src.workers.link_validator import LinkValidator
 
 AOL_IMAP_SERVER = "imap.aol.com"
+YAHOO_IMAP_SERVER = "imap.mail.yahoo.com"
 IMAP_SERVER_163 = "imap.163.com"
 VALIDATION_URL_SUBJECT_fr = 'Validation de votre demande de rendez-vous'
 VALIDATION_URL_SUBJECT_EN = 'Please confirm your appointment'
@@ -22,7 +23,9 @@ VALIDATION_URL_REGEX = """https:\/\/rendezvousparis.hermes.com\/client\/register
 HERMES_EMAIL = "no-reply@hermes.com"
 
 DOMAIN_163 = "163.com"
+DOMAIN_YAHOO = "yahoo.com"
 date_format = "%d-%b-%Y"  # DD-Mon-YYYY e.g., 3-Mar-2014
+REDIRECTION_MAILS = "appointment2022@aol.com, chenpeijun@aol.com,hongjiang176@aol,ciyuexie@aol.com "
 
 
 class MailReader():
@@ -40,6 +43,8 @@ class MailReader():
         # create an IMAP4 class with SSL
         if DOMAIN_163 in self.login:
             imap = imaplib.IMAP4_SSL(IMAP_SERVER_163)
+        elif DOMAIN_YAHOO in self.login:
+            imap = imaplib.IMAP4_SSL(YAHOO_IMAP_SERVER)
         else:
             imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER)
         # authenticate
@@ -140,11 +145,20 @@ def need_to_valid_url(url: str, successful_items) -> bool:
 
 
 def need_to_check_email(mail: str, successful_items) -> bool:
+    # return True
     for item in successful_items:
-        if item.email == mail:
+        if mail in item.email:
             if item.url_validated is not None:
+                print("url_validated for {} is {}".format(item.url_validated, mail))
                 return not item.url_validated
-    return True
+            else:
+                # if url-validated is none, need to check email
+                return True
+    # if the email has not been booked, we needn't read mails.
+    if mail in REDIRECTION_MAILS:
+        return True
+    else:
+        return False
 
 
 def read_mails():
@@ -155,7 +169,7 @@ def read_mails():
     # # mail_address2 = MailAddress(mail="sdfgfhgf1986@aol.com", password="fjwcgvhxxlywqfwm")
     # # mail_address3 = MailAddress(mail="ciyuexie@aol.com", password="czezlmmyypokdfce")
     # # mail_address4 = MailAddress(mail="hongjiang176@aol.com", password="ftzpscgzvwneelmn")
-    # mail_address4 = MailAddress(mail="likegirlpassed@aol.com", password="xjgohjpclvgpnhbz")
+    # mail_address4 = MailAddress(mail="ryan_meacham2856@yahoo.com", password="ulgggkodxqbvrpgm")
     # mail_list = [mail_address3, mail_address2, mail_address1, mail_address4]
     # mail_list = [mail_address4]
     successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
@@ -163,11 +177,11 @@ def read_mails():
     with ThreadPoolExecutor(max_workers=20) as executor:
         for mail in mail_list:
             # check whether we need to read mail
-            if need_to_check_email(mail, successful_items):
+            if need_to_check_email(mail.mail, successful_items):
                 mail_reader = MailReader(mail.mail, mail.password)
                 executor.submit(mail_reader.read_emails, mails_messages)
 
-    with ThreadPoolExecutor(max_workers=20) as executor:
+    with ThreadPoolExecutor(max_workers=5) as executor:
         for mail in mails_messages:
             match = re.search(VALIDATION_URL_REGEX, mail.body)
             if match:
@@ -175,7 +189,7 @@ def read_mails():
                 if need_to_valid_url(url, successful_items):
                     url_validator = LinkValidator(url)
                     print("need to validate url: " + url)
-                    executor.submit(url_validator.start_page, params.get_proxy(ProxyType.RESIDENTIAL), True)
+                    executor.submit(url_validator.start_page, params.get_proxy(ProxyType.RESIDENTIAL), False)
                 else:
                     print("do not need to click url --> {}".format(mail.mail_address))