From ea33611e45bd975c149c1d869a6580c782e3898b Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 23 Sep 2022 16:35:09 +0200 Subject: [PATCH] add yahoo support --- mail_scheduler.py | 34 ++++++++++++++++++---------------- src/mail/mail_reader.py | 26 ++++++++++++++++++++------ 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/mail_scheduler.py b/mail_scheduler.py index 9c46ece..9aaf4d3 100644 --- a/mail_scheduler.py +++ b/mail_scheduler.py @@ -5,22 +5,24 @@ from src.mail.mail_reader import read_mails def start_check_mail_job(sched): - sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='11', - minute='30', - misfire_grace_time=10, - second='0', timezone='Europe/Paris', max_instances=1) - sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='13', - minute='30', - misfire_grace_time=10, - second='0', timezone='Europe/Paris', max_instances=1) - sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='15', - minute='30', - misfire_grace_time=10, - second='0', timezone='Europe/Paris', max_instances=1) - sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='17', - minute='30', - misfire_grace_time=10, - second='0', timezone='Europe/Paris', max_instances=1) + sched.add_job(read_mails, 'cron', day_of_week='mon-sat', minute='*/10', jitter=240, timezone='Europe/Paris', + max_instances=1) + # sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='11', + # minute='30', + # misfire_grace_time=10, + # second='0', timezone='Europe/Paris', max_instances=1) + # sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='13', + # minute='30', + # misfire_grace_time=10, + # second='0', timezone='Europe/Paris', max_instances=1) + # sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='15', + # minute='30', + # misfire_grace_time=10, + # second='0', timezone='Europe/Paris', max_instances=1) + # sched.add_job(read_mails, 'cron', day_of_week='mon-sat', hour='17', + # minute='30', + # misfire_grace_time=10, + # second='0', timezone='Europe/Paris', max_instances=1) def config_and_start_jobs(): diff --git a/src/mail/mail_reader.py b/src/mail/mail_reader.py index 9fb1e5c..b593cc4 100644 --- a/src/mail/mail_reader.py +++ b/src/mail/mail_reader.py @@ -15,6 +15,7 @@ from src.proxy.proxy_type import ProxyType from src.workers.link_validator import LinkValidator AOL_IMAP_SERVER = "imap.aol.com" +YAHOO_IMAP_SERVER = "imap.mail.yahoo.com" IMAP_SERVER_163 = "imap.163.com" VALIDATION_URL_SUBJECT_fr = 'Validation de votre demande de rendez-vous' VALIDATION_URL_SUBJECT_EN = 'Please confirm your appointment' @@ -22,7 +23,9 @@ VALIDATION_URL_REGEX = """https:\/\/rendezvousparis.hermes.com\/client\/register HERMES_EMAIL = "no-reply@hermes.com" DOMAIN_163 = "163.com" +DOMAIN_YAHOO = "yahoo.com" date_format = "%d-%b-%Y" # DD-Mon-YYYY e.g., 3-Mar-2014 +REDIRECTION_MAILS = "appointment2022@aol.com, chenpeijun@aol.com,hongjiang176@aol,ciyuexie@aol.com " class MailReader(): @@ -40,6 +43,8 @@ class MailReader(): # create an IMAP4 class with SSL if DOMAIN_163 in self.login: imap = imaplib.IMAP4_SSL(IMAP_SERVER_163) + elif DOMAIN_YAHOO in self.login: + imap = imaplib.IMAP4_SSL(YAHOO_IMAP_SERVER) else: imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER) # authenticate @@ -140,11 +145,20 @@ def need_to_valid_url(url: str, successful_items) -> bool: def need_to_check_email(mail: str, successful_items) -> bool: + # return True for item in successful_items: - if item.email == mail: + if mail in item.email: if item.url_validated is not None: + print("url_validated for {} is {}".format(item.url_validated, mail)) return not item.url_validated - return True + else: + # if url-validated is none, need to check email + return True + # if the email has not been booked, we needn't read mails. + if mail in REDIRECTION_MAILS: + return True + else: + return False def read_mails(): @@ -155,7 +169,7 @@ def read_mails(): # # mail_address2 = MailAddress(mail="sdfgfhgf1986@aol.com", password="fjwcgvhxxlywqfwm") # # mail_address3 = MailAddress(mail="ciyuexie@aol.com", password="czezlmmyypokdfce") # # mail_address4 = MailAddress(mail="hongjiang176@aol.com", password="ftzpscgzvwneelmn") - # mail_address4 = MailAddress(mail="likegirlpassed@aol.com", password="xjgohjpclvgpnhbz") + # mail_address4 = MailAddress(mail="ryan_meacham2856@yahoo.com", password="ulgggkodxqbvrpgm") # mail_list = [mail_address3, mail_address2, mail_address1, mail_address4] # mail_list = [mail_address4] successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day() @@ -163,11 +177,11 @@ def read_mails(): with ThreadPoolExecutor(max_workers=20) as executor: for mail in mail_list: # check whether we need to read mail - if need_to_check_email(mail, successful_items): + if need_to_check_email(mail.mail, successful_items): mail_reader = MailReader(mail.mail, mail.password) executor.submit(mail_reader.read_emails, mails_messages) - with ThreadPoolExecutor(max_workers=20) as executor: + with ThreadPoolExecutor(max_workers=5) as executor: for mail in mails_messages: match = re.search(VALIDATION_URL_REGEX, mail.body) if match: @@ -175,7 +189,7 @@ def read_mails(): if need_to_valid_url(url, successful_items): url_validator = LinkValidator(url) print("need to validate url: " + url) - executor.submit(url_validator.start_page, params.get_proxy(ProxyType.RESIDENTIAL), True) + executor.submit(url_validator.start_page, params.get_proxy(ProxyType.RESIDENTIAL), False) else: print("do not need to click url --> {}".format(mail.mail_address))