From de9f2756359a7362ad0a38353d3b9c7e652e5461 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 7 Apr 2023 15:21:11 +0200 Subject: [PATCH] diff ImapCLient --- src/mail/mail_reader.py | 150 ++++++++++++++++++++++++---------------- 1 file changed, 89 insertions(+), 61 deletions(-) diff --git a/src/mail/mail_reader.py b/src/mail/mail_reader.py index dea865b..3a0ca88 100644 --- a/src/mail/mail_reader.py +++ b/src/mail/mail_reader.py @@ -7,6 +7,8 @@ from datetime import time from email.header import decode_header from email.message import Message +from imapclient import IMAPClient + from src.db.mongo_manager import MONGO_STORE_MANAGER from src.logs.AppLogging import init_logger from src.mail.mail_constants import DOMAIN_HOTMAIL, create_imap @@ -15,8 +17,8 @@ from src.utils.timeutiles import is_time_between VALIDATION_URL_SUBJECT_fr = 'Validation de votre demande de rendez-vous' VALIDATION_URL_SUBJECT_EN = 'Please confirm your appointment request' -# VALIDATION_URL_REGEX = """https:\/\/rendezvousparis.hermes.com\/client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+""" -VALIDATION_URL_REGEX = """client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+""" +VALIDATION_URL_REGEX = """https:\/\/rendezvousparis.hermes.com\/client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+""" +PART_VALIDATION_URL_REGEX = """client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+""" HERMES_EMAIL = "no-reply@hermes.com" date_format = "%d-%b-%Y" # DD-Mon-YYYY e.g., 3-Mar-2014 @@ -37,32 +39,90 @@ class MailReader(): return folders def read_emails(self, mails_messages: list) -> list: - # create an IMAP4 class with SSL imap = create_imap(self.login) - # authenticate - dat = imap.login(self.login, str(self.password)) - print("type is {} for {}".format(dat, self.login)) + isImapClient = isinstance(imap, IMAPClient) + print("isImapClient is " + str(isImapClient)) + if isImapClient: + # authenticate + dat = imap.login(self.login, str(self.password)) + print("type is {} for {}".format(dat, self.login)) + else: + responseType, dat = imap.login(self.login, str(self.password)) + print("type is {} for {}".format(responseType, self.login)) + mail_list = [] print("read mails from {}".format(self.login)) - # folder_list = self.show_folders(imap) - # for folder in folder_list: - # print("folder is {}".format(folder)) - mail_list.extend(self._get_messages_from_folder(imap, subject=VALIDATION_URL_SUBJECT_fr)) - mail_list.extend(self._get_messages_from_folder(imap, subject=VALIDATION_URL_SUBJECT_EN)) + if not isImapClient: + folder_list = self.show_folders(imap) + for folder in folder_list: + print("folder is {}".format(folder)) + mail_list.extend(self._get_messages_from_folder(imap, subject=VALIDATION_URL_SUBJECT_fr, + folder=folder)) + mail_list.extend(self._get_messages_from_folder(imap, subject=VALIDATION_URL_SUBJECT_EN, + folder=folder)) + else: + mail_list.extend(self._get_messages_from_folder_for_imapclient(imap, subject=VALIDATION_URL_SUBJECT_fr)) + mail_list.extend(self._get_messages_from_folder_for_imapclient(imap, subject=VALIDATION_URL_SUBJECT_EN)) if DOMAIN_HOTMAIL in self.login: - mail_list.extend(self._get_messages_from_folder(imap, subject=VALIDATION_URL_SUBJECT_EN, folder="Junk")) - # mail_list.extend(self._get_messages_from_folder(imap, subject=VALIDATION_URL_SUBJECT_EN, folder="Bulk")) - # close the connection and logout - # imap.close() + mail_list.extend( + self._get_messages_from_folder_for_imapclient(imap, subject=VALIDATION_URL_SUBJECT_EN, folder="Junk")) + if not isImapClient: + imap.close() imap.logout() mails_messages.extend(mail_list) return mail_list def _get_messages_from_folder(self, imap, subject, folder="INBOX") -> list: + imap.select(folder) + mail_messages = [] + typ, data = imap.search(None, '(SUBJECT "{}" SINCE "{}")'.format(subject, + datetime.datetime.today().strftime( + date_format))) + for i in data[0].split(): + # fetch the email message by ID + res, msg = imap.fetch(i.decode("utf-8"), "(RFC822)") + body = '' + for response in msg: + if isinstance(response, tuple): + # parse a bytes email into a message object + msg = email.message_from_bytes(response[1]) + # decode the email subject + subject, subject_encoded = decode_header(msg["Subject"])[0] + received_date = msg["Date"] + if isinstance(subject, bytes): + # if it's a bytes, decode to str + subject = subject.decode(subject_encoded) + # decode email sender + from_address, subject_encoded = decode_header(msg.get("From"))[0] + if isinstance(from_address, bytes): + from_address = from_address.decode(subject_encoded) + print("Email:", self.login) + print("From:", from_address) + print("Subject:", subject) + # if the email message is multipart + if msg.is_multipart(): + # iterate over email parts + for part in msg.walk(): + try: + # get the email body + payloads = part.get_payload() + if isinstance(payloads, list): + for payload in payloads: + if isinstance(payload, Message): + body = body + payload.get_payload(decode=True).decode("iso-8859-1") + # print(body) + except Exception as Error: + print(Error) + else: + body = msg.get_payload(decode=True).decode() + print(body) + if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject: + mail = MailPojo(subject=subject, body=body, from_address=from_address) + mail_messages.append(mail) + return mail_messages + + def _get_messages_from_folder_for_imapclient(self, imap, subject, folder="INBOX") -> list: mail_messages = [] - # search_terms = '(SUBJECT "{}" SINCE "{}")'.format(subject, - # datetime.datetime.today().strftime( - # date_format)) search_terms = 'SINCE "{}"'.format( datetime.datetime.today().strftime( date_format)) @@ -86,50 +146,12 @@ class MailReader(): body = body + part.get_payload() if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject: mail = MailPojo(subject=subject, body=body, from_address=from_address) + mail.isImapClient = True + print("body is {}".format(body)) mail_messages.append(mail) except Exception as error: + print(error) print("error trying to read email_Message for {}".format(self.login)) - # for i in data[0].split(): - # # fetch the email message by ID - # res, msg = imap.fetch(i, "(RFC822)") - # body = '' - # for response in msg: - # if isinstance(response, tuple): - # # parse a bytes email into a message object - # msg = email.message_from_bytes(response[1]) - # # decode the email subject - # subject, subject_encoded = decode_header(msg["Subject"])[0] - # received_date = msg["Date"] - # if isinstance(subject, bytes): - # # if it's a bytes, decode to str - # subject = subject.decode(subject_encoded) - # # decode email sender - # from_address, subject_encoded = decode_header(msg.get("From"))[0] - # if isinstance(from_address, bytes): - # from_address = from_address.decode(subject_encoded) - # print("Email:", self.login) - # print("From:", from_address) - # print("Subject:", subject) - # # if the email message is multipart - # if msg.is_multipart(): - # # iterate over email parts - # for part in msg.walk(): - # try: - # # get the email body - # payloads = part.get_payload() - # if isinstance(payloads, list): - # for payload in payloads: - # if isinstance(payload, Message): - # body = body + payload.get_payload(decode=True).decode("iso-8859-1") - # # print(body) - # except Exception as Error: - # print(Error) - # else: - # body = msg.get_payload(decode=True).decode() - # print(body) - # if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject: - # mail = MailPojo(subject=subject, body=body, from_address=from_address) - # mail_messages.append(mail) return mail_messages @@ -196,10 +218,16 @@ def read_mails(): with ThreadPoolExecutor(max_workers=10) as executor: for mail in mails_messages: - match = re.search(VALIDATION_URL_REGEX, mail.body.replace("\n", "")) + if mail.isImapClient: + match = re.search(PART_VALIDATION_URL_REGEX, mail.body.replace("\n", "")) + else: + match = re.search(VALIDATION_URL_REGEX, mail.body) if match: url_to_validate = match.group(0) - url = "https://rendezvousparis.hermes.com/" + url_to_validate + if mail.isImapClient: + url = "https://rendezvousparis.hermes.com/" + url_to_validate.replace("3D", "") + else: + url = match.group(0) if need_to_valid_url(url, successful_items): MONGO_STORE_MANAGER.save_links_to_validate(url) # url_validator = LinkValidator(url)