diff --git a/src/db/mongo_manager.py b/src/db/mongo_manager.py index e4a4f22..ff59746 100644 --- a/src/db/mongo_manager.py +++ b/src/db/mongo_manager.py @@ -166,7 +166,6 @@ class MongoDbManager: return result_list def get_accepted_items_for_one_day(self, day_in_str: str) -> list: - params.oracle_log_sender.send_read_db_event("get_accepted_items_for_one_day") collection_name = day_in_str result_list = [] cursor = self.db[collection_name] diff --git a/src/mail/get_invoices.py b/src/mail/get_invoices.py new file mode 100644 index 0000000..467cf7a --- /dev/null +++ b/src/mail/get_invoices.py @@ -0,0 +1,197 @@ +import datetime +import email +import imaplib +import uuid +from concurrent.futures import ThreadPoolExecutor +from email.header import decode_header +from email.message import Message + +from builtins import list +from src.db.mongo_manager import MONGO_STORE_MANAGER +from src.logs.AppLogging import init_logger +from src.mail.mail_constants import DOMAIN_163, DOMAIN_YAHOO, DOMAIN_SINA, IMAP_SERVER_163, YAHOO_IMAP_SERVER, \ + IMAP_SERVER_SINA, AOL_IMAP_SERVER +from src.pojo.accepted_appointment_pojo import AcceptedAppointmentPojo +from src.pojo.mail.mail_pojo import MailPojo + + +INVOICE_SUBJECT_fr = 'Votre facture' +INVOICE_SUBJECT_EN = 'Your invoice' +VALIDATION_URL_REGEX = """https:\/\/rendezvousparis.hermes.com\/client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+""" +HERMES_INVOICE_EMAIL = "no-reply@ereceipt.hermes.com" + +date_format = "%d-%b-%Y" # DD-Mon-YYYY e.g., 3-Mar-2014 + + +class InvoiceGetter(): + def __init__(self, login, password): + self.login = login + self.password = password + + @staticmethod + def show_folders(imap): + for i in imap.list()[1]: + l = i.decode().split(' "/" ') + print(l[0] + " = " + l[1]) + + def create_imap(self): + # create an IMAP4 class with SSL + if DOMAIN_163 in self.login: + imap = imaplib.IMAP4_SSL(IMAP_SERVER_163) + elif DOMAIN_YAHOO in self.login: + imap = imaplib.IMAP4_SSL(YAHOO_IMAP_SERVER) + elif DOMAIN_SINA in self.login: + imap = imaplib.IMAP4_SSL(IMAP_SERVER_SINA) + else: + imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER) + return imap + + def read_emails(self, mails_messages: list) -> list: + imap = self.create_imap() + + # authenticate + type, dat = imap.login(self.login, self.password) + mail_list = [] + print("read mails from {}".format(self.login)) + # self.show_folders(imap) + # total number of emails + # get mails from inbox + # (\Archive \HasNoChildren) = "Archive" + # (\Junk \HasNoChildren) = "Bulk" + # (\Drafts \HasNoChildren) = "Draft" + # (\HasNoChildren) = "Inbox" + # (\Sent \HasNoChildren) = "Sent" + # (\Trash \HasNoChildren) = "Trash" + mail_list.extend(self._get_messages_from_folder(imap, subject=INVOICE_SUBJECT_fr)) + mail_list.extend(self._get_messages_from_folder(imap, subject=INVOICE_SUBJECT_EN)) + # mail_list.extend(self._get_messages_from_folder(imap, folder="Bulk")) + # close the connection and logout + imap.close() + imap.logout() + mails_messages.extend(mail_list) + return mail_list + + # def downloadAllAttachmentsInInbox(self, imap): + # typ, items = imap.search(None, '(SUBJECT "{}" SINCE "{}")'.format(INVOICE_SUBJECT_fr, + # datetime.datetime.today().strftime( + # date_format))) + # items = items[0].split() + # self.downloaAttachmentsInEmail(imap, "./") + # + # def downloaAttachmentsInEmail(self, imap, outputdir): + # resp, data = imap.fetch(HERMES_INVOICE_EMAIL, "(BODY.PEEK[])") + # email_body = data[0][1] + # mail = email.message_from_string(email_body) + # if mail.get_content_maintype() != 'multipart': + # return + # for part in mail.walk(): + # if part.get_content_maintype() != 'multipart' and part.get('Content-Disposition') is not None: + # open(outputdir + '/' + part.get_filename(), 'wb').write(part.get_payload(decode=True)) + + def _get_messages_from_folder(self, imap, subject, folder="INBOX") -> list: + imap.select(folder) + mail_messages = [] + typ, data = imap.search(None, '(SUBJECT "{}" SINCE "{}")'.format(subject, datetime.datetime.today().strftime( + date_format))) + for i in data[0].split(): + # fetch the email message by ID + res, msg = imap.fetch(i.decode("utf-8"), "(RFC822)") + body = '' + for response in msg: + if isinstance(response, tuple): + # parse a bytes email into a message object + msg = email.message_from_bytes(response[1]) + # decode the email subject + subject, subject_encoded = decode_header(msg["Subject"])[0] + received_date = msg["Date"] + if isinstance(subject, bytes): + # if it's a bytes, decode to str + subject = subject.decode(subject_encoded) + # decode email sender + from_address, subject_encoded = decode_header(msg.get("From"))[0] + if isinstance(from_address, bytes): + from_address = from_address.decode(subject_encoded) + print("From:", from_address) + print("Subject:", subject) + # if the email message is multipart + if msg.is_multipart(): + # iterate over email parts + for part in msg.walk(): + try: + # get the email body + payloads = part.get_payload() + if isinstance(payloads, list): + for payload in payloads: + if isinstance(payload, Message): + if payload.get_content_type() == 'text/html': + body = body + payload.get_payload(decode=True).decode("iso-8859-1") + elif payload.get_content_type() == 'application/pdf': + # save to pdf + open("./" + str(uuid.uuid4()) + '.pdf', 'wb').write( + payload.get_payload(decode=True)) + + # print(body) + except Exception as Error: + print(Error) + else: + body = msg.get_payload(decode=True).decode() + print(body) + if INVOICE_SUBJECT_fr in subject or INVOICE_SUBJECT_EN in subject: + mail = MailPojo(subject=subject, body=body, from_address=from_address) + mail_messages.append(mail) + return mail_messages + + +def clean(text): + # clean text for creating a folder + return "".join(c if c.isalnum() else "_" for c in text) + + +def need_to_valid_url(url: str, successful_items) -> bool: + if len(successful_items) == 0: + return False + print("url is :" + url) + parts = url.split('/') + id = parts[5] + if len(id) == 6: + for item in successful_items: + # if item.url_validated is not None: + # print("id:{}, status:{} ".format(id, str(item.url_validated))) + if item.id == id: + if item.url_validated is not None: + return not item.url_validated + else: + # if url_validated is None + return True + # return True by default + return False + else: + print("id not valid:{}".format(id)) + return False + + +def get_invoices(): + # check time before start checking emails + # get email address + mail_list = MONGO_STORE_MANAGER.get_destination_emails() + mail_list_to_check = [] + yesterday = str(datetime.date.today() - datetime.timedelta(days=1)) + collection = MONGO_STORE_MANAGER.get_accepted_items_for_one_day(yesterday) + for valid_appointment in collection: + accepted_pojo = AcceptedAppointmentPojo.from_reserve(valid_appointment) + for mail in mail_list: + if mail.mail == accepted_pojo.email: + mail_list_to_check.append(mail) + # mail_list = [mail_address1] + mails_messages = [] + with ThreadPoolExecutor(max_workers=20) as executor: + for mail in mail_list_to_check: + # check whether we need to read mail + mail_reader = InvoiceGetter(mail.mail, mail.password) + executor.submit(mail_reader.read_emails, mails_messages) + + +# check whether the url has already been clicked +if __name__ == '__main__': + init_logger() + get_invoices()