Merge branch 'feature/discord'

read_confirmation_emails.py

@@ -1,4 +1,25 @@
+from src.discord_helper import send_message
 from src.mail.mail_confirmation import read_mails_and_find_confirmation_contacts
+from src.pojo import ReserveResultPojo
+
+
+def create_message_from_item(item: ReserveResultPojo):
+    return f"Phone: {item.phone}\n" \
+           f"URL: {item.url}\n" \
+           f"Email: {item.mail}\n" \
+           f"First Name: {item.first_name}\n" \
+           f"Last Name: {item.last_name}\n"
+
+
+def main():
+    # initialize discord
+    print("init discord done")
+    # wait for discord to start up
+    # time.sleep(10)
+    _accepted_appointments = read_mails_and_find_confirmation_contacts()
+    for item in _accepted_appointments:
+        send_message(create_message_from_item(item))
+
 
 if __name__ == '__main__':
-    read_mails_and_find_confirmation_contacts()
+    main()

src/discord_helper.py

@@ -0,0 +1,14 @@
+import requests
+
+DISCORD_SERVER_URL = "http://127.0.0.1:9000/discord/send"
+
+
+def send_message(message: str):
+    headers = {"Authorization": "Bearer 97e36f7e-340e-4c02-b329-9415faee38c3", "Content-Type": "application/json"}
+    message_data = {"message": message}
+    _res = requests.post(DISCORD_SERVER_URL, json=message_data, headers=headers)
+    print(_res.text)
+
+
+if __name__ == '__main__':
+    send_message('测试')

src/mail/mail_confirmation.py

@@ -1,5 +1,7 @@
 import datetime
 import email
+import logging
+import sys
 from builtins import list
 from concurrent.futures import ThreadPoolExecutor
 from email.header import decode_header
@@ -8,6 +10,7 @@ from imapclient import IMAPClient
 
 from src.db.mirgration.migration_tools import migre_accepted_appointment
 from src.db.mongo_manager import MONGO_STORE_MANAGER
+from src.logs.AppLogging import init_logger
 from src.mail.mail_constants import create_imap, show_folders
 from src.notification.AcceptedResultPojo import get_accepted_result_from
 from src.notification.SignalSender import SignalSender
@@ -15,13 +18,13 @@ from src.notification.mailer import Mailer
 from src.pojo.ResultEnum import ResultEnum
 from src.pojo.mail.mail_pojo import MailPojo, MailAddress
 
-CONFIRMATION_SUBJECT_FR = 'rendez-vous est'
+CONFIRMATION_SUBJECT_FR = 'Votre=20rendez-vous=20est=20confirm=C3'
 CONFIRMATION_SUBJECT_EN = 'confirmed'
 HERMES_EMAIL = "no-reply@hermes.com"
 
 date_format = "%d-%b-%Y"  # DD-Mon-YYYY e.g., 3-Mar-2014
 
-FRENCH_CONFIRMED_MESSAGE = "Nous avons le plaisir de vous confirmer votre rendez-vous"
+FRENCH_CONFIRMED_MESSAGE = "Nous aurons le plaisir de vous accueillir"
 
 
 class MailConfirmationReader():
@@ -60,16 +63,13 @@ class MailConfirmationReader():
         search_terms = 'SINCE "{}"'.format(
             datetime.datetime.today().strftime(
                 date_format))
-        print("search terms is " + search_terms)
         imap.select_folder(folder)
         messages = imap.search(['SINCE', datetime.datetime.today()])
-        print("%d messages from our best friend" % len(messages))
         for uid, message_data in imap.fetch(messages, 'RFC822').items():
             try:
                 email_message = email.message_from_bytes(message_data[b'RFC822'])
                 from_address = email_message.get('FROM')
                 subject = email_message.get('subject')
-                # print("{}, {},{}".format(from_address, subject, email_message))
                 body = ""
                 if "no-reply@hermes.com" in from_address:
                     for part in email_message.walk():
@@ -78,12 +78,11 @@ class MailConfirmationReader():
                             body = body + str(part.get_payload(decode=True).decode("utf-8"))
                         elif part.get_content_type() == "text/plain":
                             body = body + part.get_payload()
-                    print("mail is {} and subject is {}".format(self.login, subject))
+                    logger.info("mail is {} and subject is {}, body is {}".format(self.login, subject, body))
                     if CONFIRMATION_SUBJECT_FR in subject or CONFIRMATION_SUBJECT_EN in subject or "Votre_rendez-vous_est_confirm" in subject:
                         mail = MailPojo(subject=subject, body=body, from_address=from_address)
                         mail.isImapClient = True
-                        print("subject is {}".format(subject))
+                        mail.mail_address = self.login
-                        print("body is {}".format(body))
                         mail_messages.append(mail)
             except Exception as error:
                 print(error)
@@ -93,7 +92,7 @@ class MailConfirmationReader():
     def _get_messages_from_folder(self, imap, subject, folder="INBOX") -> list:
         imap.select(folder)
         mail_messages = []
-        typ, data = imap.search(None, '(SUBJECT "{}" SINCE "{}")'.format(subject,
+        typ, data = imap.search(None, '(SINCE "{}")'.format(
             datetime.datetime.today().strftime(
                 date_format)))
         for i in data[0].split():
@@ -114,8 +113,8 @@ class MailConfirmationReader():
                     from_address, subject_encoded = decode_header(msg.get("From"))[0]
                     if isinstance(from_address, bytes):
                         from_address = from_address.decode(subject_encoded)
-                    print("From:", from_address)
+                    logger.info("From:{}".format(from_address))
-                    print("Subject:", subject)
+                    logger.info("Subject: {}".format(subject))
                     # if the email message is multipart
                     if msg.is_multipart():
                         # iterate over email parts
@@ -132,7 +131,7 @@ class MailConfirmationReader():
                                 print(Error)
                     else:
                         body = msg.get_payload(decode=True).decode()
-                        print(body)
+                        logger.info(body)
                     if CONFIRMATION_SUBJECT_FR in subject or CONFIRMATION_SUBJECT_EN in subject:
                         mail = MailPojo(subject=subject, body=body, from_address=from_address)
                         mail.mail_address = self.login
@@ -156,7 +155,7 @@ def accept_appointment_found(accepted_result_list: list):
         for user in _all_register_account:
             if user.mail == result.email:
                 result.account_password = user.password
-        mailer.send_email(result, to_all=False)
+        mailer.send_email(result, to_all=True)
         MONGO_STORE_MANAGER.update_reserve_result(reserve.id, ResultEnum.ACCEPTED, reserve.message)
         # sginal.send_result(result)
 
@@ -173,7 +172,7 @@ def find_confirmation_contacts_for_today():
     for _item in _all_appointments_today:
         for _mail in _all_mail_list:
             if _mail.mail == _item.mail:
-                if _item.url_validated is True:
+                # if _item.url_validated is True:
                 _mail_list_to_scan.append(_mail)
                 break
     print("Found {} emails to scan".format(len(_mail_list_to_scan)))
@@ -218,6 +217,7 @@ def find_confirmation_contacts_mail_list(mail_list):
             print(mail.subject)
             print(mail.body)
         accept_appointment_found(accepted_appointment_list)
+    return accepted_appointment_list
 
 
 def read_mails_and_find_confirmation_contacts(all_mails=False):
@@ -226,7 +226,7 @@ def read_mails_and_find_confirmation_contacts(all_mails=False):
         mail_list = MONGO_STORE_MANAGER.get_destination_emails()
     else:
         mail_list = find_confirmation_contacts_for_today()
-    find_confirmation_contacts_mail_list(mail_list)
+    return find_confirmation_contacts_mail_list(mail_list)
     # mail_list.reverse()
     # excel_reader = ExcelHelper()
     # mail_list =excel_reader.read_email_pojo(file_name="/Users/lpan/Desktop/hotmail_list.xlsx")
@@ -234,6 +234,9 @@ def read_mails_and_find_confirmation_contacts(all_mails=False):
     # mail_list = [mail_address3]
 
 
+# init_logger()
+logger = logging.getLogger()
+logger.addHandler(logging.StreamHandler(stream=sys.stdout))
 # check whether the url has already been clicked
 if __name__ == '__main__':
     # read_mails_and_find_confirmation_contacts()

src/mail/mail_constants.py

@@ -86,47 +86,65 @@ def create_imap(login: str):
     if DOMAIN_163 in login:
         imap = IMAPClient(IMAP_SERVER_163, use_uid=True)
     elif DOMAIN_YAHOO in login:
-        imap = imaplib.IMAP4_SSL(YAHOO_IMAP_SERVER)
+        # imap = imaplib.IMAP4_SSL(YAHOO_IMAP_SERVER)
+        imap = IMAPClient(YAHOO_IMAP_SERVER, use_uid=True)
     elif DOMAIN_FIREMAIL_DE in login:
         # imap = imaplib.IMAP4_SSL(SERVER_FIREMAIL_DE)
         imap = IMAPClient(SERVER_FIREMAIL_DE, use_uid=True)
     elif DOMAIN_GMX in login or DOMAIN_GMX_FR in login or DOMAIN_GMX_US in login or DOMAIN_GMX_CH in login or DOMAIN_GMX_PT in login or DOMAIN_GMX_SG in login:
-        imap = imaplib.IMAP4_SSL(SERVER_GMX)
+        # imap = imaplib.IMAP4_SSL(SERVER_GMX)
+        imap = IMAPClient(SERVER_GMX, use_uid=True)
     elif DOMAIN_SINA in login:
         # imap = imaplib.IMAP4_SSL(IMAP_SERVER_SINA)
         imap = IMAPClient(IMAP_SERVER_SINA, use_uid=True)
     elif DOMAIN_HOTMAIL in login or DOMAIN_OUTLOOK_COM in login:
-        imap = imaplib.IMAP4_SSL(HOTMAIL_IMAP_SERVER)
+        # imap = imaplib.IMAP4_SSL(HOTMAIL_IMAP_SERVER)
+        imap = IMAPClient(HOTMAIL_IMAP_SERVER, use_uid=True)
     elif DOMAIN_RAMBLER_RU in login:
-        imap = imaplib.IMAP4_SSL(RAMBLER_IMAP_SERVER)
+        # imap = imaplib.IMAP4_SSL(RAMBLER_IMAP_SERVER)
+        imap = IMAPClient(RAMBLER_IMAP_SERVER, use_uid=True)
     elif DOMAN_BTVM_NE_JP in login:
-        imap = imaplib.IMAP4_SSL(BTVM_NE_JP)
+        # imap = imaplib.IMAP4_SSL(BTVM_NE_JP)
+        imap = IMAPClient(BTVM_NE_JP, use_uid=True)
     elif DOMAN_GMAIL in login:
-        imap = imaplib.IMAP4_SSL(SEREVER_GMAIL, port=993)
+        # imap = imaplib.IMAP4_SSL(SEREVER_GMAIL, port=993)
+        imap = IMAPClient(SEREVER_GMAIL, use_uid=True)
     elif DOMAIN_ONET in login:
         imap = IMAPClient(SERVER_IMAGE_ONET, use_uid=True)
     elif DOMAIN_TIM_IT in login:
-        imap = imaplib.IMAP4(TIME_IT_SERVER)
+        # imap = imaplib.IMAP4(TIME_IT_SERVER)
+        imap = IMAPClient(TIME_IT_SERVER, use_uid=True)
     elif DOMAIN_ALICE_IT in login:
-        imap = imaplib.IMAP4(ALICE_IMAP_SERVER, port=143)
+        # imap = imaplib.IMAP4(ALICE_IMAP_SERVER, port=143)
+        imap = IMAPClient(ALICE_IMAP_SERVER, use_uid=True)
     elif DOMAIN_MARS_DTI_NE_JP in login:
-        imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143)
+        # imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143)
+        imap = IMAPClient(MARS_DTI_NE_JP_SERVER, use_uid=True)
     elif DOMAN_AURORA_DTI_NE_JP in login:
-        imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143)
+        # imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143)
+        imap = IMAPClient(MARS_DTI_NE_JP_SERVER, use_uid=True)
     elif DOMAIN_NAVER in login:
-        imap = imaplib.IMAP4_SSL(NAVER_SERVER, port=993)
+        # imap = imaplib.IMAP4_SSL(NAVER_SERVER, port=993)
+        imap = IMAPClient(NAVER_SERVER, use_uid=True)
     elif DOMAIN_GMX_DE in login or DOMAIN_GMX_NET in login:
-        imap = imaplib.IMAP4_SSL(SERVER_GMX_NET, port=993)
+        # imap = imaplib.IMAP4_SSL(SERVER_GMX_NET, port=993)
+        imap = IMAPClient(SERVER_GMX_NET, use_uid=True)
     elif DOMAIN_GMX_AT in login:
-        imap = imaplib.IMAP4_SSL(SERVER_GMX_AT, port=993)
+        # imap = imaplib.IMAP4_SSL(SERVER_GMX_AT, port=993)
+        imap = IMAPClient(SERVER_GMX_AT, use_uid=True)
     elif DOMAIN_GAZETA_PL in login:
-        imap = imaplib.IMAP4_SSL(IMAP_SERVER_DOMAIN_GAZETA_PL, port=993)
+        # imap = imaplib.IMAP4_SSL(IMAP_SERVER_DOMAIN_GAZETA_PL, port=993)
+        imap = IMAPClient(IMAP_SERVER_DOMAIN_GAZETA_PL, use_uid=True)
     elif DOMAIN_INBOX_LV in login:
-        imap = imaplib.IMAP4_SSL(INBOX_LV, port=993)
+        # imap = imaplib.IMAP4_SSL(INBOX_LV, port=993)
+        imap = IMAPClient(INBOX_LV, use_uid=True)
     elif DOMAIN_WEB_DE in login:
-        imap = imaplib.IMAP4_SSL(SERVER_WEB_DE, port=993)
+        # imap = imaplib.IMAP4_SSL(SERVER_WEB_DE, port=993)
+        imap = IMAPClient(SERVER_WEB_DE, use_uid=True)
     elif DOMAIN_PISS_MAIL in login or DOMAIN_CHILD_PIZZA in login or DOMAIN_DMC_CHAT in login or DOMAIN_GENOCIDE_FUN in login or DOMAIN_HATESJE_WS in login or DOMAIN_INCEL_EMAIL in login or DOMAIN_SHITPOSTING_EXPERT in login:
-        imap = imaplib.IMAP4_SSL(SERVER_PISS_MAIL, port=993)
+        # imap = imaplib.IMAP4_SSL(SERVER_PISS_MAIL, port=993)
+        imap = IMAPClient(SERVER_PISS_MAIL, use_uid=True)
     else:
-        imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER)
+        # imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER)
+        imap = IMAPClient(AOL_IMAP_SERVER, use_uid=True)
     return imap

src/mail/mail_reader.py

@@ -166,7 +166,7 @@ class MailReader():
                             body = body + part.get_payload(decode=True).decode("utf-8")
                         elif part.get_content_type() == "text/plain":
                             body = body + part.get_payload()
-                    if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject or "Validation=20de=20votre=20demande=20de=20rendez-vous" in subject:
+                    if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject or "Votre=20demande=20de=20rendez-vous" in subject:
                         mail = MailPojo(subject=subject, body=body, from_address=from_address)
                         mail.isImapClient = True
                         print("email is {}".format(self.login))

src/mail/mail_reader_all_contacts.py

@@ -185,7 +185,7 @@ class MailReader():
                             body = body + part.get_payload(decode=True).decode("utf-8")
                         elif part.get_content_type() == "text/plain":
                             body = body + part.get_payload()
-                    if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject or "Validation=20de=20votre=20demande=20de=20rendez-vous" in subject:
+                    if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject or "Votre=20demande=20de=20rendez-vous" in subject:
                         mail = MailPojo(subject=subject, body=body, from_address=from_address)
                         mail.isImapClient = True
                         print("email is {}".format(self.login))
@@ -255,7 +255,7 @@ def need_to_check_email(mail: str, successful_items) -> bool:
 
 def read_all_mails(contact_to_book_list=None):
     # check time before start checking emails
-    if is_time_between(time(7, 30), time(19, 30)):
+    if is_time_between(time(7, 30), time(23, 50)):
         # get email address
         if contact_to_book_list is None:
             contact_to_book_list = MONGO_STORE_MANAGER.get_all_contact_to_book_list()
@@ -283,9 +283,6 @@ def read_all_mails(contact_to_book_list=None):
         _refreshed_successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
         with ThreadPoolExecutor(max_workers=10) as executor:
             for mail in mails_messages:
-                # if mail.isImapClient:
-                #     match = re.search(PART_VALIDATION_URL_REGEX, mail.body.replace("\n", ""))
-                # else:
                 match = re.search(VALIDATION_URL_REGEX, mail.body)
                 if match:
                     # url_to_validate = match.group(0)
@@ -297,8 +294,6 @@ def read_all_mails(contact_to_book_list=None):
                         print("need to validate url: " + url)
                         MONGO_STORE_MANAGER.save_links_to_validate(url, mail.to_address,
                                                                    _all_contact_list=contact_to_book_list)
-                        # url_validator = LinkValidator(url)
-                        # executor.submit(url_validator.start_page, params.get_proxy(ProxyType.OXYLABS), False)
                     else:
                         print("do not need to click url --> {}".format(mail.mail_address))
 

src/pojo/accepted_appointment_pojo.py

@@ -1,3 +1,5 @@
+from typing import Union
+
 import time
 
 from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType
@@ -5,7 +7,7 @@ from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType
 
 class AcceptedAppointmentPojo(ReserveResultPojo):
     accepted_at: float
-    day: str
+    day: Union[str,None]
 
     def __init__(self, reserve: ReserveResultPojo):
         self.id = reserve.id
@@ -13,15 +15,16 @@ class AcceptedAppointmentPojo(ReserveResultPojo):
         self.passport = reserve.passport
         self.url = reserve.url
         self.store_type = reserve.store_type
-        self.lastName = reserve.last_name
+        self.last_name = reserve.last_name
-        self.firstName = reserve.first_name
+        self.first_name = reserve.first_name
         self.ccid = reserve.ccid
-        self.email = reserve.mail
+        self.mail = reserve.mail
         self.phone = reserve.phone
         self.message = reserve.message
         self.source_from = reserve.source_from
         self.created_at = reserve.created_at
         self.validated_at = reserve.validated_at
+        self.day = None
 
     @staticmethod
     def from_reserve(reserve: ReserveResultPojo):
@@ -33,10 +36,10 @@ class AcceptedAppointmentPojo(ReserveResultPojo):
         dest = {
             u'id': self.id,
             u'phone': self.phone,
-            u'firstName': self.firstName,
+            u'firstName': self.first_name,
-            u'lastName': self.lastName,
+            u'lastName': self.last_name,
             u'message': self.message,
-            u'email': self.email,
+            u'email': self.mail,
             u'passport': self.passport,
             u'url': self.url,
             u'ccid': self.ccid,
@@ -65,8 +68,8 @@ class AcceptedAppointmentPojo(ReserveResultPojo):
         lastName = source['lastName']
         firstName = source['firstName']
         result = ReserveResultPojo(type=publish_type, phone=phone,
-                                   url=url, email=email,
+                                   url=url, mail=email,
-                                   firstName=firstName, lastName=lastName)
+                                   first_name=firstName, last_name=lastName)
         if 'accepted' in source:
             accepted = source['accepted']
             result.accepted = accepted
@@ -95,5 +98,9 @@ class AcceptedAppointmentPojo(ReserveResultPojo):
         accepted_pojo = AcceptedAppointmentPojo.from_reserve(result)
         if 'accepted_at' in source:
             accepted_at = source['accepted_at']
+            # accepted_pojo.accepted_at = int(accepted_at)
             accepted_pojo.accepted_at = accepted_at
+        if 'day' in source:
+            day = source['day']
+            accepted_pojo.day = day
         return accepted_pojo