Merge branch 'feature/log_mails' into feature/discord

src/check_results.py

@@ -1,146 +0,0 @@
-import datetime
-import logging
-import random
-import threading
-from concurrent.futures import ThreadPoolExecutor
-from typing import Union
-
-from playwright.sync_api import sync_playwright
-
-from src.db.mirgration.migration_tools import migre_accepted_appointment
-from src.db.mongo_manager import MONGO_STORE_MANAGER
-from src.definitions import LOG_SUBJECT_EVENT, TYPE_EVENT_CHECK_RESULTS
-from src.logs.LogSender import LogSender
-from src.notification.AcceptedResultPojo import get_accepted_result_from
-from src.notification.mailer import Mailer
-from src.pojo.ReserveResultPojo import ReserveResultPojo
-from src.pojo.ResultEnum import ResultEnum
-from src.proxy.proxy_type import ProxyType
-from src import definitions, params
-from src.workers.TlsPlaywright import TlsPlaywright
-
-SORRY_SENTENCE_FR = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci"
-SORRY_SENTENCE_EN = "we are extremely sorry that we were not able to fulfill"
-NOT_AVAILABLE_CONTENT = "For more than 130 years, our House has offered its full expertise to satisfy"
-PENDING_SENTENCE = "Ce soir, entre 20:00 et 20:30, vous obtiendrez une réponse par e-mail."
-URL_TO_VALID_SENTENCE = "Nous avons envoyé un lien par e-mail."
-PENDING_SENTENCE_EN = "This evening between 20:00 and 20:30 you will receive a response by email."
-CAPTCHA_URL = "https://geo.captcha-delivery.com"
-# URLs to ignore during checking results
-BLANK_URL = "about:blank"
-WELCOME_URL = "https://rendezvousparis.hermes.com/client/welcome"
-
-mailer = Mailer()
-oracle_log_sender = LogSender()
-
-
-class ResultChecker:
-    tls = TlsPlaywright()
-
-    def __init__(self):
-        self.logger = logging.getLogger("Worker")
-
-    def load_page(self, playwright, proxy, url, device, headless) -> Union[str, None]:
-        try:
-            self.browser = playwright.webkit.launch(headless=headless, timeout=90000, proxy=proxy)
-            pixel_2 = self.tls.playwright.devices[device]
-            context = self.browser.new_context(**pixel_2, locale='fr-FR')
-            self.page = context.new_page()
-            # hide webdriver information
-            self.page.add_init_script("""() => {
-        Object.defineProperty(navigator,'webdriver',{get: () => undefined});
-        Object.defineProperty(navigator, 'platform', {
-            get: () => {
-                return "iPhone";
-            }});
-    }
-                """)
-            self.page.goto(url, timeout=90000)
-            return self.page.content()
-        except Exception as error:
-            print(error)
-            self.logger.info("will close browser")
-            self.browser.close()
-            return None
-
-    def run(self, reserve_pojo: ReserveResultPojo, firestore_collection, headless=False, need_send_email=False):
-        url = reserve_pojo.url
-        print("url is " + url)
-        content = None
-        proxy = params.get_proxy(ProxyType.OXYLABS)
-        device = random.choice(params.DEVICES)
-        self.logger.info("模拟设备: " + device)
-        while content is None:
-            content = self.load_page(self.tls.playwright, proxy, url, device, headless)
-            proxy = params.get_proxy(ProxyType.OXYLABS)
-        print(content)
-        print("Stopped worker in ", threading.current_thread().name)
-        if SORRY_SENTENCE_FR in content:
-            print("status is REFUSED")
-            status = ResultEnum.REFUSED
-        elif SORRY_SENTENCE_EN in content:
-            print("status is REFUSED")
-            status = ResultEnum.REFUSED
-        elif PENDING_SENTENCE in content:
-            print("status is PENDING")
-            status = ResultEnum.PENDING
-        elif URL_TO_VALID_SENTENCE in content:
-            print("status is REFUSED")
-            status = ResultEnum.REFUSED
-        elif PENDING_SENTENCE_EN in content:
-            print("status is PENDING")
-            status = ResultEnum.PENDING
-        elif NOT_AVAILABLE_CONTENT in content:
-            print("status is REFUSED")
-            status = ResultEnum.REFUSED
-        elif CAPTCHA_URL in content:
-            print("status is BLOCKED")
-            self.browser.close()
-        else:
-            print("status is ACCEPTED")
-            status = ResultEnum.ACCEPTED
-            # send email
-            try:
-                mailer.send_email(get_accepted_result_from(reserve_pojo), to_all=need_send_email)
-            except Exception as err:
-                print(err)
-        reserve_pojo.accepted = status
-        MONGO_STORE_MANAGER.update_reserve_result(reserve_pojo.id, status)
-        self.browser.close()
-
-
-def check_results(headless=False):
-    # get the list
-    oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS)
-    db_manager = definitions.firebase_store_manager
-    firestore_collection = db_manager.get_all_successful_items()
-    reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
-    print("size is " + str(len(reserve_list)))
-    start_check(reserve_list, firestore_collection, headless, need_send_email=False)
-    reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
-    start_check(reserve_list, firestore_collection, headless, need_send_email=True)
-    # copy the accepted info to the accepted collection
-    migre_accepted_appointment(str(datetime.date.today()))
-
-
-def start_check(reserve_list, firestore_collection, headless: bool, need_send_email: bool):
-    count = 0
-    with ThreadPoolExecutor(max_workers=20) as executor:
-        for reserve in reserve_list:
-            count = count + 1
-            if reserve.accepted is None or ResultEnum.ACCEPTED.value == reserve.accepted:
-                print("will check result")
-                if "hotmail" in reserve.email:
-                    if reserve.url != BLANK_URL:
-                        if reserve.url != WELCOME_URL:
-                            executor.submit(ResultChecker().run, reserve, firestore_collection, headless,
-                                            need_send_email)
-            else:
-                print("status is " + reserve.accepted)
-
-        print(count)
-
-
-# need to start at 21h00
-if __name__ == '__main__':
-    check_results()

src/db/mongo_manager.py

@@ -1,6 +1,7 @@
 import datetime
 import logging
 import time
+from typing import Union
 
 from pymongo import MongoClient
 
@@ -24,6 +25,7 @@ DESTINATION_EMAIL_LIST = "DESTINATION_EMAIL_LIST"
 LINKS_TO_VALIDATE = "LINKS_TO_VALIDATE"
 INVALID_EMAIL_LIST = "INVALID_EMAIL_LIST"
 ADRESSE_LIST = "address"
+CONTACT_LIST_SERIAL_MAP = "CONTACT_LIST_SERIAL_MAP"
 
 
 class MongoDbManager:
@@ -87,6 +89,17 @@ class MongoDbManager:
         except Exception as Error:
             self.logger.info(Error)
 
+    def upload_contact_list_to_collection(self, contact_list: list, collection_name):
+
+        try:
+            collection_to_use = self.db[collection_name]
+            for contact in contact_list:
+                # collection_to_use.insert_one(contact.to_firestore_dict())
+                collection_to_use.replace_one(filter={'_id': contact.mail, }, replacement=contact.to_firestore_dict(),
+                                              upsert=True)
+        except Exception as Error:
+            self.logger.info(Error)
+
     def get_all_accepted_appointments(self) -> list:
         collection_name = ACCEPTED_APPOINTMENT_LIST
         appointment_list_contacts = []
@@ -306,10 +319,30 @@ class MongoDbManager:
             _all_contact_list.append(ContactPojo.from_firestore_dict(document))
         return _all_contact_list
 
-    def save_links_to_validate(self, link: str, mail_address: str, _all_contact_list: list):
+    def get_all_contact_serial_list(self) -> list:
+        result_list = []
+        cursor = self.db[CONTACT_LIST_SERIAL_MAP]
+        for document in cursor.find():
+            result_list.append(ContactPojo.from_firestore_dict(document))
+        return result_list
+
+    def save_links_to_validate(self, link: str, mail_address: str, _all_contact_list: list,
+                               _item: Union[ReserveResultPojo, None], contact_serial_map_list):
         collection_to_use = self.db[LINKS_TO_VALIDATE]
         updated_at = time.strftime("%H:%M:%S", time.localtime())
         _ip_country = "FR"
+        _model = ""
+        serial = ""
+        if _item:
+            _model = _item.source_from
+            serial = _item.serial
+        if len(serial) == 0:
+            for contact in contact_serial_map_list:
+                if contact.mail == mail_address:
+                    serial = contact.serial
+                    _model = contact.model
+                    break
+
         # find ip_country info
         for _contact in _all_contact_list:
             if _contact.mail == mail_address:
@@ -319,6 +352,8 @@ class MongoDbManager:
             collection_to_use.replace_one(filter={'_id': mail_address, }, replacement={
                 u'url': link,
                 u'email': mail_address,
+                u'model': _model,
+                u'serial': serial,
                 u'ip_country': _ip_country,
                 "updated_at": updated_at
             },
@@ -326,6 +361,8 @@ class MongoDbManager:
         else:
             collection_to_use.replace_one(filter={'_id': link, }, replacement={
                 u'url': link,
+                u'model': _model,
+                u'serial': serial,
                 u'ip_country': _ip_country,
                 "updated_at": updated_at
             },

src/mail/mail_reader.py

@@ -11,9 +11,8 @@ from imapclient import IMAPClient
 
 from src.db.mongo_manager import MONGO_STORE_MANAGER
 from src.logs.AppLogging import init_logger
-from src.mail.mail_constants import DOMAIN_HOTMAIL, create_imap, show_folders
+from src.mail.mail_constants import create_imap, show_folders
-from src.pojo.mail.mail_pojo import MailPojo, MailAddress
+from src.pojo.mail.mail_pojo import MailPojo
-from src.utils.excel_reader import ExcelHelper
 from src.utils.timeutiles import is_time_between
 
 VALIDATION_URL_SUBJECT_fr = 'Votre demande de rendez-vous'
@@ -182,10 +181,7 @@ class MailReader():
         return mail_messages
 
 
-def need_to_valid_url(url: str, successful_items) -> bool:
+def need_to_valid_url(url: str, email, successful_items):
-    # return True
-    # if len(successful_items) == 0:
-    #     return False
     print("url is :" + url)
     parts = url.split('/')
     id = parts[5]
@@ -193,16 +189,19 @@ def need_to_valid_url(url: str, successful_items) -> bool:
         for item in successful_items:
             if item.id == id:
                 if item.url_validated is not None:
-                    return not item.url_validated
+                    return item, not item.url_validated
                 else:
                     # if url_validated is None
                     if item.url_validated is not None:
-                        return not item.url_validated
+                        return item, not item.url_validated
-                    return True
+                    return item, True
-        return True
+        for item in successful_items:
+            if item.mail == email:
+                return item, True
+        return None, True
     else:
         print("id not valid:{}".format(id))
-        return False
+        return None, False
 
 
 def need_to_check_email(mail: str, successful_items) -> bool:
@@ -257,20 +256,16 @@ def read_mails():
         # get ip_country info
         _refreshed_successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
         _all_contact_list = MONGO_STORE_MANAGER.get_all_contacts_to_book()
+        contact_serial_map_list = MONGO_STORE_MANAGER.get_all_contact_serial_list()
         with ThreadPoolExecutor(max_workers=10) as executor:
             for mail in mails_messages:
-                # if mail.isImapClient:
-                #     match = re.search(PART_VALIDATION_URL_REGEX, mail.body.replace("\n", ""))
-                # else:
                 match = re.search(VALIDATION_URL_REGEX, mail.body)
                 if match:
-                    # url_to_validate = match.group(0)
-                    # if mail.isImapClient:
-                    #     url = "https://rendezvousparis.hermes.com/" + url_to_validate.replace("3D", "")
-                    # else:
                     url = match.group(0)
-                    if need_to_valid_url(url, _refreshed_successful_items):
+                    _item, is_need_to = need_to_valid_url(url, mail.to_address, _refreshed_successful_items)
-                        MONGO_STORE_MANAGER.save_links_to_validate(url, mail.to_address, _all_contact_list)
+                    if is_need_to:
+                        MONGO_STORE_MANAGER.save_links_to_validate(url, mail.to_address, _all_contact_list, _item,
+                                                                   contact_serial_map_list)
                         # url_validator = LinkValidator(url)
                         print("need to validate url: " + url)
                         # executor.submit(url_validator.start_page, params.get_proxy(ProxyType.OXYLABS), False)

src/mail/mail_reader_all_contacts.py

@@ -12,6 +12,7 @@ from imapclient import IMAPClient
 from src.db.mongo_manager import MONGO_STORE_MANAGER
 from src.logs.AppLogging import init_logger
 from src.mail.mail_constants import DOMAIN_HOTMAIL, create_imap
+from src.mail.mail_reader import need_to_valid_url
 from src.pojo.mail.mail_pojo import MailPojo
 from src.utils.timeutiles import is_time_between
 
@@ -200,25 +201,25 @@ class MailReader():
         return mail_messages
 
 
-def need_to_valid_url(url: str, successful_items) -> bool:
+# def need_to_valid_url(url: str, successful_items) -> bool:
+#     # return True
+#     # if len(successful_items) == 0:
+#     #     return False
+#     print("url is :" + url)
+#     parts = url.split('/')
+#     id = parts[5]
+#     if len(id) == 6:
+#         for item in successful_items:
+#             if item.id == id:
+#                 if item.url_validated is not None:
+#                     return not item.url_validated
+#                 else:
+#                     # if url_validated is None
 #                     return True
-    # if len(successful_items) == 0:
+#         return True
+#     else:
+#         print("id not valid:{}".format(id))
 #         return False
-    print("url is :" + url)
-    parts = url.split('/')
-    id = parts[5]
-    if len(id) == 6:
-        for item in successful_items:
-            if item.id == id:
-                if item.url_validated is not None:
-                    return not item.url_validated
-                else:
-                    # if url_validated is None
-                    return True
-        return True
-    else:
-        print("id not valid:{}".format(id))
-        return False
 
 
 def need_to_check_email(mail: str, successful_items) -> bool:
@@ -281,6 +282,8 @@ def read_all_mails(contact_to_book_list=None):
                     executor.submit(mail_reader.read_emails, mails_messages)
         # 在读邮件时候，可能会有其他的约会提交或者约会的链接确认，所以需要刷新一下成功的列表
         _refreshed_successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+        contact_serial_map_list = MONGO_STORE_MANAGER.get_all_contact_serial_list()
+
         with ThreadPoolExecutor(max_workers=10) as executor:
             for mail in mails_messages:
                 match = re.search(VALIDATION_URL_REGEX, mail.body)
@@ -290,10 +293,12 @@ def read_all_mails(contact_to_book_list=None):
                     #     url = "https://rendezvousparis.hermes.com/" + url_to_validate.replace("3D", "")
                     # else:
                     url = match.group(0)
-                    if need_to_valid_url(url, _refreshed_successful_items):
+                    _item, is_need_to = need_to_valid_url(url, mail.to_address, _refreshed_successful_items)
+                    if is_need_to:
                         print("need to validate url: " + url)
                         MONGO_STORE_MANAGER.save_links_to_validate(url, mail.to_address,
-                                                                   _all_contact_list=contact_to_book_list)
+                                                                   _all_contact_list=contact_to_book_list, _item=_item,
+                                                                   contact_serial_map_list=contact_serial_map_list)
                     else:
                         print("do not need to click url --> {}".format(mail.mail_address))
 

src/person_name/contact_manager.py

@@ -86,7 +86,7 @@ def write_new_contacts_to_excel(valid_contacts: list, file_name=str(datetime.dat
 
 
 def generate_valid_contact_list_for_day(segment_number=1):
-    _collection_name = "2024-09-23"
+    _collection_name = "2024-12-07"
     _valid_contact_list = MONGO_STORE_MANAGER.get_all_successful_items_for_one_day(_collection_name)
     _all_contacts = MONGO_STORE_MANAGER.get_all_contacts_to_book()
     _contact_to_save = []
@@ -204,14 +204,14 @@ if __name__ == '__main__':
     # contacts_to_book = upload_contacts_list()
     # MONGO_STORE_MANAGER.upload_contact_list(contacts_to_book)
     # print("start at {}".format(datetime.datetime.now()))
-    # generate_valid_contact_list_for_day(segment_number=2)
+    generate_valid_contact_list_for_day(segment_number=2)
     # generate_contact_from_mail_list("/Users/lpan/Downloads/邮箱及密码.xlsx")
     # print("end at {}".format(datetime.datetime.now()))
     # update_contact_list_not_received_mail()
     # generate_valid_contact_list_for_day(segment_number=2)
     # get_old_validated_contact_list()
     # print("end at {}".format(datetime.datetime.now()))
-    generate_all_contact_list()
+    # generate_all_contact_list()
     # merge_contact_list_files(
     #     ["/Users/lpan/Desktop/contact_list_all_old_not_used_contact.xlsx",
     #      "/Users/lpan/Desktop/contact_list_2024-06-26.xlsx"])

src/pojo/ReserveResultPojo.py

@@ -39,6 +39,7 @@ class ReserveResultPojo:
     validated_at = None
     ip_address = None
     ua = ""
+    model = ""
     serial = ""
 
     def __hash__(self):
@@ -109,9 +110,10 @@ class ReserveResultPojo:
         if 'message' in source:
             message = source['message']
             result.message = message
-        if 'source' in source:
+        if 'source_from' in source:
-            source_from = source['source']
+            source_from = source['source_from']
             result.source_from = source_from
+            result.model = source_from
         if 'sim_position' in source:
             sim_position = source['sim_position']
             result.sim_position = sim_position

src/pojo/contact_pojo.py

@@ -16,6 +16,7 @@ class ContactPojo:
     isp: str = None
     ua: str = ""
     serial: str = ""
+    model: str = ""
 
     def __repr__(self):
         return "phone:{}, passport:{}, last_name:{}, first_name:{}, mail:{}, store:{}, ip_country:{},isp:{}".format(
@@ -79,5 +80,8 @@ class ContactPojo:
         if source.get('serial'):
             serial = source['serial']
             result.serial = serial
+        if source.get('source_from'):
+            model = source['source_from']
+            result.model = model
         result.ua = ua
         return result

src/utils/contacts/check_all_collections.py

@@ -1,6 +1,8 @@
+from pathlib import Path
+
 from src.db.mongo_manager import MONGO_STORE_MANAGER
-from src.person_name.contact_manager import write_new_contacts_to_excel
 from src.pojo.contact_pojo import ContactPojo
+from src.utils.excel_reader import read_contacts
 
 DOMAIN_TO_INCLUDE = ["aol.com", "yahoo.com", "gmx.de", "gmx.net", "inbox.lv"]
 
@@ -32,11 +34,11 @@ def get_all_not_used_mails():
     return not_used_emails
 
 
-def get_all_old_contact_set():
+def get_all_old_contact_set(start_with="2024"):
     all_collection_list = get_all_collections()
     collection_to_check = []
     for col in all_collection_list:
-        if col.startswith('20'):
+        if col.startswith(start_with):
             collection_to_check.append(col)
     items_set = set()
     for collection in collection_to_check:
@@ -45,6 +47,15 @@ def get_all_old_contact_set():
     return items_set
 
 
+def get_all_contact_with_source_from():
+    _list_to_check = get_all_old_contact_set()
+    _to_return = []
+    for contact in _list_to_check:
+        if contact.serial is not None and len(contact.serial) > 0:
+            _to_return.append(contact)
+    return _to_return
+
+
 def generate_not_used_contact_list():
     not_used_mail_list = get_all_not_used_mails()
     all_old_contact_set = get_all_old_contact_set()
@@ -63,6 +74,12 @@ def generate_not_used_contact_list():
     return contact_list
 
 
+def upload_to_collection():
+    _contacts = read_contacts(str(Path.home()) + "/Desktop/contact_list_2024-11-05.xlsx")
+    MONGO_STORE_MANAGER.upload_contact_list_to_collection(_contacts,"CONTACT_LIST_SERIAL_MAP")
+
+
 if __name__ == '__main__':
-    contact_to_save = generate_not_used_contact_list()
+    upload_to_collection()
-    write_new_contacts_to_excel(valid_contacts=contact_to_save, file_name="all_old_not_used_contact")
+    # contact_to_save = get_all_contact_with_source_from()
+    # write_new_contacts_to_excel(valid_contacts=contact_to_save, file_name="contacts_with_source")