diff --git a/.gitignore b/.gitignore index c81b0ff..e9e658d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,10 +4,13 @@ pojo/__pycache__/ appointment.log utils/*.log .~contact.xlsx +*.xlsx +*.so .env ./venv out lib +*.txt appointment_*.log ./build venv diff --git a/src/db/mongo_manager.py b/src/db/mongo_manager.py index 6037bae..3b60077 100755 --- a/src/db/mongo_manager.py +++ b/src/db/mongo_manager.py @@ -14,6 +14,7 @@ from src.pojo.mail.mail_pojo import MailAddress from src.pojo.users.regisered_user_pojo import RegisteredUserPojo MONGO_DB_URL = "mongo.lpaconsulting.fr" +# MONGO_DB_URL = "clusterfr.zlmpq.mongodb.net" CAPTCHA_ERROR_COLLECTION_PREFIX = "CAPTCHA_ERROR_" BLACK_LIST = "BLACK_LIST" ACCEPTED_APPOINTMENT_LIST = "ACCEPTED_APPOINTMENT_LIST" diff --git a/src/mail/get_invoices.py b/src/mail/get_invoices.py index ae2f4fe..a0a9a93 100644 --- a/src/mail/get_invoices.py +++ b/src/mail/get_invoices.py @@ -10,14 +10,14 @@ from builtins import list from src.db.mongo_manager import MONGO_STORE_MANAGER from src.logs.AppLogging import init_logger from src.mail.mail_constants import DOMAIN_163, DOMAIN_YAHOO, DOMAIN_SINA, IMAP_SERVER_163, YAHOO_IMAP_SERVER, \ - IMAP_SERVER_SINA, AOL_IMAP_SERVER + IMAP_SERVER_SINA, AOL_IMAP_SERVER, create_imap from src.pojo.accepted_appointment_pojo import AcceptedAppointmentPojo from src.pojo.mail.mail_pojo import MailPojo INVOICE_SUBJECT_fr = 'Votre facture' INVOICE_SUBJECT_EN = 'Your invoice' VALIDATION_URL_REGEX = """https:\/\/rendezvousparis.hermes.com\/client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+""" -HERMES_INVOICE_EMAIL = "no-reply@ereceipt.hermes.com" +HERMES_INVOICE_EMAIL = "no-reply@hermes.com" date_format = "%d-%b-%Y" # DD-Mon-YYYY e.g., 3-Mar-2014 @@ -33,20 +33,9 @@ class InvoiceGetter(): l = i.decode().split(' "/" ') print(l[0] + " = " + l[1]) - def create_imap(self): - # create an IMAP4 class with SSL - if DOMAIN_163 in self.login: - imap = imaplib.IMAP4_SSL(IMAP_SERVER_163) - elif DOMAIN_YAHOO in self.login: - imap = imaplib.IMAP4_SSL(YAHOO_IMAP_SERVER) - elif DOMAIN_SINA in self.login: - imap = imaplib.IMAP4_SSL(IMAP_SERVER_SINA) - else: - imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER) - return imap def read_emails(self, mails_messages: list) -> list: - imap = self.create_imap() + imap = create_imap(self.login) # authenticate type, dat = imap.login(self.login, self.password) @@ -128,7 +117,8 @@ def get_invoices(): # get email address mail_list = MONGO_STORE_MANAGER.get_destination_emails() mail_list_to_check = [] - yesterday = str(datetime.date.today() - datetime.timedelta(days=1)) + # yesterday = str(datetime.date.today() - datetime.timedelta(days=2)) + yesterday = str(datetime.date.today()) collection = MONGO_STORE_MANAGER.get_accepted_items_for_one_day(yesterday) for valid_appointment in collection: accepted_pojo = AcceptedAppointmentPojo.from_reserve(valid_appointment) diff --git a/src/mail/mail_address_validator.py b/src/mail/mail_address_validator.py index 443dc25..72d7e90 100755 --- a/src/mail/mail_address_validator.py +++ b/src/mail/mail_address_validator.py @@ -58,7 +58,7 @@ if __name__ == '__main__': # remove_invalid_email() mail_list = MONGO_STORE_MANAGER.get_destination_emails() - # excel_reader = ExcelHelper()ge + # excel_reader = ExcelHelper() # mail_list = excel_reader.read_mails_and_pwd(file_name="/Users/lpan/Downloads/hotmail_list.xlsx") # mail_address1 = MailAddress(mail="christinnyua@gmx.net", password="q1J1HHY2sXN") # mail_list = [mail_address1] diff --git a/src/mail/mail_confirmation.py b/src/mail/mail_confirmation.py index 9f0e15a..4bc064f 100755 --- a/src/mail/mail_confirmation.py +++ b/src/mail/mail_confirmation.py @@ -78,6 +78,7 @@ class MailConfirmationReader(): body = body + str(part.get_payload(decode=True).decode("utf-8")) elif part.get_content_type() == "text/plain": body = body + part.get_payload() + print("mail is {} and subject is {}".format(self.login, subject)) if CONFIRMATION_SUBJECT_FR in subject or CONFIRMATION_SUBJECT_EN in subject or "Votre_rendez-vous_est_confirm" in subject: mail = MailPojo(subject=subject, body=body, from_address=from_address) mail.isImapClient = True @@ -155,7 +156,7 @@ def accept_appointment_found(accepted_result_list: list): for user in _all_register_account: if user.mail == result.email: result.account_password = user.password - mailer.send_email(result, to_all=True) + mailer.send_email(result, to_all=False) MONGO_STORE_MANAGER.update_reserve_result(reserve.id, ResultEnum.ACCEPTED, reserve.message) # sginal.send_result(result) @@ -172,8 +173,10 @@ def find_confirmation_contacts_for_today(): for _item in _all_appointments_today: for _mail in _all_mail_list: if _mail.mail == _item.mail: - _mail_list_to_scan.append(_mail) + if _item.url_validated is True: + _mail_list_to_scan.append(_mail) break + print("Found {} emails to scan".format(len(_mail_list_to_scan))) return _mail_list_to_scan diff --git a/src/mail/mail_reader.py b/src/mail/mail_reader.py index 3bdc4da..d4c1dd1 100755 --- a/src/mail/mail_reader.py +++ b/src/mail/mail_reader.py @@ -24,7 +24,7 @@ HERMES_EMAIL = "no-reply@hermes.com" EMAIL_ADDRESS_REGEX = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b' date_format = "%d-%b-%Y" # DD-Mon-YYYY e.g., 3-Mar-2014 -REDIRECTION_MAILS = "chenpeijun@aol.com,hongjiang176@aol.com,ciyuexie@aol.com,rutger.62@aol.com,ciccidaniel@aol.com,armasgoodman@aol.com,wknd.gemerine@aol.com,rafmail1981@aol.com,tonovichivanenaki@aol.com,hetland.ari@aol.com,mateusiversen@aol.com,lacerdaraffaello@aol.com,anasida76@aol.com,liamolinari@aol.com,sen70zib@aol.com,mezeiderrick@aol.com,stanisl49avchic@aol.com,damcvrobaneuron@aol.com,suyzanna_fleona@aol.com,dxealing.dissa@aol.com,hogg.karen@aol.com,obocharovamarina@aol.com,buchholzjohann@aol.com,orn.cecchini@aol.com,percivaltorgersen@aol.com,candalgudrun@aol.com,filimonis.76@aol.com,bengann_100@aol.com,axelhanne@aol.com,tiffanylarochelle@aol.com,nicoleta.r@aol.com,eichenbaum.1963@aol.com,kotensasharev@aol.com,samognat32@aol.com,edem_headshot@aol.com,kozmakuzmich1960@aol.com,damonsvensson@aol.com,anders.riva@aol.com,caiminwei123@gmail.com,yulingguo086@gmail.com,yingxiaolu086@gmail.com,lijiazhen0035@gmail.com,fangp370@gmail.com,huangyayu10086@gmail.com,fuziyuan110@gmail.com,xinyingdu886@gmail.com,yasiaforever.1971@aol.com,lukaszfidalgo@aol.com,zaichi29@aol.com,prostotakitak.1974@aol.com,mo90nroe@aol.com,blonde.87@aol.com,dimidrol.1969@aol.com,duwei1998@gmx.com" +REDIRECTION_MAILS = "chenpeijun@aol.com,hongjiang176@aol.com,ciyuexie@aol.com,rutger.62@aol.com,ciccidaniel@aol.com,armasgoodman@aol.com,wknd.gemerine@aol.com,rafmail1981@aol.com,tonovichivanenaki@aol.com,hetland.ari@aol.com,mateusiversen@aol.com,anasida76@aol.com,liamolinari@aol.com,sen70zib@aol.com,mezeiderrick@aol.com,stanisl49avchic@aol.com,damcvrobaneuron@aol.com,suyzanna_fleona@aol.com,dxealing.dissa@aol.com,hogg.karen@aol.com,obocharovamarina@aol.com,buchholzjohann@aol.com,orn.cecchini@aol.com,percivaltorgersen@aol.com,candalgudrun@aol.com,filimonis.76@aol.com,bengann_100@aol.com,axelhanne@aol.com,tiffanylarochelle@aol.com,nicoleta.r@aol.com,eichenbaum.1963@aol.com,kotensasharev@aol.com,samognat32@aol.com,edem_headshot@aol.com,kozmakuzmich1960@aol.com,damonsvensson@aol.com,anders.riva@aol.com,yasiaforever.1971@aol.com,zaichi29@aol.com,prostotakitak.1974@aol.com,mo90nroe@aol.com,blonde.87@aol.com,dimidrol.1969@aol.com,duwei1998@gmx.com" def check_email_address(email): diff --git a/src/passport/Passport_MRZutile.py b/src/passport/Passport_MRZutile.py index b792236..726ac46 100755 --- a/src/passport/Passport_MRZutile.py +++ b/src/passport/Passport_MRZutile.py @@ -22,11 +22,11 @@ def decode(s: str) -> str: return bytes(res).decode('gb2312') -chinese_name = "君若韵" -first_name = "reyun" -last_name = "jun" -passport_number = "E33634603" -birth_day = "991215" +chinese_name = "瞿烈棠" +first_name = "lietang" +last_name = "ju" +passport_number = "E41195503" +birth_day = "921215" sex = "F" # sex = "M" expire_date = "270116" diff --git a/src/person_name/.~contact_list_2024-09-26.xlsx b/src/person_name/.~contact_list_2024-09-26.xlsx new file mode 100644 index 0000000..4c491c6 Binary files /dev/null and b/src/person_name/.~contact_list_2024-09-26.xlsx differ diff --git a/src/person_name/contact_manager.py b/src/person_name/contact_manager.py index 8f1ea94..31029be 100755 --- a/src/person_name/contact_manager.py +++ b/src/person_name/contact_manager.py @@ -1,17 +1,20 @@ import datetime import random +from pathlib import Path import xlsxwriter +# from src.person_name.cython_extract_methods import filter_already_validated_contacts, read_pinyin_list_from_file from src.db.mongo_manager import MONGO_STORE_MANAGER -from src.person_name.cython_extract_methods import filter_already_validated_contacts, read_pinyin_list_from_file from src.pojo.contact_pojo import ContactPojo from src.utils.contacts.generate_random_passport_id import get_random_passport_id_number from src.utils.excel_reader import read_contacts, fr_phone_number_prefix, get_random_fr_phone_numbers, ExcelHelper +DEFAULT_SERIAL = "47e7e36b" + def upload_contacts_list(): - _contacts_to_book = read_contacts("/Users/lpan/Desktop/contact_list_2024-09-16.xlsx") + _contacts_to_book = read_contacts(str(Path.home()) + "/Desktop/contact_list_2024-10-03.xlsx") return _contacts_to_book @@ -61,7 +64,7 @@ def write_new_contacts_to_excel(valid_contacts: list, file_name=str(datetime.dat col = 0 # Create a workbook and add a worksheet. workbook = xlsxwriter.Workbook('contact_list_{}.xlsx'.format(file_name)) - header_data = ['name', 'phone', 'passport', 'email', 'store', 'ip_country', 'ua'] + header_data = ['name', 'phone', 'passport', 'email', 'store', 'serial', 'ip_country', 'ua'] worksheet = workbook.add_worksheet() header_format = workbook.add_format({'bold': True}) @@ -75,14 +78,89 @@ def write_new_contacts_to_excel(valid_contacts: list, file_name=str(datetime.dat worksheet.write(row, col + 2, info.passport) worksheet.write(row, col + 3, info.mail) worksheet.write(row, col + 4, info.store) - worksheet.write(row, col + 5, info.ip_country) - worksheet.write(row, col + 6, info.ua) + worksheet.write(row, col + 5, info.serial) + worksheet.write(row, col + 6, info.ip_country) + worksheet.write(row, col + 7, info.ua) row += 1 workbook.close() def generate_valid_contact_list_for_day(segment_number=1): - _valid_contact_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day() + _collection_name = "2024-09-23" + _valid_contact_list = MONGO_STORE_MANAGER.get_all_successful_items_for_one_day(_collection_name) + _all_contacts = MONGO_STORE_MANAGER.get_all_contacts_to_book() + _contact_to_save = [] + for _contact in _valid_contact_list: + # _contact.store = "faubourg" + _contact.store = "random" + if _contact.last_name is None or len(_contact.last_name) == 0: + for _true_contact in _all_contacts: + if _true_contact.mail == _contact.mail: + _contact.last_name = _true_contact.last_name + _contact.phone = _true_contact.phone + _contact.passport = _true_contact.passport + _contact.first_name = _true_contact.first_name + + if len(_contact.serial) == 0: + _contact.serial = DEFAULT_SERIAL + if _contact.url_validated: + if _contact.last_name is not None and len(_contact.last_name) > 0: + _need_to_save = True + # remove the duplicated items + for _added_item in _contact_to_save: + if (_added_item.mail == _contact.mail + and _added_item.phone == _contact.phone + and _added_item.passport == _contact.passport + and _added_item.last_name == _contact.last_name + and _added_item.first_name == _contact.first_name): + _need_to_save = False + + if _need_to_save: + _contact_to_save.append(_contact) + _contact_to_save_list = _contact_to_save + write_new_contacts_to_excel(_contact_to_save_list, file_name=_collection_name) + for i in range(0, segment_number): + _step = int(len(_contact_to_save_list) / segment_number) + _sublist = _contact_to_save_list[i * _step:_step * (i + 1)] + _file_name = _collection_name + "_" + str(i + 1) + write_new_contacts_to_excel(_sublist, file_name=_file_name) + + +def merge_contact_list_files(file_list: list, final_file_name="merged_contact_list"): + _all_contact_list = [] + for file in file_list: + _all_contact_list.extend(read_contacts(file)) + for _con in _all_contact_list: + _con.store = "random" + print(len(_all_contact_list)) + _list_without_duplicate = list(set(_all_contact_list)) + print(len(_list_without_duplicate)) + write_new_contacts_to_excel(_list_without_duplicate, file_name=final_file_name) + + +def generate_all_contact_list(): + _all_contacts = MONGO_STORE_MANAGER.get_all_contacts_to_book() + random.shuffle(_all_contacts) + for _contact in _all_contacts: + _contact.store = "random" + write_new_contacts_to_excel(_all_contacts, file_name="all") + + +def write_to_black_list(contacts: list): + for contact in contacts: + MONGO_STORE_MANAGER.insert_blacklist_contact(contact) + + +def update_contact_list_not_received_mail(): + _contact_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day() + for _contact in _contact_list: + if not _contact.url_validated: + write_to_black_list([_contact]) + + +def get_old_validated_contact_list(remove_blacklisted_contact=True, _day_in_str="2024-09-06"): + _valid_contact_list = MONGO_STORE_MANAGER.get_all_successful_items_for_one_day(_day_in_str) + _domain_list_to_remove = ["firemail.de", "onet.pl", "yahoo.com", "gmx.com"] _all_contacts = MONGO_STORE_MANAGER.get_all_contacts_to_book() _contact_to_save = [] for _contact in _valid_contact_list: @@ -108,34 +186,17 @@ def generate_valid_contact_list_for_day(segment_number=1): and _added_item.first_name == _contact.first_name): _need_to_save = False if _need_to_save: - _contact_to_save.append(_contact) + # remove the contact if domain is not in the list + if _contact.mail.split("@")[1] not in _domain_list_to_remove: + _contact_to_save.append(_contact) + # remove blacklisted contact if needed + if remove_blacklisted_contact: + _blacklisted_contact = MONGO_STORE_MANAGER.get_blacklist_contacts() + _list_without_blacklisted = [contact for contact in _contact_to_save if + contact.mail not in [bl.mail for bl in _blacklisted_contact]] + write_new_contacts_to_excel(_list_without_blacklisted, file_name=_day_in_str + "_without_blacklisted") _contact_to_save_list = _contact_to_save - write_new_contacts_to_excel(_contact_to_save_list) - for i in range(0, segment_number): - _step = int(len(_contact_to_save_list) / segment_number) - _sublist = _contact_to_save_list[i * _step:_step * (i + 1)] - _file_name = str(datetime.date.today()) + "_" + str(i + 1) - write_new_contacts_to_excel(_sublist, file_name=_file_name) - - -def merge_contact_list_files(file_list: list, final_file_name="merged_contact_list"): - _all_contact_list = [] - for file in file_list: - _all_contact_list.extend(read_contacts(file)) - for _con in _all_contact_list: - _con.store = "random" - print(len(_all_contact_list)) - _list_without_duplicate = list(set(_all_contact_list)) - print(len(_list_without_duplicate)) - write_new_contacts_to_excel(_list_without_duplicate, file_name=final_file_name) - - -def generate_all_contact_list(): - _all_contacts = MONGO_STORE_MANAGER.get_all_contacts_to_book() - random.shuffle(_all_contacts) - for _contact in _all_contacts: - _contact.store = "random" - write_new_contacts_to_excel(_all_contacts, file_name="all") + write_new_contacts_to_excel(_contact_to_save_list, file_name=_day_in_str) # 把新的联系人存到网上 @@ -146,6 +207,10 @@ if __name__ == '__main__': # generate_valid_contact_list_for_day(segment_number=2) # generate_contact_from_mail_list("/Users/lpan/Downloads/邮箱及密码.xlsx") # print("end at {}".format(datetime.datetime.now())) + # update_contact_list_not_received_mail() + # generate_valid_contact_list_for_day(segment_number=2) + # get_old_validated_contact_list() + # print("end at {}".format(datetime.datetime.now())) generate_all_contact_list() # merge_contact_list_files( # ["/Users/lpan/Desktop/contact_list_all_old_not_used_contact.xlsx", diff --git a/src/pojo/ReserveResultPojo.py b/src/pojo/ReserveResultPojo.py index ba86c50..602f7bc 100644 --- a/src/pojo/ReserveResultPojo.py +++ b/src/pojo/ReserveResultPojo.py @@ -39,6 +39,7 @@ class ReserveResultPojo: validated_at = None ip_address = None ua = "" + serial = "" def __hash__(self): return hash("{}".format(self.mail)) @@ -102,6 +103,9 @@ class ReserveResultPojo: if 'accepted' in source: accepted = source['accepted'] result.accepted = accepted + if 'serial' in source: + serial = source['serial'] + result.serial = serial if 'message' in source: message = source['message'] result.message = message diff --git a/src/pojo/black_contact.py b/src/pojo/black_contact.py index 097f600..f028db8 100644 --- a/src/pojo/black_contact.py +++ b/src/pojo/black_contact.py @@ -9,9 +9,8 @@ class BlackContactPojo(ContactPojo): def __init__(self, phone_number: str, passport_number: str, last_name: str, first_name: str, mail: str, update_at_in_s: Union[float, None] = None, - ccid: str = "", - position: int = 0, ): - super().__init__(phone_number, passport_number, last_name, first_name, mail, ccid, position) + ): + super().__init__(phone_number, passport_number, last_name, first_name, mail, store="random") if update_at_in_s: self.update_at_in_s = update_at_in_s else: @@ -24,15 +23,13 @@ class BlackContactPojo(ContactPojo): @staticmethod def from_firestore_dict(source): - ccid = source['ccid'] phone = source['phone'] - position = source['position'] passport = source['passport'] email = source['mail'] last_name = source['last_name'] first_name = source['first_name'] update_at_in_s = source['update_at_in_s'] - result = BlackContactPojo(ccid=ccid, phone_number=phone, passport_number=passport, position=position, + result = BlackContactPojo(phone_number=phone, passport_number=passport, mail=email, update_at_in_s=update_at_in_s, last_name=last_name, first_name=first_name) return result diff --git a/src/pojo/contact_pojo.py b/src/pojo/contact_pojo.py index b66a114..e46a4f6 100644 --- a/src/pojo/contact_pojo.py +++ b/src/pojo/contact_pojo.py @@ -15,6 +15,7 @@ class ContactPojo: ip_address: str isp: str = None ua: str = "" + serial: str = "" def __repr__(self): return "phone:{}, passport:{}, last_name:{}, first_name:{}, mail:{}, store:{}, ip_country:{},isp:{}".format( @@ -45,7 +46,8 @@ class ContactPojo: u'mail': self.mail, u'store': self.store, u'ip_country': self.ip_country, - u'ua': self.ua + u'ua': self.ua, + u'serial': self.serial } return dest @@ -74,5 +76,8 @@ class ContactPojo: result = ContactPojo(phone_number=phone, passport_number=passport, mail=email, last_name=last_name, first_name=first_name, store=store) result.ip_country = ip_country + if source.get('serial'): + serial = source['serial'] + result.serial = serial result.ua = ua return result diff --git a/src/utils/contacts/generate_ip_with_contact.py b/src/utils/contacts/generate_ip_with_contact.py index d1dfad8..1940dc9 100644 --- a/src/utils/contacts/generate_ip_with_contact.py +++ b/src/utils/contacts/generate_ip_with_contact.py @@ -41,7 +41,7 @@ def write_contact_with_ip_info_to_file(contact_list): col = 0 # Create a workbook and add a worksheet. workbook = xlsxwriter.Workbook('ip_info_{}.xlsx'.format(len(contact_list))) - header_data = ['name', 'email', 'isp', 'ip_address', 'created_at'] + header_data = ['name', 'email', 'isp', 'ip_address', 'created_at', "passport"] worksheet = workbook.add_worksheet() header_format = workbook.add_format({'bold': True}) @@ -55,6 +55,7 @@ def write_contact_with_ip_info_to_file(contact_list): worksheet.write(row, col + 2, info.isp) worksheet.write(row, col + 3, info.ip_address) worksheet.write(row, col + 4, info.created_at) + worksheet.write(row, col + 5, info.passport) row += 1 workbook.close() diff --git a/src/utils/excel_reader.py b/src/utils/excel_reader.py index 9c784e8..b249193 100755 --- a/src/utils/excel_reader.py +++ b/src/utils/excel_reader.py @@ -58,6 +58,9 @@ def read_contacts(file_name) -> list: first_name=first_name, passport_number=contact_dict['passport'], mail=contact_dict['email'], store=store) + if contact_dict.get('serial') is not None: + serial = contact_dict['serial'] + contact.serial = serial contact.ip_country = ip_country contact.ua = ua contact_list.append(contact)