diff --git a/requirements.txt b/requirements.txt index eb0105a..85f4fcc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ SpeechRecognition==3.8.1 pymongo==4.1.1 wget==3.2.0 pocketsphinx==0.1.15 -oci~=2.54.1 XlsxWriter~=3.0.3 boto3~=1.21.13 openpyxl==3.0.9 diff --git a/src/db/DbManager.py b/src/db/DbManager.py deleted file mode 100644 index 019cb7d..0000000 --- a/src/db/DbManager.py +++ /dev/null @@ -1,105 +0,0 @@ -import datetime -from typing import Union - -import firebase_admin -from firebase_admin import credentials, firestore - -from src import params, config -from src.pojo.ReserveResultPojo import ReserveResultPojo -from src.pojo.ResultEnum import ResultEnum -from src.pojo.SimInfoPojo import SimInfoPojo -from src.pojo.contact_pojo import ContactPojo - -ERROR_COLLECTION_NAME = "error_items" -CONTACT_COLLECTION_NAME = "contact_list" -MAIL_COLLECTION_NAME = "mail_list" -SIM_INFOS = "sim_infos" -TIMEOUT = "timeout_items" - - -class DataManager: - batch_size = 20 - - def __init__(self): - cred = credentials.Certificate(config.FIREBASE_CONFIG_FILE) - self._app = firebase_admin.initialize_app(cred) - self._db = firestore.client() - - def get_all_sim_infos(self): - params.oracle_log_sender.send_read_db_event("get_all_sim_infos") - sim_info_collection = self._db.collection(SIM_INFOS) - return sim_info_collection - - def get_all_successful_items(self): - params.oracle_log_sender.send_read_db_event("get_all_successful_items") - return self.get_all_successful_items_for_day(str(datetime.date.today()), None) - - def get_all_successful_items_for_day(self, day, source_from: Union[str, None]): - params.oracle_log_sender.send_read_db_event("get_all_successful_items_for_day for {}".format(day)) - doc_ref = self._db.collection(day) - if source_from is not None: - doc_ref.where(u'source_from', u'==', source_from) - return doc_ref - - def get_all_accepted_items_for_day(self, day): - params.oracle_log_sender.send_read_db_event("get_all_successful_items_for_day for {}".format(day)) - doc_ref = self._db.collection(day) - accepted_ref = doc_ref.where(u'accepted', u'==', "ACCEPTED") - return accepted_ref - - def get_successful_item_for_day_by_status(self, day, status: ResultEnum): - params.oracle_log_sender.send_read_db_event("get_successful_item_for_day_by_status for {}".format(day)) - doc_ref = self._db.collection(day) - if status is not None: - doc_ref.where(u'accepted', u'==', status.value) - return doc_ref - - def save_sim_info(self, sim_info: SimInfoPojo): - params.oracle_log_sender.send_read_db_event("save_sim_info") - doc_ref = self._db.collection(SIM_INFOS).document(sim_info.phone) - doc_ref.set(sim_info.to_firestore_dict()) - - def save(self, result: ReserveResultPojo): - id = result.url.split("/")[-1] - result.id = id - collection_name = str(datetime.date.today()) - doc_ref = self._db.collection(collection_name).document(result.id) - doc_ref.set(result.to_firestore_dict()) - - def find_appointment_detail_via_phone(self, day, phone) -> ReserveResultPojo: - params.oracle_log_sender.send_read_db_event("find_appointment_detail_via_phone") - doc_ref = self._db.collection(day) - results = doc_ref.where(u'phone', u'==', phone).stream() - result_list = [] - for result in results: - result_list.append(ReserveResultPojo.from_firestore_dict(result.to_dict())) - if len(result_list) > 0: - return result_list[0] - - def save_timeout_contact(self, contact: ContactPojo): - doc_ref = self._db.collection(TIMEOUT).document(str(contact.phone)) - doc_ref.set(contact.to_firestore_dict()) - - def _delete_collection(self, coll_ref, batch_size): - params.oracle_log_sender.send_read_db_event("_delete_collection") - docs = coll_ref.limit(batch_size).stream() - deleted = 0 - - for doc in docs: - print(f'Deleting doc {doc.id} => {doc.to_dict()}') - doc.reference.delete() - deleted = deleted + 1 - - if deleted >= batch_size: - return self._delete_collection(coll_ref, batch_size) - - # 删除数据库中所有的sim卡信息 - def clear_all_sim_info(self): - params.oracle_log_sender.send_read_db_event("clear_all_sim_info") - coll_ref = self._db.collection(SIM_INFOS) - self._delete_collection(coll_ref, self.batch_size) - - def read_contacts_from_db(self) -> list: - params.oracle_log_sender.send_read_db_event("read_contacts_from_db") - contact_collection = self._db.collection(CONTACT_COLLECTION_NAME) - return contact_collection diff --git a/src/definitions.py b/src/definitions.py index f0b072c..9af3acd 100644 --- a/src/definitions.py +++ b/src/definitions.py @@ -1,7 +1,3 @@ LOG_SUBJECT_EVENT = "EVENT" -TYPE_EVENT_CHECK_RESULTS = "EVENT_CHECK_RESULTS" - -from src.db.DbManager import DataManager - -firebase_store_manager = DataManager() +TYPE_EVENT_CHECK_RESULTS = "EVENT_CHECK_RESULTS" \ No newline at end of file diff --git a/src/logs/LogSender.py b/src/logs/LogSender.py deleted file mode 100644 index e68e1b0..0000000 --- a/src/logs/LogSender.py +++ /dev/null @@ -1,143 +0,0 @@ -import datetime -import logging -import sys - -import oci -from oci.loggingingestion import LoggingClient -from oci.loggingingestion.models import PutLogsDetails, LogEntryBatch, LogEntry - -from src import config -from src.logs.AppLogging import init_logger -from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType -from src.pojo.contact_pojo import ContactPojo - -LOG_SUBJECT_EVENT = "EVENT" -LOG_SUBJECT_SMS = "SMS" -SUBJECT_SIM_INFO = "sim_card" -# Log type -TYPE_EVENT_CHECK_RESULTS = "EVENT_CHECK_RESULTS" -TYPE_EVENT_READ_DB = "EVENT_READ_DB" -TYPE_EVENT_RESET_ALL_SIM_CARDS = "EVENT_RESET_ALL_SIM_CARDS" -TYPE_EVENT_CHANGE_SLOT = "EVENT_CHANGE_SLOT" -TYPE_SMS_RECEIVED = "TYPE_SMS_RECEIVED" -LOG_ERROR = "ERROR" -LOG_TYPE_INFO = "INFO" -LOG_ERROR_TYPE_DOUBLE_DATA = "DOUBLE_REQUEST_FOR_SAME_DATA" -LOG_ERROR_TOO_MANY_REQUEST_TODAY = "TOO_MANY_REQUEST_TODAY" -LOG_ERROR_CAPTCHA_ERROR_MESSAGE = "CAPTCHA_ERROR" -LOG_SUBJECT_ERROR = "ERROR" -LOG_APPOINTMENT_ERROR = "APPOINTMENT_ERROR" -LOG_APPOINTMENT_TIMEOUT = "TIMEOUT" -LOG_APPOINTMENT_CONTACT_NOT_FOUND = "CONTACT_NOT_FOUND" -LOG_APPOINTMENT_SUCCESS = "SUCCESS" -URL_VALIDATION_SUCCESS = "URL_VALIDATION_SUCCESS" - -custom_retry_strategy = oci.retry.RetryStrategyBuilder( - # Make up to 10 service calls - max_attempts_check=True, - max_attempts=10, - - # Don't exceed a total of 600 seconds for all service calls - total_elapsed_time_check=True, - total_elapsed_time_seconds=600, - - # Wait 45 seconds between attempts - retry_max_wait_between_calls_seconds=45, - - # Use 2 seconds as the base number for doing sleep time calculations - retry_base_sleep_time_seconds=2, - - # Retry on certain service errors: - # - # - 5xx code received for the request - # - Any 429 (this is signified by the empty array in the retry config) - # - 400s where the code is QuotaExceeded or LimitExceeded - service_error_check=True, - service_error_retry_on_any_5xx=True, - service_error_retry_config={ - 400: ['QuotaExceeded', 'LimitExceeded'], - 429: [] - }, - - # Use exponential backoff and retry with full jitter, but on throttles use - # exponential backoff and retry with equal jitter - backoff_type=oci.retry.BACKOFF_FULL_JITTER_EQUAL_ON_THROTTLE_VALUE -).get_retry_strategy() - - -class LogSender: - def __init__(self): - self._config = oci.config.from_file("~/.oci/logger_config_appointment") - self._identity = oci.identity.IdentityClient(self._config) - self._loggingingestion_client = LoggingClient(self._config, timeout=60.0, retry_strategy=custom_retry_strategy) - - def send_double_data_error(self, contact: ContactPojo): - error_msg = contact.mail - self.send_log(msg=error_msg, type=LOG_ERROR_TYPE_DOUBLE_DATA, subject=LOG_SUBJECT_ERROR) - - def send_too_many_error(self, contact: ContactPojo): - error_msg = contact.mail - self.send_log(msg=error_msg, type=LOG_ERROR_TOO_MANY_REQUEST_TODAY, subject=LOG_SUBJECT_ERROR) - - def send_captcha_error(self, contact: ContactPojo): - error_msg = contact.mail - self.send_log(msg=error_msg, type=LOG_ERROR_CAPTCHA_ERROR_MESSAGE, subject=LOG_SUBJECT_ERROR) - - def send_appoint_result(self, result: ReserveResultPojo): - if result.type == PublishType.SUCCESS: - # get id - self.send_log(result.id, type=LOG_APPOINTMENT_SUCCESS) - else: - msg = "{}, email: {}".format(result.message, result.email) - self.send_log(msg, type=LOG_APPOINTMENT_ERROR) - - def send_url_validation_result(self): - self.send_log(msg='', type=URL_VALIDATION_SUCCESS) - - def send_error(self, msg: str): - self.send_log(msg=msg, type=LOG_ERROR) - - def send_read_db_event(self, msg: str): - self.send_log(msg=msg, type=TYPE_EVENT_READ_DB, subject=LOG_SUBJECT_EVENT) - - def send_log(self, msg: str, source=config.LOG_SOURCE, subject="appointment", type: str = "INFO"): - log_id = "ocid1.log.oc1.eu-frankfurt-1.amaaaaaas4ft22ya3ub6glkltqqbnmkxo3ui7xwq3dxtjd2scdhme4deyu2q" - response = self._loggingingestion_client.put_logs( - log_id=log_id, - put_logs_details=PutLogsDetails( - specversion="1.0", - log_entry_batches=[ - LogEntryBatch( - entries=[LogEntry( - data=msg, - id=log_id, - time=datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ"))], - source=source, - type=type, - defaultlogentrytime=datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ"), - subject=subject)]) - ) - return response - - def send_sms_reception_log(self, phone, sms_text, ccid): - msg = "from:{}, sms:{}, ccid:{}".format(phone, sms_text, ccid) - self.send_log(msg=msg, subject=LOG_SUBJECT_SMS, type=TYPE_SMS_RECEIVED) - - def send_timeout_log(self, contact: ContactPojo): - msg = "phone:{}, mail:{}".format(contact.phone, contact.mail) - self.send_log(msg, type=LOG_APPOINTMENT_TIMEOUT) - - def send_contact_not_found(self, msg: str): - self.send_log(msg, subject=SUBJECT_SIM_INFO, type=LOG_APPOINTMENT_CONTACT_NOT_FOUND) - - def send_wait_sms_log(self): - self.send_log("等待短信", subject=LOG_SUBJECT_EVENT, type=TYPE_SMS_RECEIVED) - - -if __name__ == '__main__': - init_logger() - logger = logging.getLogger() - logger.addHandler(logging.StreamHandler(stream=sys.stdout)) - log_sender = LogSender() - response = log_sender.send_log("test") - print(response) diff --git a/src/mail/mail_confirmation.py b/src/mail/mail_confirmation.py index c35bc85..4a6ad09 100755 --- a/src/mail/mail_confirmation.py +++ b/src/mail/mail_confirmation.py @@ -4,7 +4,6 @@ from builtins import list from concurrent.futures import ThreadPoolExecutor from email.header import decode_header from email.message import Message - from imapclient import IMAPClient from src.db.mirgration.migration_tools import migre_accepted_appointment @@ -146,12 +145,16 @@ def clean(text): def accept_appointment_found(accepted_result_list: list): _all_contact_list = MONGO_STORE_MANAGER.get_all_contact_to_book_list() + _all_register_account = MONGO_STORE_MANAGER.get_all_registered_users() mailer = Mailer() # sginal = SignalSender() print(accepted_result_list) for reserve in accepted_result_list: result = get_accepted_result_from(reserve, MONGO_STORE_MANAGER, _all_contact_list) - mailer.send_email(result, to_all=True) + for user in _all_register_account: + if user.mail == result.email: + result.account_password = user.password + mailer.send_email(result, to_all=False) MONGO_STORE_MANAGER.update_reserve_result(reserve.id, ResultEnum.ACCEPTED, reserve.message) # sginal.send_result(result) @@ -229,3 +232,5 @@ if __name__ == '__main__': _mail_list_today = find_confirmation_contacts_for_today() # print("size is {}".format(len(_mail_list_today))) find_confirmation_contacts_mail_list(_mail_list_today) + # _items = MONGO_STORE_MANAGER.get_all_successful_items_for_day() + # accept_appointment_found([random.choice(_items)]) diff --git a/src/mail/mail_constants.py b/src/mail/mail_constants.py index 7e88937..d18710a 100755 --- a/src/mail/mail_constants.py +++ b/src/mail/mail_constants.py @@ -20,6 +20,7 @@ DOMAIN_GMX_CH = "gmx.ch" DOMAIN_ONET = "onet.pl" DOMAIN_NAVER = "naver.com" DOMAIN_INBOX_LV = "inbox.lv" +DOMAIN_GMX_DE = "gmx.de" DOMAIN_PISS_MAIL = "pissmail.com" DOMAIN_INCEL_EMAIL = "incel.email" @@ -34,6 +35,7 @@ IMAP_SERVER_163 = "imap.163.com" IMAP_SERVER_SINA = "imap.sina.com" YAHOO_IMAP_SERVER = "imap.mail.yahoo.com" HOTMAIL_IMAP_SERVER = "outlook.office365.com" + RAMBLER_IMAP_SERVER = "imap.rambler.ru" ALICE_IMAP_SERVER = "in.alice.it" TIME_IT_SERVER = "imap.tim.it" @@ -43,6 +45,7 @@ BTVM_NE_JP = "imap.btvm.ne.jp" SEREVER_GMAIL = "imap.gmail.com" SERVER_IMAGE_ONET = "imap.poczta.onet.pl" SERVER_GMX = "imap.gmx.com" +SERVER_GMX_NET ="imap.gmx.net" SERVER_PISS_MAIL = "mail.pissmail.com" INBOX_LV = "mail.inbox.lv" @@ -97,6 +100,8 @@ def create_imap(login: str): imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143) elif DOMAIN_NAVER in login: imap = imaplib.IMAP4_SSL(NAVER_SERVER, port=993) + elif DOMAIN_GMX_DE in login: + imap = imaplib.IMAP4_SSL(SERVER_GMX_NET, port=993) elif DOMAIN_INBOX_LV in login: imap = imaplib.IMAP4_SSL(INBOX_LV, port=993) elif DOMAIN_PISS_MAIL in login or DOMAIN_CHILD_PIZZA in login or DOMAIN_DMC_CHAT in login or DOMAIN_GENOCIDE_FUN in login or DOMAIN_HATESJE_WS in login or DOMAIN_INCEL_EMAIL in login or DOMAIN_SHITPOSTING_EXPERT in login: diff --git a/src/notification/AcceptedResultPojo.py b/src/notification/AcceptedResultPojo.py index e916982..802cf29 100644 --- a/src/notification/AcceptedResultPojo.py +++ b/src/notification/AcceptedResultPojo.py @@ -24,6 +24,7 @@ class AcceptedResultPojo: self.created_at = created_at self.validated_at = validated_at self.mail_password = "" + self.account_password = "" def get_accepted_result_from(reserve_pojo: ReserveResultPojo, mongo_db_manager: MongoDbManager, diff --git a/src/params.py b/src/params.py index ee24cdf..0fb38d0 100644 --- a/src/params.py +++ b/src/params.py @@ -1,11 +1,7 @@ import random import string - -from src.logs.LogSender import LogSender from src.proxy.proxy_type import ProxyType -oracle_log_sender = LogSender() - # proxy PROXY_SERVER = "http://gw.ntnt.io:5959" PROXY_PASSWORD = "94sY7zwBG13i" diff --git a/src/person_name/contact_manager.py b/src/person_name/contact_manager.py index 302d2fc..0980d4e 100755 --- a/src/person_name/contact_manager.py +++ b/src/person_name/contact_manager.py @@ -8,7 +8,7 @@ from src.utils.excel_reader import read_contacts, fr_phone_number_prefix, get_ra def upload_contacts_list(): - _contacts_to_book = read_contacts("/Users/panlei/Desktop/contact_list_2024-03-30.xlsx") + _contacts_to_book = read_contacts("/Users/lpan/Desktop/contact_list_2024-04-15.xlsx") return _contacts_to_book @@ -19,7 +19,7 @@ def fix_phone_number_format(file_path): if _contact.phone[0:2] not in fr_phone_number_prefix: print(_contact) _contact.phone = get_random_fr_phone_numbers() - write_new_contacts_to_excel(_contact_list, file_name="25_03_to_test") + write_new_contacts_to_excel(_contact_list, file_name="09_04_to_test") def write_new_contacts_to_excel(valid_contacts: list, file_name=str(datetime.date.today())): @@ -46,7 +46,7 @@ def write_new_contacts_to_excel(valid_contacts: list, file_name=str(datetime.dat workbook.close() -def generate_valid_contact_list_for_day(): +def generate_valid_contact_list_for_day(sement_number = 1): _valid_contact_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day() _all_contacts = MONGO_STORE_MANAGER.get_all_contacts_to_book() _contact_to_save = [] @@ -62,8 +62,15 @@ def generate_valid_contact_list_for_day(): _contact.first_name = _true_contact.first_name if _contact.url_validated: - _contact_to_save.append(_contact) + if _contact.last_name is not None and len(_contact.last_name) > 0: + _contact_to_save.append(_contact) write_new_contacts_to_excel(_contact_to_save) + for i in range(0, sement_number): + _step = int(len(_contact_to_save) / sement_number) + _sublist = _contact_to_save[i * _step:_step * (i + 1)] + _file_name = str(datetime.date.today()) +"_"+ str(i+1) + write_new_contacts_to_excel(_sublist,file_name=_file_name) + def merge_contact_list_files(file_list: list, final_file_name="merged_contact_list"): @@ -90,14 +97,11 @@ def generate_all_contact_list(): if __name__ == '__main__': # contacts_to_book = upload_contacts_list() # MONGO_STORE_MANAGER.upload_contact_list(contacts_to_book) - # generate_valid_contact_list_for_day() + generate_valid_contact_list_for_day(sement_number=3) # generate_all_contact_list() - merge_contact_list_files( - ["/Users/panlei/Desktop/contact_list_2024-03-29.xlsx", - "/Users/panlei/Desktop/contact_list_2024-03-27.xlsx", - "/Users/panlei/Desktop/contact_list_2024-03-28.xlsx", - "/Users/panlei/Desktop/contact_list_2024-03-26.xlsx", - "/Users/panlei/Desktop/contact_list_2024-03-25.xlsx", - "/Users/panlei/Desktop/contact_list_2024-03-23.xlsx", - "/Users/panlei/Desktop/contact_list_2024-03-22.xlsx"]) - # fix_phone_number_format("/Users/panlei/Desktop/25_03_to_test.xlsx") + # merge_contact_list_files( + # ["/Users/lpan/Desktop/contact_list_2024-04-10.xlsx", + # "/Users/lpan/Desktop/contact_list_2024-04-09.xlsx", + # "/Users/lpan/Desktop/contact_list_2024-04-06.xlsx", + # "/Users/lpan/Desktop/contact_list_2024-04-04.xlsx"]) + # fix_phone_number_format("/Users/lpan/Desktop/09_04_to_test.xlsx") diff --git a/src/templates/appointment_results.html b/src/templates/appointment_results.html index 9f45d55..441264d 100644 --- a/src/templates/appointment_results.html +++ b/src/templates/appointment_results.html @@ -2,7 +2,7 @@

电话: ${result.phone}
-
邮件: ${result.email}, 密码: ${result.mail_password} (用网易邮箱大师登录) +
邮件: ${result.email}, 密码: ${result.mail_password} (用QQ邮箱或者网易邮箱大师登录)

护照: ${result.passport}
@@ -10,6 +10,8 @@


邮件: ${result.message} + +
帐户密码: ${result.account_password} (内部使用,转发的时候要删除)