panleicim/appointment_tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of bitbucket.org:panleicim/appointment_tool

Browse Source
This commit is contained in:
PAN Lei
2024-04-16 22:50:49 +02:00
parent 0e9221db24 eba82e7498
commit 0685a66251
10 changed files with 35 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files
requirements.txt
-1
View File
@@ -8,7 +8,6 @@ SpeechRecognition==3.8.1
pymongo==4.1.1
wget==3.2.0
pocketsphinx==0.1.15
oci~=2.54.1
XlsxWriter~=3.0.3
boto3~=1.21.13
openpyxl==3.0.9
src/db/DbManager.py
-105
View File
@@ -1,105 +0,0 @@
import datetime
from typing import Union
import firebase_admin
from firebase_admin import credentials, firestore
from src import params, config
from src.pojo.ReserveResultPojo import ReserveResultPojo
from src.pojo.ResultEnum import ResultEnum
from src.pojo.SimInfoPojo import SimInfoPojo
from src.pojo.contact_pojo import ContactPojo
ERROR_COLLECTION_NAME = "error_items"
CONTACT_COLLECTION_NAME = "contact_list"
MAIL_COLLECTION_NAME = "mail_list"
SIM_INFOS = "sim_infos"
TIMEOUT = "timeout_items"
class DataManager:
batch_size = 20
def __init__(self):
cred = credentials.Certificate(config.FIREBASE_CONFIG_FILE)
self._app = firebase_admin.initialize_app(cred)
self._db = firestore.client()
def get_all_sim_infos(self):
params.oracle_log_sender.send_read_db_event("get_all_sim_infos")
sim_info_collection = self._db.collection(SIM_INFOS)
return sim_info_collection
def get_all_successful_items(self):
params.oracle_log_sender.send_read_db_event("get_all_successful_items")
return self.get_all_successful_items_for_day(str(datetime.date.today()), None)
def get_all_successful_items_for_day(self, day, source_from: Union[str, None]):
params.oracle_log_sender.send_read_db_event("get_all_successful_items_for_day for {}".format(day))
doc_ref = self._db.collection(day)
if source_from is not None:
doc_ref.where(u'source_from', u'==', source_from)
return doc_ref
def get_all_accepted_items_for_day(self, day):
params.oracle_log_sender.send_read_db_event("get_all_successful_items_for_day for {}".format(day))
doc_ref = self._db.collection(day)
accepted_ref = doc_ref.where(u'accepted', u'==', "ACCEPTED")
return accepted_ref
def get_successful_item_for_day_by_status(self, day, status: ResultEnum):
params.oracle_log_sender.send_read_db_event("get_successful_item_for_day_by_status for {}".format(day))
doc_ref = self._db.collection(day)
if status is not None:
doc_ref.where(u'accepted', u'==', status.value)
return doc_ref
def save_sim_info(self, sim_info: SimInfoPojo):
params.oracle_log_sender.send_read_db_event("save_sim_info")
doc_ref = self._db.collection(SIM_INFOS).document(sim_info.phone)
doc_ref.set(sim_info.to_firestore_dict())
def save(self, result: ReserveResultPojo):
id = result.url.split("/")[-1]
result.id = id
collection_name = str(datetime.date.today())
doc_ref = self._db.collection(collection_name).document(result.id)
doc_ref.set(result.to_firestore_dict())
def find_appointment_detail_via_phone(self, day, phone) -> ReserveResultPojo:
params.oracle_log_sender.send_read_db_event("find_appointment_detail_via_phone")
doc_ref = self._db.collection(day)
results = doc_ref.where(u'phone', u'==', phone).stream()
result_list = []
for result in results:
result_list.append(ReserveResultPojo.from_firestore_dict(result.to_dict()))
if len(result_list) > 0:
return result_list[0]
def save_timeout_contact(self, contact: ContactPojo):
doc_ref = self._db.collection(TIMEOUT).document(str(contact.phone))
doc_ref.set(contact.to_firestore_dict())
def _delete_collection(self, coll_ref, batch_size):
params.oracle_log_sender.send_read_db_event("_delete_collection")
docs = coll_ref.limit(batch_size).stream()
deleted = 0
for doc in docs:
print(f'Deleting doc {doc.id} => {doc.to_dict()}')
doc.reference.delete()
deleted = deleted + 1
if deleted >= batch_size:
return self._delete_collection(coll_ref, batch_size)
# 删除数据库中所有的sim卡信息
def clear_all_sim_info(self):
params.oracle_log_sender.send_read_db_event("clear_all_sim_info")
coll_ref = self._db.collection(SIM_INFOS)
self._delete_collection(coll_ref, self.batch_size)
def read_contacts_from_db(self) -> list:
params.oracle_log_sender.send_read_db_event("read_contacts_from_db")
contact_collection = self._db.collection(CONTACT_COLLECTION_NAME)
return contact_collection
src/definitions.py
-4
View File
@@ -1,7 +1,3 @@
LOG_SUBJECT_EVENT = "EVENT"
TYPE_EVENT_CHECK_RESULTS = "EVENT_CHECK_RESULTS"
from src.db.DbManager import DataManager
firebase_store_manager = DataManager()
src/logs/LogSender.py
-143
View File
@@ -1,143 +0,0 @@
import datetime
import logging
import sys
import oci
from oci.loggingingestion import LoggingClient
from oci.loggingingestion.models import PutLogsDetails, LogEntryBatch, LogEntry
from src import config
from src.logs.AppLogging import init_logger
from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType
from src.pojo.contact_pojo import ContactPojo
LOG_SUBJECT_EVENT = "EVENT"
LOG_SUBJECT_SMS = "SMS"
SUBJECT_SIM_INFO = "sim_card"
# Log type
TYPE_EVENT_CHECK_RESULTS = "EVENT_CHECK_RESULTS"
TYPE_EVENT_READ_DB = "EVENT_READ_DB"
TYPE_EVENT_RESET_ALL_SIM_CARDS = "EVENT_RESET_ALL_SIM_CARDS"
TYPE_EVENT_CHANGE_SLOT = "EVENT_CHANGE_SLOT"
TYPE_SMS_RECEIVED = "TYPE_SMS_RECEIVED"
LOG_ERROR = "ERROR"
LOG_TYPE_INFO = "INFO"
LOG_ERROR_TYPE_DOUBLE_DATA = "DOUBLE_REQUEST_FOR_SAME_DATA"
LOG_ERROR_TOO_MANY_REQUEST_TODAY = "TOO_MANY_REQUEST_TODAY"
LOG_ERROR_CAPTCHA_ERROR_MESSAGE = "CAPTCHA_ERROR"
LOG_SUBJECT_ERROR = "ERROR"
LOG_APPOINTMENT_ERROR = "APPOINTMENT_ERROR"
LOG_APPOINTMENT_TIMEOUT = "TIMEOUT"
LOG_APPOINTMENT_CONTACT_NOT_FOUND = "CONTACT_NOT_FOUND"
LOG_APPOINTMENT_SUCCESS = "SUCCESS"
URL_VALIDATION_SUCCESS = "URL_VALIDATION_SUCCESS"
custom_retry_strategy = oci.retry.RetryStrategyBuilder(
# Make up to 10 service calls
max_attempts_check=True,
max_attempts=10,
# Don't exceed a total of 600 seconds for all service calls
total_elapsed_time_check=True,
total_elapsed_time_seconds=600,
# Wait 45 seconds between attempts
retry_max_wait_between_calls_seconds=45,
# Use 2 seconds as the base number for doing sleep time calculations
retry_base_sleep_time_seconds=2,
# Retry on certain service errors:
#
# - 5xx code received for the request
# - Any 429 (this is signified by the empty array in the retry config)
# - 400s where the code is QuotaExceeded or LimitExceeded
service_error_check=True,
service_error_retry_on_any_5xx=True,
service_error_retry_config={
400: ['QuotaExceeded', 'LimitExceeded'],
429: []
},
# Use exponential backoff and retry with full jitter, but on throttles use
# exponential backoff and retry with equal jitter
backoff_type=oci.retry.BACKOFF_FULL_JITTER_EQUAL_ON_THROTTLE_VALUE
).get_retry_strategy()
class LogSender:
def __init__(self):
self._config = oci.config.from_file("~/.oci/logger_config_appointment")
self._identity = oci.identity.IdentityClient(self._config)
self._loggingingestion_client = LoggingClient(self._config, timeout=60.0, retry_strategy=custom_retry_strategy)
def send_double_data_error(self, contact: ContactPojo):
error_msg = contact.mail
self.send_log(msg=error_msg, type=LOG_ERROR_TYPE_DOUBLE_DATA, subject=LOG_SUBJECT_ERROR)
def send_too_many_error(self, contact: ContactPojo):
error_msg = contact.mail
self.send_log(msg=error_msg, type=LOG_ERROR_TOO_MANY_REQUEST_TODAY, subject=LOG_SUBJECT_ERROR)
def send_captcha_error(self, contact: ContactPojo):
error_msg = contact.mail
self.send_log(msg=error_msg, type=LOG_ERROR_CAPTCHA_ERROR_MESSAGE, subject=LOG_SUBJECT_ERROR)
def send_appoint_result(self, result: ReserveResultPojo):
if result.type == PublishType.SUCCESS:
# get id
self.send_log(result.id, type=LOG_APPOINTMENT_SUCCESS)
else:
msg = "{}, email: {}".format(result.message, result.email)
self.send_log(msg, type=LOG_APPOINTMENT_ERROR)
def send_url_validation_result(self):
self.send_log(msg='', type=URL_VALIDATION_SUCCESS)
def send_error(self, msg: str):
self.send_log(msg=msg, type=LOG_ERROR)
def send_read_db_event(self, msg: str):
self.send_log(msg=msg, type=TYPE_EVENT_READ_DB, subject=LOG_SUBJECT_EVENT)
def send_log(self, msg: str, source=config.LOG_SOURCE, subject="appointment", type: str = "INFO"):
log_id = "ocid1.log.oc1.eu-frankfurt-1.amaaaaaas4ft22ya3ub6glkltqqbnmkxo3ui7xwq3dxtjd2scdhme4deyu2q"
response = self._loggingingestion_client.put_logs(
log_id=log_id,
put_logs_details=PutLogsDetails(
specversion="1.0",
log_entry_batches=[
LogEntryBatch(
entries=[LogEntry(
data=msg,
id=log_id,
time=datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ"))],
source=source,
type=type,
defaultlogentrytime=datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
subject=subject)])
)
return response
def send_sms_reception_log(self, phone, sms_text, ccid):
msg = "from:{}, sms:{}, ccid:{}".format(phone, sms_text, ccid)
self.send_log(msg=msg, subject=LOG_SUBJECT_SMS, type=TYPE_SMS_RECEIVED)
def send_timeout_log(self, contact: ContactPojo):
msg = "phone:{}, mail:{}".format(contact.phone, contact.mail)
self.send_log(msg, type=LOG_APPOINTMENT_TIMEOUT)
def send_contact_not_found(self, msg: str):
self.send_log(msg, subject=SUBJECT_SIM_INFO, type=LOG_APPOINTMENT_CONTACT_NOT_FOUND)
def send_wait_sms_log(self):
self.send_log("等待短信", subject=LOG_SUBJECT_EVENT, type=TYPE_SMS_RECEIVED)
if __name__ == '__main__':
init_logger()
logger = logging.getLogger()
logger.addHandler(logging.StreamHandler(stream=sys.stdout))
log_sender = LogSender()
response = log_sender.send_log("test")
print(response)
src/mail/mail_confirmation.py
+7 -2
View File
@@ -4,7 +4,6 @@ from builtins import list
from concurrent.futures import ThreadPoolExecutor
from email.header import decode_header
from email.message import Message
from imapclient import IMAPClient
from src.db.mirgration.migration_tools import migre_accepted_appointment
@@ -146,12 +145,16 @@ def clean(text):
def accept_appointment_found(accepted_result_list: list):
_all_contact_list = MONGO_STORE_MANAGER.get_all_contact_to_book_list()
_all_register_account = MONGO_STORE_MANAGER.get_all_registered_users()
mailer = Mailer()
# sginal = SignalSender()
print(accepted_result_list)
for reserve in accepted_result_list:
result = get_accepted_result_from(reserve, MONGO_STORE_MANAGER, _all_contact_list)
mailer.send_email(result, to_all=True)
for user in _all_register_account:
if user.mail == result.email:
result.account_password = user.password
mailer.send_email(result, to_all=False)
MONGO_STORE_MANAGER.update_reserve_result(reserve.id, ResultEnum.ACCEPTED, reserve.message)
# sginal.send_result(result)
@@ -229,3 +232,5 @@ if __name__ == '__main__':
_mail_list_today = find_confirmation_contacts_for_today()
# print("size is {}".format(len(_mail_list_today)))
find_confirmation_contacts_mail_list(_mail_list_today)
# _items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
# accept_appointment_found([random.choice(_items)])
src/mail/mail_constants.py
+5
View File
@@ -20,6 +20,7 @@ DOMAIN_GMX_CH = "gmx.ch"
DOMAIN_ONET = "onet.pl"
DOMAIN_NAVER = "naver.com"
DOMAIN_INBOX_LV = "inbox.lv"
DOMAIN_GMX_DE = "gmx.de"
DOMAIN_PISS_MAIL = "pissmail.com"
DOMAIN_INCEL_EMAIL = "incel.email"
@@ -34,6 +35,7 @@ IMAP_SERVER_163 = "imap.163.com"
IMAP_SERVER_SINA = "imap.sina.com"
YAHOO_IMAP_SERVER = "imap.mail.yahoo.com"
HOTMAIL_IMAP_SERVER = "outlook.office365.com"
RAMBLER_IMAP_SERVER = "imap.rambler.ru"
ALICE_IMAP_SERVER = "in.alice.it"
TIME_IT_SERVER = "imap.tim.it"
@@ -43,6 +45,7 @@ BTVM_NE_JP = "imap.btvm.ne.jp"
SEREVER_GMAIL = "imap.gmail.com"
SERVER_IMAGE_ONET = "imap.poczta.onet.pl"
SERVER_GMX = "imap.gmx.com"
SERVER_GMX_NET ="imap.gmx.net"
SERVER_PISS_MAIL = "mail.pissmail.com"
INBOX_LV = "mail.inbox.lv"
@@ -97,6 +100,8 @@ def create_imap(login: str):
imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143)
elif DOMAIN_NAVER in login:
imap = imaplib.IMAP4_SSL(NAVER_SERVER, port=993)
elif DOMAIN_GMX_DE in login:
imap = imaplib.IMAP4_SSL(SERVER_GMX_NET, port=993)
elif DOMAIN_INBOX_LV in login:
imap = imaplib.IMAP4_SSL(INBOX_LV, port=993)
elif DOMAIN_PISS_MAIL in login or DOMAIN_CHILD_PIZZA in login or DOMAIN_DMC_CHAT in login or DOMAIN_GENOCIDE_FUN in login or DOMAIN_HATESJE_WS in login or DOMAIN_INCEL_EMAIL in login or DOMAIN_SHITPOSTING_EXPERT in login:
src/notification/AcceptedResultPojo.py
+1
View File
@@ -24,6 +24,7 @@ class AcceptedResultPojo:
self.created_at = created_at
self.validated_at = validated_at
self.mail_password = ""
self.account_password = ""
def get_accepted_result_from(reserve_pojo: ReserveResultPojo, mongo_db_manager: MongoDbManager,
src/params.py
-4
View File
@@ -1,11 +1,7 @@
import random
import string
from src.logs.LogSender import LogSender
from src.proxy.proxy_type import ProxyType
oracle_log_sender = LogSender()
# proxy
PROXY_SERVER = "http://gw.ntnt.io:5959"
PROXY_PASSWORD = "94sY7zwBG13i"
src/person_name/contact_manager.py
+17 -13
View File
@@ -8,7 +8,7 @@ from src.utils.excel_reader import read_contacts, fr_phone_number_prefix, get_ra
def upload_contacts_list():
_contacts_to_book = read_contacts("/Users/panlei/Desktop/contact_list_2024-03-30.xlsx")
_contacts_to_book = read_contacts("/Users/lpan/Desktop/contact_list_2024-04-15.xlsx")
return _contacts_to_book
@@ -19,7 +19,7 @@ def fix_phone_number_format(file_path):
if _contact.phone[0:2] not in fr_phone_number_prefix:
print(_contact)
_contact.phone = get_random_fr_phone_numbers()
write_new_contacts_to_excel(_contact_list, file_name="25_03_to_test")
write_new_contacts_to_excel(_contact_list, file_name="09_04_to_test")
def write_new_contacts_to_excel(valid_contacts: list, file_name=str(datetime.date.today())):
@@ -46,7 +46,7 @@ def write_new_contacts_to_excel(valid_contacts: list, file_name=str(datetime.dat
workbook.close()
def generate_valid_contact_list_for_day():
def generate_valid_contact_list_for_day(sement_number = 1):
_valid_contact_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
_all_contacts = MONGO_STORE_MANAGER.get_all_contacts_to_book()
_contact_to_save = []
@@ -62,8 +62,15 @@ def generate_valid_contact_list_for_day():
_contact.first_name = _true_contact.first_name
if _contact.url_validated:
if _contact.last_name is not None and len(_contact.last_name) > 0:
_contact_to_save.append(_contact)
write_new_contacts_to_excel(_contact_to_save)
for i in range(0, sement_number):
_step = int(len(_contact_to_save) / sement_number)
_sublist = _contact_to_save[i * _step:_step * (i + 1)]
_file_name = str(datetime.date.today()) +"_"+ str(i+1)
write_new_contacts_to_excel(_sublist,file_name=_file_name)
def merge_contact_list_files(file_list: list, final_file_name="merged_contact_list"):
@@ -90,14 +97,11 @@ def generate_all_contact_list():
if __name__ == '__main__':
# contacts_to_book = upload_contacts_list()
# MONGO_STORE_MANAGER.upload_contact_list(contacts_to_book)
# generate_valid_contact_list_for_day()
generate_valid_contact_list_for_day(sement_number=3)
# generate_all_contact_list()
merge_contact_list_files(
["/Users/panlei/Desktop/contact_list_2024-03-29.xlsx",
"/Users/panlei/Desktop/contact_list_2024-03-27.xlsx",
"/Users/panlei/Desktop/contact_list_2024-03-28.xlsx",
"/Users/panlei/Desktop/contact_list_2024-03-26.xlsx",
"/Users/panlei/Desktop/contact_list_2024-03-25.xlsx",
"/Users/panlei/Desktop/contact_list_2024-03-23.xlsx",
"/Users/panlei/Desktop/contact_list_2024-03-22.xlsx"])
# fix_phone_number_format("/Users/panlei/Desktop/25_03_to_test.xlsx")
# merge_contact_list_files(
# ["/Users/lpan/Desktop/contact_list_2024-04-10.xlsx",
# "/Users/lpan/Desktop/contact_list_2024-04-09.xlsx",
# "/Users/lpan/Desktop/contact_list_2024-04-06.xlsx",
# "/Users/lpan/Desktop/contact_list_2024-04-04.xlsx"])
# fix_phone_number_format("/Users/lpan/Desktop/09_04_to_test.xlsx")
src/templates/appointment_results.html
+3 -1
View File
@@ -2,7 +2,7 @@
<br/>
<br/>电话: ${result.phone}
<br/>
<br/>邮件: ${result.email}, 密码: ${result.mail_password} (用网易邮箱大师登录)
<br/>邮件: ${result.email}, 密码: ${result.mail_password} (用QQ邮箱或者网易邮箱大师登录)
<br/>
<br/>护照: ${result.passport}
<br/>
@@ -10,6 +10,8 @@
<br/>
<br/>
<br/> 邮件: ${result.message}
<br/> 帐户密码: ${result.account_password} (内部使用，转发的时候要删除)
<!--?<br/>-->
<!--?<br/> SIM卡CCID: ${result.ccid}-->