diff --git a/.gitignore b/.gitignore index b36ba63..ed351c5 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ out lib appointment_*.log ./build +./dist diff --git a/appointment.spec b/appointment.spec index a23460d..928f9d5 100644 --- a/appointment.spec +++ b/appointment.spec @@ -49,7 +49,7 @@ coll = COLLECT( name='appointment', ) import shutil -shutil.copyfile('C:\\Users\\landd\\IdeaProjects\\appointment_tool\\appointment.json', '{0}/appointment.json'.format(DISTPATH)) -shutil.copyfile('C:\\Users\\landd\\IdeaProjects\\appointment_tool\\contact_all.xlsx', '{0}/contact_all.xlsx'.format(DISTPATH)) -shutil.copyfile('.\config.ini', '{0}/config.ini'.format(DISTPATH)) +shutil.copyfile('./appointment.json', '{0}/appointment.json'.format(DISTPATH)) +shutil.copyfile('./contact_all.xlsx', '{0}/contact_all.xlsx'.format(DISTPATH)) +shutil.copyfile('./config.ini', '{0}/config.ini'.format(DISTPATH)) shutil.copytree('./venv\Lib/site-packages/grpc/_cython/_credentials', '{0}/appointment/grpc/_cython/_credentials'.format(DISTPATH)) \ No newline at end of file diff --git a/appointment_mac.spec b/appointment_mac.spec new file mode 100644 index 0000000..ddb9e2c --- /dev/null +++ b/appointment_mac.spec @@ -0,0 +1,55 @@ +# -*- mode: python ; coding: utf-8 -*- + + +block_cipher = None + + +a = Analysis( + ['appointment.py'], + pathex=[], + binaries=[], + datas=[], + hiddenimports=[], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False, +) +pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) + +exe = EXE( + pyz, + a.scripts, + [], + exclude_binaries=True, + name='appointment', + debug=True, + bootloader_ignore_signals=False, + strip=False, + upx=False, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) +coll = COLLECT( + exe, + a.binaries, + a.zipfiles, + a.datas, + strip=False, + upx=True, + upx_exclude=[], + name='appointment', +) +import shutil +shutil.copyfile('./appointment.json', '{0}/appointment.json'.format(DISTPATH)) +shutil.copyfile('./contact_all.xlsx', '{0}/contact_all.xlsx'.format(DISTPATH)) +shutil.copyfile('./config.ini', '{0}/config.ini'.format(DISTPATH)) +shutil.copytree('./venv/lib/python3.8/site-packages/grpc/_cython/_credentials', '{0}/appointment/grpc/_cython/_credentials'.format(DISTPATH)) \ No newline at end of file diff --git a/check_results.py b/check_results.py index efdc2b8..d3348ee 100644 --- a/check_results.py +++ b/check_results.py @@ -8,12 +8,13 @@ from typing import Union from playwright.sync_api import sync_playwright import params -from workers.commandor_page import get_random_id_number_for_proxy from logs.LogSender import TYPE_EVENT_CHECK_RESULTS, LOG_SUBJECT_EVENT -from pojo.ReserveResultPojo import ReserveResultPojo +from pojo.ReserveResultPojo import ReserveResultPojo, PublishType SORRY_SENTENCE = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci" +NOT_AVAILABLE_CONTENT = "For more than 130 years, our House has offered its full expertise to satisfy" PENDING_SENTENCE = "Ce soir, entre 20:00 et 20:30, vous obtiendrez une réponse par e-mail." +PENDING_SENTENCE_EN = "This evening between 20:00 and 20:30 you will receive a response by email." class ResultEnum(Enum): @@ -22,14 +23,14 @@ class ResultEnum(Enum): PENDING = "PENDING" -class Tls(threading.local): +class TlsPlaywright(threading.local): def __init__(self) -> None: self.playwright = sync_playwright().start() print("Create playwright instance in Thread", threading.current_thread().name) -class Worker: - tls = Tls() +class ResultChecker: + tls = TlsPlaywright() def __init__(self): self.logger = logging.getLogger("Worker") @@ -38,7 +39,7 @@ class Worker: try: self.browser = playwright.webkit.launch(headless=False, timeout=90000, proxy=proxy) device = random.choice(params.DEVICES) - self.logger.info("device is " + device) + self.logger.info("模拟设备: " + device) pixel_2 = self.tls.playwright.devices[device] context = self.browser.new_context(**pixel_2, locale='en-GB') self.page = context.new_page() @@ -74,8 +75,8 @@ class Worker: "password": params.PROXY_PASSWORD } while content is None: - content = self.load_page(self.tls.playwright, url, proxy) - random_id_number = get_random_id_number_for_proxy() + content = self.load_page(self.tls.playwright, proxy, url) + random_id_number = params.get_random_id_number_for_proxy() proxy_username = "panleicim-res-fr-" + random_id_number print("proxy_username is " + proxy_username) proxy = { @@ -93,6 +94,12 @@ class Worker: elif PENDING_SENTENCE in content: print("status is PENDING") status = ResultEnum.PENDING + elif PENDING_SENTENCE_EN in content: + print("status is PENDING") + status = ResultEnum.PENDING + elif NOT_AVAILABLE_CONTENT in content: + print("status is REFUSED") + status = ResultEnum.REFUSED else: print("status is ACCEPTED") status = ResultEnum.ACCEPTED @@ -104,15 +111,28 @@ if __name__ == '__main__': # get the list params.oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS) db_manager = params.firebase_store_manager + # collection = db_manager.get_all_successful_items_for_day("2022-05-14", "landd") collection = db_manager.get_all_successful_items() count = 0 - task_list = list() - with ThreadPoolExecutor(max_workers=2) as executor: - for appointment in collection.stream(): + # result_pojo = ReserveResultPojo(type=PublishType.SUCCESS, phone="0649614591", email="panleicim@gmail.com", + # message="SUCCESS", firstName="Lei", lastName="PAN", url='https://api.ipify.org') + result_list = [] + for appointment in collection.stream(): + reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) + result_list.append(reserve_pojo) + # result_list.append(result_pojo) + # for result in result_list: + # if result.accepted is None or ResultEnum.PENDING.value == result.accepted: + # ResultChecker().run(result, collection) + # else: + # print("status is " + result.accepted) + + with ThreadPoolExecutor(max_workers=5) as executor: + for reserve in result_list: count = count + 1 - reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) - if reserve_pojo.accepted: - print("status is " + reserve_pojo.accepted) - if reserve_pojo.accepted is None or ResultEnum.PENDING.value == reserve_pojo.accepted: - executor.submit(Worker().run, threading.Event(), reserve_pojo, collection) + if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted: + executor.submit(ResultChecker().run, reserve, collection) + else: + print("status is " + reserve.accepted) + print(count) diff --git a/config.ini b/config.ini index 654e4db..151338a 100644 --- a/config.ini +++ b/config.ini @@ -1,4 +1,4 @@ [DEFAULT] -contact_list_file = .\contact_all.xlsx -firebase_config_file = .\appointment.json -LOGS_DIR = .\ \ No newline at end of file +contact_list_file = ./contact_all.xlsx +firebase_config_file = ./appointment.json +LOGS_DIR = ./ \ No newline at end of file diff --git a/db/DbManager.py b/db/DbManager.py index 241d58f..c448813 100644 --- a/db/DbManager.py +++ b/db/DbManager.py @@ -1,4 +1,5 @@ import datetime +from typing import Union import firebase_admin from firebase_admin import credentials, firestore @@ -15,6 +16,7 @@ MAIL_COLLECTION_NAME = "mail_list" SIM_INFOS = "sim_infos" TIMEOUT = "timeout_items" + class DataManager: batch_size = 20 @@ -28,10 +30,12 @@ class DataManager: return sim_info_collection def get_all_successful_items(self): - return self.get_all_successful_items_for_day(str(datetime.date.today())) + return self.get_all_successful_items_for_day(str(datetime.date.today()), None) - def get_all_successful_items_for_day(self, day): + def get_all_successful_items_for_day(self, day, source_from: Union[str, None]): doc_ref = self._db.collection(day) + if source_from is not None: + doc_ref.where(u'source_from', u'==', source_from) return doc_ref def save_sim_info(self, sim_info: SimInfoPojo): diff --git a/logs/LogSender.py b/logs/LogSender.py index efad0cf..477cf38 100644 --- a/logs/LogSender.py +++ b/logs/LogSender.py @@ -23,6 +23,9 @@ TYPE_EVENT_CHANGE_SLOT = "EVENT_CHANGE_SLOT" TYPE_SMS_RECEIVED = "TYPE_SMS_RECEIVED" LOG_ERROR = "ERROR" LOG_TYPE_INFO = "INFO" +LOG_ERROR_TYPE_DOUBLE_DATA = "DOUBLE_REQUEST_FOR_SAME_DATA" +LOG_ERROR_TOO_MANY_REQUEST_TODAY = "TOO_MANY_REQUEST_TODAY" +LOG_SUBJECT_ERROR = "ERROR" LOG_APPOINTMENT_ERROR = "APPOINTMENT_ERROR" LOG_APPOINTMENT_TIMEOUT = "TIMEOUT" LOG_APPOINTMENT_CONTACT_NOT_FOUND = "CONTACT_NOT_FOUND" @@ -67,6 +70,14 @@ class LogSender: self._identity = oci.identity.IdentityClient(self._config) self._loggingingestion_client = LoggingClient(self._config, timeout=60.0, retry_strategy=custom_retry_strategy) + def send_double_data_error(self, contact: ContactPojo): + error_msg = contact.mail + self.send_log(msg=error_msg, type=LOG_ERROR_TYPE_DOUBLE_DATA, subject=LOG_SUBJECT_ERROR) + + def send_too_many_error(self, contact: ContactPojo): + error_msg = contact.mail + self.send_log(msg=error_msg, type=LOG_ERROR_TOO_MANY_REQUEST_TODAY, subject=LOG_SUBJECT_ERROR) + def send_appoint_result(self, result: ReserveResultPojo): if result.type == PublishType.SUCCESS: # get id diff --git a/main.py b/main.py index 120cc1e..45a2d81 100644 --- a/main.py +++ b/main.py @@ -42,4 +42,4 @@ def get_proxy(phone_number): if __name__ == '__main__': # 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机，传入1的时候是总店 - start_book(1, 1, store_choose_state=0) + start_book(16, 16, store_choose_state=0) diff --git a/params.py b/params.py index 8699e3b..0f413bf 100644 --- a/params.py +++ b/params.py @@ -1,4 +1,8 @@ import configparser +import random +import string + +from playwright.sync_api import sync_playwright from db.DbManager import DataManager from logs.LogSender import LogSender @@ -10,6 +14,13 @@ oracle_log_sender = LogSender() PROXY_SERVER = "http://gw.ntnt.io:5959" PROXY_PASSWORD = "94sY7zwBG13i" +def get_random_id_number_for_proxy() -> str: + S = 8 # number of characters in the string. + ran = ''.join(random.choices(string.digits, k=S)) + id_number = str(ran) + print("The randomly generated string is : " + str(ran)) # print the random data + return id_number + DEVICES = ['iPad (gen 6)', 'iPad (gen 6) landscape', 'iPad (gen 7)', 'iPad (gen 7) landscape', 'iPad Mini', 'iPad Mini landscape', 'iPad Pro 11', 'iPad Pro 11 landscape', 'iPhone 6', 'iPhone 6 landscape', 'iPhone 6 Plus', 'iPhone 6 Plus landscape', 'iPhone 7', 'iPhone 7 landscape', 'iPhone 7 Plus', diff --git a/utils/excel_reader.py b/utils/excel_reader.py index fc7d133..ca72027 100644 --- a/utils/excel_reader.py +++ b/utils/excel_reader.py @@ -20,14 +20,15 @@ class ExcelHelper: contact_dict_list = json.loads(contact_list_in_json) contact_list = [] for contact_dict in contact_dict_list: - raw_name = contact_dict['name'].strip() - name = raw_name.split(' ') - last_name = name[0] - first_name = name[-1] - contact = ContactPojo(phone_number=contact_dict['phone'], - last_name=last_name, - first_name=first_name, - passport_number=contact_dict['passport'], - mail=contact_dict['email']) - contact_list.append(contact) + if contact_dict['name']: + raw_name = contact_dict['name'].strip() + name = raw_name.split(' ') + last_name = name[0] + first_name = name[-1] + contact = ContactPojo(phone_number=contact_dict['phone'], + last_name=last_name, + first_name=first_name, + passport_number=contact_dict['passport'], + mail=contact_dict['email']) + contact_list.append(contact) return contact_list diff --git a/workers/commandor_page.py b/workers/commandor_page.py index f2dc76b..34a0ed5 100644 --- a/workers/commandor_page.py +++ b/workers/commandor_page.py @@ -22,6 +22,8 @@ OTP_FIELD_ID = "#sms_code" MESSAGE_FIELD_CLASS = ".message" CONFIRMED_MESSAGE = "Your request for a Leather Goods appointment has been registered" +DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today." +TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests" TIME_OUT = 400000 OTP_TIMEOUT = 240 @@ -76,7 +78,7 @@ class CommandorPage: first_page = None while first_page is None: first_page = self.start_browser(proxy, self.tls.playwright, devices) - proxy_username = "panleicim-res-fr-" + get_random_id_number_for_proxy() + proxy_username = "panleicim-res-fr-" + params.get_random_id_number_for_proxy() self.logger.info("proxy_username is " + proxy_username) proxy = { "server": params.PROXY_SERVER, @@ -152,7 +154,7 @@ class CommandorPage: if CONFIRMED_MESSAGE in message: # publish the successful message self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url) - self.getErrors() + self.get_errors() def on_document_loaded(self): print("on_document_loaded called") @@ -191,16 +193,32 @@ class CommandorPage: except Exception as error: self.logger.error(error) - def getErrors(self): + def get_errors(self): # send error result self.publish_message_to_queue(self.contact, PublishType.ERROR, self.page.url) try: items = self.page.query_selector("div.alert") if items: - print("error is " + items.inner_html()) + erro_content = items.inner_html() + print("错误:" + erro_content) + self._handle_errors(erro_content) except Exception as ext: self.logger.error(ext) + def _handle_errors(self, erro_content: str): + if DOUBLE_REQUEST_ERROR_MESSAGE in erro_content: + # this email has been already used + params.oracle_log_sender.send_double_data_error(self.contact) + # close browser + time.sleep(2) + self.browser.close() + elif TOO_MANY_REQUEST_ERROR_MESSAGE in erro_content: + # this email has been already used + params.oracle_log_sender.send_too_many_error(self.contact) + # close browser + time.sleep(2) + self.browser.close() + def _set_email(self, email): time.sleep(get_random_wait_time()) try: @@ -266,14 +284,6 @@ class CommandorPage: self.browser.close() -def get_random_id_number_for_proxy() -> str: - S = 8 # number of characters in the string. - ran = ''.join(random.choices(string.digits, k=S)) - id_number = str(ran) - print("The randomly generated string is : " + str(ran)) # print the random data - return id_number - - def get_random_id_number() -> str: S = 8 # number of characters in the string. ran = ''.join(random.choices(string.digits, k=S)) @@ -287,7 +297,7 @@ def on_success(result: ReserveResultPojo): def launch_page(): - PROXY_USERNAME = "panleicim-res-fr-" + get_random_id_number_for_proxy() + PROXY_USERNAME = "panleicim-res-fr-" + params.get_random_id_number_for_proxy() print("proxy_username is " + PROXY_USERNAME) proxy = { "server": PROXY_SERVER,