From 9e4972c8c9b777003af4279fb4ae3641eda274e2 Mon Sep 17 00:00:00 2001 From: panlei Date: Sun, 15 May 2022 22:52:42 +0200 Subject: [PATCH 1/6] sync problem with playwright --- .gitignore | 1 + appointment.spec | 6 ++--- appointment_mac.spec | 55 ++++++++++++++++++++++++++++++++++++++++++++ check_results.py | 21 ++++++++++------- config.ini | 6 ++--- 5 files changed, 75 insertions(+), 14 deletions(-) create mode 100644 appointment_mac.spec diff --git a/.gitignore b/.gitignore index b36ba63..ed351c5 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ out lib appointment_*.log ./build +./dist diff --git a/appointment.spec b/appointment.spec index a23460d..928f9d5 100644 --- a/appointment.spec +++ b/appointment.spec @@ -49,7 +49,7 @@ coll = COLLECT( name='appointment', ) import shutil -shutil.copyfile('C:\\Users\\landd\\IdeaProjects\\appointment_tool\\appointment.json', '{0}/appointment.json'.format(DISTPATH)) -shutil.copyfile('C:\\Users\\landd\\IdeaProjects\\appointment_tool\\contact_all.xlsx', '{0}/contact_all.xlsx'.format(DISTPATH)) -shutil.copyfile('.\config.ini', '{0}/config.ini'.format(DISTPATH)) +shutil.copyfile('./appointment.json', '{0}/appointment.json'.format(DISTPATH)) +shutil.copyfile('./contact_all.xlsx', '{0}/contact_all.xlsx'.format(DISTPATH)) +shutil.copyfile('./config.ini', '{0}/config.ini'.format(DISTPATH)) shutil.copytree('./venv\Lib/site-packages/grpc/_cython/_credentials', '{0}/appointment/grpc/_cython/_credentials'.format(DISTPATH)) \ No newline at end of file diff --git a/appointment_mac.spec b/appointment_mac.spec new file mode 100644 index 0000000..ddb9e2c --- /dev/null +++ b/appointment_mac.spec @@ -0,0 +1,55 @@ +# -*- mode: python ; coding: utf-8 -*- + + +block_cipher = None + + +a = Analysis( + ['appointment.py'], + pathex=[], + binaries=[], + datas=[], + hiddenimports=[], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False, +) +pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) + +exe = EXE( + pyz, + a.scripts, + [], + exclude_binaries=True, + name='appointment', + debug=True, + bootloader_ignore_signals=False, + strip=False, + upx=False, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) +coll = COLLECT( + exe, + a.binaries, + a.zipfiles, + a.datas, + strip=False, + upx=True, + upx_exclude=[], + name='appointment', +) +import shutil +shutil.copyfile('./appointment.json', '{0}/appointment.json'.format(DISTPATH)) +shutil.copyfile('./contact_all.xlsx', '{0}/contact_all.xlsx'.format(DISTPATH)) +shutil.copyfile('./config.ini', '{0}/config.ini'.format(DISTPATH)) +shutil.copytree('./venv/lib/python3.8/site-packages/grpc/_cython/_credentials', '{0}/appointment/grpc/_cython/_credentials'.format(DISTPATH)) \ No newline at end of file diff --git a/check_results.py b/check_results.py index efdc2b8..dc5feee 100644 --- a/check_results.py +++ b/check_results.py @@ -10,7 +10,7 @@ from playwright.sync_api import sync_playwright import params from workers.commandor_page import get_random_id_number_for_proxy from logs.LogSender import TYPE_EVENT_CHECK_RESULTS, LOG_SUBJECT_EVENT -from pojo.ReserveResultPojo import ReserveResultPojo +from pojo.ReserveResultPojo import ReserveResultPojo, PublishType SORRY_SENTENCE = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci" PENDING_SENTENCE = "Ce soir, entre 20:00 et 20:30, vous obtiendrez une réponse par e-mail." @@ -106,13 +106,18 @@ if __name__ == '__main__': db_manager = params.firebase_store_manager collection = db_manager.get_all_successful_items() count = 0 - task_list = list() + result_pojo = ReserveResultPojo(type=PublishType.SUCCESS, phone="0649614591", email="panleicim@gmail.com", + message="SUCCESS", firstName="Lei", lastName="PAN") + result_list = [] + for appointment in collection.stream(): + reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) + result_list.append(result_pojo) + with ThreadPoolExecutor(max_workers=2) as executor: - for appointment in collection.stream(): + for reserve in result_list: count = count + 1 - reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) - if reserve_pojo.accepted: - print("status is " + reserve_pojo.accepted) - if reserve_pojo.accepted is None or ResultEnum.PENDING.value == reserve_pojo.accepted: - executor.submit(Worker().run, threading.Event(), reserve_pojo, collection) + if reserve.accepted: + print("status is " + reserve.accepted) + if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted: + executor.submit(Worker().run, reserve, collection) print(count) diff --git a/config.ini b/config.ini index 654e4db..4d8ff0b 100644 --- a/config.ini +++ b/config.ini @@ -1,4 +1,4 @@ [DEFAULT] -contact_list_file = .\contact_all.xlsx -firebase_config_file = .\appointment.json -LOGS_DIR = .\ \ No newline at end of file +contact_list_file = /Users/panlei/IdeaProjects/appointment_tool/contact_all.xlsx +firebase_config_file = /Users/panlei/IdeaProjects/appointment_tool/appointment.json +LOGS_DIR = ./ \ No newline at end of file From dbdb88600adfd9ee99590e736dfe6146c0c78911 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Sun, 15 May 2022 23:13:51 +0200 Subject: [PATCH 2/6] porbleme of sync resolved --- check_results.py | 22 +++++++++++----------- db/DbManager.py | 6 +++++- params.py | 3 +++ workers/commandor_page.py | 4 +--- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/check_results.py b/check_results.py index dc5feee..7c39ebd 100644 --- a/check_results.py +++ b/check_results.py @@ -22,14 +22,14 @@ class ResultEnum(Enum): PENDING = "PENDING" -class Tls(threading.local): +class TlsPlaywright(threading.local): def __init__(self) -> None: - self.playwright = sync_playwright().start() + self.playwright = params.SINGLE_PLAYWRIGHT_INSTANCE print("Create playwright instance in Thread", threading.current_thread().name) -class Worker: - tls = Tls() +class ResultChecker: + tls = TlsPlaywright() def __init__(self): self.logger = logging.getLogger("Worker") @@ -38,7 +38,7 @@ class Worker: try: self.browser = playwright.webkit.launch(headless=False, timeout=90000, proxy=proxy) device = random.choice(params.DEVICES) - self.logger.info("device is " + device) + self.logger.info("模拟设备: " + device) pixel_2 = self.tls.playwright.devices[device] context = self.browser.new_context(**pixel_2, locale='en-GB') self.page = context.new_page() @@ -104,14 +104,14 @@ if __name__ == '__main__': # get the list params.oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS) db_manager = params.firebase_store_manager - collection = db_manager.get_all_successful_items() + collection = db_manager.get_all_successful_items_for_day() count = 0 result_pojo = ReserveResultPojo(type=PublishType.SUCCESS, phone="0649614591", email="panleicim@gmail.com", - message="SUCCESS", firstName="Lei", lastName="PAN") + message="SUCCESS", firstName="Lei", lastName="PAN", url='https://api.ipify.org') result_list = [] - for appointment in collection.stream(): - reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) - result_list.append(result_pojo) + # for appointment in collection.stream(): + # reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) + result_list.append(result_pojo) with ThreadPoolExecutor(max_workers=2) as executor: for reserve in result_list: @@ -119,5 +119,5 @@ if __name__ == '__main__': if reserve.accepted: print("status is " + reserve.accepted) if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted: - executor.submit(Worker().run, reserve, collection) + executor.submit(ResultChecker().run, reserve, collection) print(count) diff --git a/db/DbManager.py b/db/DbManager.py index 241d58f..ff7ae1a 100644 --- a/db/DbManager.py +++ b/db/DbManager.py @@ -1,4 +1,5 @@ import datetime +from typing import Union import firebase_admin from firebase_admin import credentials, firestore @@ -15,6 +16,7 @@ MAIL_COLLECTION_NAME = "mail_list" SIM_INFOS = "sim_infos" TIMEOUT = "timeout_items" + class DataManager: batch_size = 20 @@ -30,8 +32,10 @@ class DataManager: def get_all_successful_items(self): return self.get_all_successful_items_for_day(str(datetime.date.today())) - def get_all_successful_items_for_day(self, day): + def get_all_successful_items_for_day(self, day, source_from: Union[str, None]): doc_ref = self._db.collection(day) + if source_from is not None: + doc_ref.where() return doc_ref def save_sim_info(self, sim_info: SimInfoPojo): diff --git a/params.py b/params.py index 8699e3b..d154bc9 100644 --- a/params.py +++ b/params.py @@ -1,5 +1,7 @@ import configparser +from playwright.sync_api import sync_playwright + from db.DbManager import DataManager from logs.LogSender import LogSender @@ -9,6 +11,7 @@ oracle_log_sender = LogSender() # proxy PROXY_SERVER = "http://gw.ntnt.io:5959" PROXY_PASSWORD = "94sY7zwBG13i" +SINGLE_PLAYWRIGHT_INSTANCE = sync_playwright().start() DEVICES = ['iPad (gen 6)', 'iPad (gen 6) landscape', 'iPad (gen 7)', 'iPad (gen 7) landscape', 'iPad Mini', 'iPad Mini landscape', 'iPad Pro 11', 'iPad Pro 11 landscape', 'iPhone 6', 'iPhone 6 landscape', diff --git a/workers/commandor_page.py b/workers/commandor_page.py index ad66bc4..da84d83 100644 --- a/workers/commandor_page.py +++ b/workers/commandor_page.py @@ -5,8 +5,6 @@ import threading import time from typing import Union -from playwright.sync_api import sync_playwright - import params from params import PROXY_SERVER, PROXY_PASSWORD from pojo.ReserveResultPojo import ReserveResultPojo, PublishType @@ -33,7 +31,7 @@ def get_random_wait_time() -> float: class Tls(threading.local): def __init__(self) -> None: - self.playwright = sync_playwright().start() + self.playwright = params.SINGLE_PLAYWRIGHT_INSTANCE class CommandorPage: From 5ac4f6a0fb2ba735ed9900b7cf278425549959ea Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 16 May 2022 07:01:33 +0200 Subject: [PATCH 3/6] can check results in one thread --- check_results.py | 40 +++++++++++++++++++++++++--------------- config.ini | 4 ++-- db/DbManager.py | 2 +- main.py | 2 +- 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/check_results.py b/check_results.py index 7c39ebd..ca698e1 100644 --- a/check_results.py +++ b/check_results.py @@ -13,6 +13,7 @@ from logs.LogSender import TYPE_EVENT_CHECK_RESULTS, LOG_SUBJECT_EVENT from pojo.ReserveResultPojo import ReserveResultPojo, PublishType SORRY_SENTENCE = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci" +NOT_AVAILABLE_CONTENT = "For more than 130 years, our House has offered its full expertise to satisfy" PENDING_SENTENCE = "Ce soir, entre 20:00 et 20:30, vous obtiendrez une réponse par e-mail." @@ -74,7 +75,7 @@ class ResultChecker: "password": params.PROXY_PASSWORD } while content is None: - content = self.load_page(self.tls.playwright, url, proxy) + content = self.load_page(self.tls.playwright, proxy, url) random_id_number = get_random_id_number_for_proxy() proxy_username = "panleicim-res-fr-" + random_id_number print("proxy_username is " + proxy_username) @@ -93,6 +94,9 @@ class ResultChecker: elif PENDING_SENTENCE in content: print("status is PENDING") status = ResultEnum.PENDING + elif NOT_AVAILABLE_CONTENT in content: + print("status is REFUSED") + status = ResultEnum.REFUSED else: print("status is ACCEPTED") status = ResultEnum.ACCEPTED @@ -104,20 +108,26 @@ if __name__ == '__main__': # get the list params.oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS) db_manager = params.firebase_store_manager - collection = db_manager.get_all_successful_items_for_day() + collection = db_manager.get_all_successful_items_for_day("2022-05-14", "landd") count = 0 - result_pojo = ReserveResultPojo(type=PublishType.SUCCESS, phone="0649614591", email="panleicim@gmail.com", - message="SUCCESS", firstName="Lei", lastName="PAN", url='https://api.ipify.org') + # result_pojo = ReserveResultPojo(type=PublishType.SUCCESS, phone="0649614591", email="panleicim@gmail.com", + # message="SUCCESS", firstName="Lei", lastName="PAN", url='https://api.ipify.org') result_list = [] - # for appointment in collection.stream(): - # reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) - result_list.append(result_pojo) + for appointment in collection.stream(): + reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) + result_list.append(reserve_pojo) + # result_list.append(result_pojo) + for result in result_list: + if result.accepted is None or ResultEnum.ACCEPTED.value == result.accepted: + ResultChecker().run(result, collection) + else: + print("status is " + result.accepted) - with ThreadPoolExecutor(max_workers=2) as executor: - for reserve in result_list: - count = count + 1 - if reserve.accepted: - print("status is " + reserve.accepted) - if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted: - executor.submit(ResultChecker().run, reserve, collection) - print(count) + # with ThreadPoolExecutor(max_workers=1) as executor: + # for reserve in result_list: + # count = count + 1 + # if reserve.accepted: + # print("status is " + reserve.accepted) + # if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted: + # executor.submit(ResultChecker().run, reserve, collection) + # print(count) diff --git a/config.ini b/config.ini index 4d8ff0b..151338a 100644 --- a/config.ini +++ b/config.ini @@ -1,4 +1,4 @@ [DEFAULT] -contact_list_file = /Users/panlei/IdeaProjects/appointment_tool/contact_all.xlsx -firebase_config_file = /Users/panlei/IdeaProjects/appointment_tool/appointment.json +contact_list_file = ./contact_all.xlsx +firebase_config_file = ./appointment.json LOGS_DIR = ./ \ No newline at end of file diff --git a/db/DbManager.py b/db/DbManager.py index ff7ae1a..6799b33 100644 --- a/db/DbManager.py +++ b/db/DbManager.py @@ -35,7 +35,7 @@ class DataManager: def get_all_successful_items_for_day(self, day, source_from: Union[str, None]): doc_ref = self._db.collection(day) if source_from is not None: - doc_ref.where() + doc_ref.where(u'source_from', u'==', source_from) return doc_ref def save_sim_info(self, sim_info: SimInfoPojo): diff --git a/main.py b/main.py index 120cc1e..45a2d81 100644 --- a/main.py +++ b/main.py @@ -42,4 +42,4 @@ def get_proxy(phone_number): if __name__ == '__main__': # 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机，传入1的时候是总店 - start_book(1, 1, store_choose_state=0) + start_book(16, 16, store_choose_state=0) From 08ed7c8954d861237badd8e0ed3c01a2ee0cd98c Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 16 May 2022 08:48:55 +0200 Subject: [PATCH 4/6] can run playwright in multi-threads --- check_results.py | 44 +++++++++++++++++++-------------------- params.py | 10 ++++++++- workers/commandor_page.py | 16 +++++--------- 3 files changed, 36 insertions(+), 34 deletions(-) diff --git a/check_results.py b/check_results.py index ca698e1..8d7915c 100644 --- a/check_results.py +++ b/check_results.py @@ -8,7 +8,6 @@ from typing import Union from playwright.sync_api import sync_playwright import params -from workers.commandor_page import get_random_id_number_for_proxy from logs.LogSender import TYPE_EVENT_CHECK_RESULTS, LOG_SUBJECT_EVENT from pojo.ReserveResultPojo import ReserveResultPojo, PublishType @@ -25,7 +24,7 @@ class ResultEnum(Enum): class TlsPlaywright(threading.local): def __init__(self) -> None: - self.playwright = params.SINGLE_PLAYWRIGHT_INSTANCE + self.playwright = sync_playwright().start() print("Create playwright instance in Thread", threading.current_thread().name) @@ -76,7 +75,7 @@ class ResultChecker: } while content is None: content = self.load_page(self.tls.playwright, proxy, url) - random_id_number = get_random_id_number_for_proxy() + random_id_number = params.get_random_id_number_for_proxy() proxy_username = "panleicim-res-fr-" + random_id_number print("proxy_username is " + proxy_username) proxy = { @@ -110,24 +109,25 @@ if __name__ == '__main__': db_manager = params.firebase_store_manager collection = db_manager.get_all_successful_items_for_day("2022-05-14", "landd") count = 0 - # result_pojo = ReserveResultPojo(type=PublishType.SUCCESS, phone="0649614591", email="panleicim@gmail.com", - # message="SUCCESS", firstName="Lei", lastName="PAN", url='https://api.ipify.org') + result_pojo = ReserveResultPojo(type=PublishType.SUCCESS, phone="0649614591", email="panleicim@gmail.com", + message="SUCCESS", firstName="Lei", lastName="PAN", url='https://api.ipify.org') result_list = [] - for appointment in collection.stream(): - reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) - result_list.append(reserve_pojo) - # result_list.append(result_pojo) - for result in result_list: - if result.accepted is None or ResultEnum.ACCEPTED.value == result.accepted: - ResultChecker().run(result, collection) - else: - print("status is " + result.accepted) + # for appointment in collection.stream(): + # reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) + # result_list.append(reserve_pojo) + result_list.append(result_pojo) + # for result in result_list: + # if result.accepted is None or ResultEnum.PENDING.value == result.accepted: + # ResultChecker().run(result, collection) + # else: + # print("status is " + result.accepted) - # with ThreadPoolExecutor(max_workers=1) as executor: - # for reserve in result_list: - # count = count + 1 - # if reserve.accepted: - # print("status is " + reserve.accepted) - # if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted: - # executor.submit(ResultChecker().run, reserve, collection) - # print(count) + with ThreadPoolExecutor(max_workers=1) as executor: + for reserve in result_list: + count = count + 1 + if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted: + executor.submit(ResultChecker().run, reserve, collection) + else: + print("status is " + reserve.accepted) + + print(count) diff --git a/params.py b/params.py index d154bc9..0f413bf 100644 --- a/params.py +++ b/params.py @@ -1,4 +1,6 @@ import configparser +import random +import string from playwright.sync_api import sync_playwright @@ -11,7 +13,13 @@ oracle_log_sender = LogSender() # proxy PROXY_SERVER = "http://gw.ntnt.io:5959" PROXY_PASSWORD = "94sY7zwBG13i" -SINGLE_PLAYWRIGHT_INSTANCE = sync_playwright().start() + +def get_random_id_number_for_proxy() -> str: + S = 8 # number of characters in the string. + ran = ''.join(random.choices(string.digits, k=S)) + id_number = str(ran) + print("The randomly generated string is : " + str(ran)) # print the random data + return id_number DEVICES = ['iPad (gen 6)', 'iPad (gen 6) landscape', 'iPad (gen 7)', 'iPad (gen 7) landscape', 'iPad Mini', 'iPad Mini landscape', 'iPad Pro 11', 'iPad Pro 11 landscape', 'iPhone 6', 'iPhone 6 landscape', diff --git a/workers/commandor_page.py b/workers/commandor_page.py index da84d83..0a42839 100644 --- a/workers/commandor_page.py +++ b/workers/commandor_page.py @@ -5,6 +5,8 @@ import threading import time from typing import Union +from playwright.sync_api import sync_playwright + import params from params import PROXY_SERVER, PROXY_PASSWORD from pojo.ReserveResultPojo import ReserveResultPojo, PublishType @@ -31,7 +33,7 @@ def get_random_wait_time() -> float: class Tls(threading.local): def __init__(self) -> None: - self.playwright = params.SINGLE_PLAYWRIGHT_INSTANCE + self.playwright = sync_playwright().start() class CommandorPage: @@ -74,7 +76,7 @@ class CommandorPage: first_page = None while first_page is None: first_page = self.start_browser(proxy, self.tls.playwright, devices) - proxy_username = "panleicim-res-fr-" + get_random_id_number_for_proxy() + proxy_username = "panleicim-res-fr-" + params.get_random_id_number_for_proxy() self.logger.info("proxy_username is " + proxy_username) proxy = { "server": params.PROXY_SERVER, @@ -264,14 +266,6 @@ class CommandorPage: self.browser.close() -def get_random_id_number_for_proxy() -> str: - S = 8 # number of characters in the string. - ran = ''.join(random.choices(string.digits, k=S)) - id_number = str(ran) - print("The randomly generated string is : " + str(ran)) # print the random data - return id_number - - def get_random_id_number() -> str: S = 8 # number of characters in the string. ran = ''.join(random.choices(string.digits, k=S)) @@ -285,7 +279,7 @@ def on_success(result: ReserveResultPojo): def launch_page(): - PROXY_USERNAME = "panleicim-res-fr-" + get_random_id_number_for_proxy() + PROXY_USERNAME = "panleicim-res-fr-" + params.get_random_id_number_for_proxy() print("proxy_username is " + PROXY_USERNAME) proxy = { "server": PROXY_SERVER, From 12d95355f832db4dc4f837c8e499a69620a94c6d Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 16 May 2022 17:07:27 +0200 Subject: [PATCH 5/6] log double data error and too many requests error --- logs/LogSender.py | 11 +++++++++++ utils/excel_reader.py | 21 +++++++++++---------- workers/commandor_page.py | 24 +++++++++++++++++++++--- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/logs/LogSender.py b/logs/LogSender.py index efad0cf..477cf38 100644 --- a/logs/LogSender.py +++ b/logs/LogSender.py @@ -23,6 +23,9 @@ TYPE_EVENT_CHANGE_SLOT = "EVENT_CHANGE_SLOT" TYPE_SMS_RECEIVED = "TYPE_SMS_RECEIVED" LOG_ERROR = "ERROR" LOG_TYPE_INFO = "INFO" +LOG_ERROR_TYPE_DOUBLE_DATA = "DOUBLE_REQUEST_FOR_SAME_DATA" +LOG_ERROR_TOO_MANY_REQUEST_TODAY = "TOO_MANY_REQUEST_TODAY" +LOG_SUBJECT_ERROR = "ERROR" LOG_APPOINTMENT_ERROR = "APPOINTMENT_ERROR" LOG_APPOINTMENT_TIMEOUT = "TIMEOUT" LOG_APPOINTMENT_CONTACT_NOT_FOUND = "CONTACT_NOT_FOUND" @@ -67,6 +70,14 @@ class LogSender: self._identity = oci.identity.IdentityClient(self._config) self._loggingingestion_client = LoggingClient(self._config, timeout=60.0, retry_strategy=custom_retry_strategy) + def send_double_data_error(self, contact: ContactPojo): + error_msg = contact.mail + self.send_log(msg=error_msg, type=LOG_ERROR_TYPE_DOUBLE_DATA, subject=LOG_SUBJECT_ERROR) + + def send_too_many_error(self, contact: ContactPojo): + error_msg = contact.mail + self.send_log(msg=error_msg, type=LOG_ERROR_TOO_MANY_REQUEST_TODAY, subject=LOG_SUBJECT_ERROR) + def send_appoint_result(self, result: ReserveResultPojo): if result.type == PublishType.SUCCESS: # get id diff --git a/utils/excel_reader.py b/utils/excel_reader.py index fc7d133..ca72027 100644 --- a/utils/excel_reader.py +++ b/utils/excel_reader.py @@ -20,14 +20,15 @@ class ExcelHelper: contact_dict_list = json.loads(contact_list_in_json) contact_list = [] for contact_dict in contact_dict_list: - raw_name = contact_dict['name'].strip() - name = raw_name.split(' ') - last_name = name[0] - first_name = name[-1] - contact = ContactPojo(phone_number=contact_dict['phone'], - last_name=last_name, - first_name=first_name, - passport_number=contact_dict['passport'], - mail=contact_dict['email']) - contact_list.append(contact) + if contact_dict['name']: + raw_name = contact_dict['name'].strip() + name = raw_name.split(' ') + last_name = name[0] + first_name = name[-1] + contact = ContactPojo(phone_number=contact_dict['phone'], + last_name=last_name, + first_name=first_name, + passport_number=contact_dict['passport'], + mail=contact_dict['email']) + contact_list.append(contact) return contact_list diff --git a/workers/commandor_page.py b/workers/commandor_page.py index ad66bc4..1454e4a 100644 --- a/workers/commandor_page.py +++ b/workers/commandor_page.py @@ -22,6 +22,8 @@ OTP_FIELD_ID = "#sms_code" MESSAGE_FIELD_CLASS = ".message" CONFIRMED_MESSAGE = "Your request for a Leather Goods appointment has been registered" +DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today." +TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests" TIME_OUT = 400000 OTP_TIMEOUT = 240 @@ -152,7 +154,7 @@ class CommandorPage: if CONFIRMED_MESSAGE in message: # publish the successful message self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url) - self.getErrors() + self.get_errors() def on_document_loaded(self): print("on_document_loaded called") @@ -191,16 +193,32 @@ class CommandorPage: except Exception as error: self.logger.error(error) - def getErrors(self): + def get_errors(self): # send error result self.publish_message_to_queue(self.contact, PublishType.ERROR, self.page.url) try: items = self.page.query_selector("div.alert") if items: - print(items.inner_html()) + erro_content = items.inner_html() + print("错误:" + erro_content) + self._handle_errors(erro_content) except Exception as ext: self.logger.error(ext) + def _handle_errors(self, erro_content: str): + if DOUBLE_REQUEST_ERROR_MESSAGE in erro_content: + # this email has been already used + params.oracle_log_sender.send_double_data_error(self.contact) + # close browser + time.sleep(2) + self.browser.close() + elif TOO_MANY_REQUEST_ERROR_MESSAGE in erro_content: + # this email has been already used + params.oracle_log_sender.send_too_many_error(self.contact) + # close browser + time.sleep(2) + self.browser.close() + def _set_email(self, email): time.sleep(get_random_wait_time()) try: From 2dcff47fb3ab2b720a41324226e3ca40b335a17d Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 16 May 2022 22:25:19 +0200 Subject: [PATCH 6/6] optimize the check_results.py --- check_results.py | 23 ++++++++++++++--------- db/DbManager.py | 2 +- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/check_results.py b/check_results.py index 8d7915c..d3348ee 100644 --- a/check_results.py +++ b/check_results.py @@ -14,6 +14,7 @@ from pojo.ReserveResultPojo import ReserveResultPojo, PublishType SORRY_SENTENCE = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci" NOT_AVAILABLE_CONTENT = "For more than 130 years, our House has offered its full expertise to satisfy" PENDING_SENTENCE = "Ce soir, entre 20:00 et 20:30, vous obtiendrez une réponse par e-mail." +PENDING_SENTENCE_EN = "This evening between 20:00 and 20:30 you will receive a response by email." class ResultEnum(Enum): @@ -74,7 +75,7 @@ class ResultChecker: "password": params.PROXY_PASSWORD } while content is None: - content = self.load_page(self.tls.playwright, proxy, url) + content = self.load_page(self.tls.playwright, proxy, url) random_id_number = params.get_random_id_number_for_proxy() proxy_username = "panleicim-res-fr-" + random_id_number print("proxy_username is " + proxy_username) @@ -93,6 +94,9 @@ class ResultChecker: elif PENDING_SENTENCE in content: print("status is PENDING") status = ResultEnum.PENDING + elif PENDING_SENTENCE_EN in content: + print("status is PENDING") + status = ResultEnum.PENDING elif NOT_AVAILABLE_CONTENT in content: print("status is REFUSED") status = ResultEnum.REFUSED @@ -107,22 +111,23 @@ if __name__ == '__main__': # get the list params.oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS) db_manager = params.firebase_store_manager - collection = db_manager.get_all_successful_items_for_day("2022-05-14", "landd") + # collection = db_manager.get_all_successful_items_for_day("2022-05-14", "landd") + collection = db_manager.get_all_successful_items() count = 0 - result_pojo = ReserveResultPojo(type=PublishType.SUCCESS, phone="0649614591", email="panleicim@gmail.com", - message="SUCCESS", firstName="Lei", lastName="PAN", url='https://api.ipify.org') + # result_pojo = ReserveResultPojo(type=PublishType.SUCCESS, phone="0649614591", email="panleicim@gmail.com", + # message="SUCCESS", firstName="Lei", lastName="PAN", url='https://api.ipify.org') result_list = [] - # for appointment in collection.stream(): - # reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) - # result_list.append(reserve_pojo) - result_list.append(result_pojo) + for appointment in collection.stream(): + reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) + result_list.append(reserve_pojo) + # result_list.append(result_pojo) # for result in result_list: # if result.accepted is None or ResultEnum.PENDING.value == result.accepted: # ResultChecker().run(result, collection) # else: # print("status is " + result.accepted) - with ThreadPoolExecutor(max_workers=1) as executor: + with ThreadPoolExecutor(max_workers=5) as executor: for reserve in result_list: count = count + 1 if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted: diff --git a/db/DbManager.py b/db/DbManager.py index 6799b33..c448813 100644 --- a/db/DbManager.py +++ b/db/DbManager.py @@ -30,7 +30,7 @@ class DataManager: return sim_info_collection def get_all_successful_items(self): - return self.get_all_successful_items_for_day(str(datetime.date.today())) + return self.get_all_successful_items_for_day(str(datetime.date.today()), None) def get_all_successful_items_for_day(self, day, source_from: Union[str, None]): doc_ref = self._db.collection(day)