From fe0e144a1acedd07ba1d25821db521ad3e19d3b8 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Thu, 16 Jun 2022 12:57:10 +0200 Subject: [PATCH] can retry the contacts in error --- appointment.py | 52 +++++++++++++++++++++++---------------- db/local_db_manager.py | 8 ------ db/mongo_manager.py | 37 ++++++++++++++++++++++++++-- main.py | 11 ++++++--- workers/commandor_page.py | 30 ++++++++++------------ 5 files changed, 86 insertions(+), 52 deletions(-) diff --git a/appointment.py b/appointment.py index f573bee..a721a47 100644 --- a/appointment.py +++ b/appointment.py @@ -1,7 +1,7 @@ import PySimpleGUI as sg # First the window layout in 2 columns -from main import start_book +from main import start_book, recheck_the_captcha_error_contacts from pojo.ModeEnum import ModeEnum from proxy.proxy_type import ProxyType @@ -20,13 +20,15 @@ KEY_MANUAL = "KEY_MANUAL" GROUP_MODE = "GROUP_MODE" GROUP_STORE = "STORE" GROUP_PROXY = "GROUP_PROXY" +KEY_RUN_ERROR_CONTACT = "KEY_RUN_ERROR_CONTACT" file_list_column = [ [sg.Text('请输入联系人表的起始和结束行')], [sg.Text('起始行', size=(15, 1)), sg.InputText(tooltip="起始行", key=KEY_START_NUMBER)], [sg.Text('结束行', size=(15, 1)), sg.InputText(tooltip="结束行", key=KEY_END_NUMBER)], [sg.Text('并发数', size=(15, 1)), sg.InputText(tooltip="结束行", key=KEY_MAX_WORKERS, default_text=10)], - [sg.Submit(button_text="运行", key="submit")] + [sg.Submit(button_text="运行", key="submit")], + [sg.Submit(button_text="重约发生错误的约会", key=KEY_RUN_ERROR_CONTACT)] ] # For now will only show the name of the file that was chosen store_settings_column = [ @@ -59,34 +61,42 @@ layout = [ window = sg.Window("爱马仕约会", layout) + +def on_no_contact_found(): + sg.popup("对应联系人列表为空") + + while True: event, values = window.read() print(event) print(values) + # try: + max_workers = int(values[KEY_MAX_WORKERS]) + store_type = 0 + mode = ModeEnum.MANUAL + if values[KEY_FAUBOURG]: + store_type = 1 + elif values[KEY_GEORGE]: + store_type = 2 + elif values[KEY_SEVRES]: + store_type = 3 + + proxy_type = ProxyType.BRIGHT_DATA + if values[KEY_PROXY_NETNUT]: + proxy_type = ProxyType.NETNUT + elif values[KEY_PROXY_BRIGHTDATA]: + proxy_type = ProxyType.BRIGHT_DATA + + if values[KEY_AUTOMATIC]: + mode = ModeEnum.AUTOMATIC if event == "submit": - # try: start_line = int(values[KEY_START_NUMBER]) end_line = int(values[KEY_END_NUMBER]) - max_workers = int(values[KEY_MAX_WORKERS]) - store_type = 0 - mode = ModeEnum.MANUAL - if values[KEY_FAUBOURG]: - store_type = 1 - elif values[KEY_GEORGE]: - store_type = 2 - elif values[KEY_SEVRES]: - store_type = 3 - - proxy_type = ProxyType.BRIGHT_DATA - if values[KEY_PROXY_NETNUT]: - proxy_type = ProxyType.NETNUT - elif values[KEY_PROXY_BRIGHTDATA]: - proxy_type = ProxyType.BRIGHT_DATA - - if values[KEY_AUTOMATIC]: - mode = ModeEnum.AUTOMATIC start_book(start_line, end_line, store_choose_state=store_type, max_workers=max_workers, proxy_type=proxy_type, mode=mode) + elif event == KEY_RUN_ERROR_CONTACT: + recheck_the_captcha_error_contacts(max_workers=max_workers, store_type=store_type, mode=mode, + on_no_contact_found=on_no_contact_found) elif event == "Exit" or event == sg.WIN_CLOSED: break diff --git a/db/local_db_manager.py b/db/local_db_manager.py index be79267..f6879fd 100644 --- a/db/local_db_manager.py +++ b/db/local_db_manager.py @@ -5,7 +5,6 @@ from sqlalchemy import MetaData, Column, String, Integer, DateTime, Table from sqlalchemy.orm import Session from sqlalchemy_utils import database_exists, create_database -import definitions import params from pojo.ReserveResultPojo import ReserveResultPojo from pojo.captcha_error_contact_pojo import ContactInErrorPojo, ERROR_TYPE_CAPTCHA @@ -49,15 +48,8 @@ class LocalDbManager: self.session.merge(instance) self.session.commit() - def get_all_captcha_error_contacts(self) -> list: - return self.session.query(ContactInErrorPojo).filter(ContactInErrorPojo.error_type == ERROR_TYPE_CAPTCHA).all() - def handle_success(self, reservePojo: ReserveResultPojo): # delete the contact from table self.session.query(ContactInErrorPojo).filter(ContactInErrorPojo.mail == reservePojo.email).delete() -if __name__ == '__main__': - list = params.local_db_manager.get_all_captcha_error_contacts() - for item in list: - print(item.mail) diff --git a/db/mongo_manager.py b/db/mongo_manager.py index cdaf636..9568d81 100644 --- a/db/mongo_manager.py +++ b/db/mongo_manager.py @@ -1,9 +1,12 @@ +import datetime import logging from pymongo import MongoClient from pojo.ReserveResultPojo import ReserveResultPojo +from pojo.contact_pojo import ContactPojo -MONGO_DB_URL = "91.121.210.60" +MONGO_DB_URL = "mongo.lpaconsulting.fr" +CAPTCHA_ERROR_COLLECTION_PREFIX = "CAPTCHA_ERROR_" class MongoDbManager: @@ -22,8 +25,38 @@ class MongoDbManager: collection_to_use.replace_one(filter={'_id': reserve.id, }, replacement=reserve.to_firestore_dict(), upsert=True) except Exception as Error: - print(Error) + self.logger.info(Error) + + def insert_captcha_error_contact(self, contact: ContactPojo): + day = str(datetime.date.today()) + collection_name = CAPTCHA_ERROR_COLLECTION_PREFIX + day + try: + collection_to_use = self.db[collection_name] + collection_to_use.replace_one(filter={'_id': contact.mail, }, replacement=contact.to_firestore_dict(), + upsert=True) + except Exception as error: + self.logger.info(error) + + def get_captcha_error_contacts_for_current_day(self) -> list: + day = str(datetime.date.today()) + collection_name = CAPTCHA_ERROR_COLLECTION_PREFIX + day + cursor = self.db[collection_name] + contact_list = [] + for document in cursor.find(): + contact_list.append(ContactPojo.from_firestore_dict(document)) + return contact_list + + def delete_captcha_error_contact_for_current_day(self, contact: ContactPojo): + day = str(datetime.date.today()) + collection_name = CAPTCHA_ERROR_COLLECTION_PREFIX + day + collection = self.db[collection_name] + to_delete = {'_id': contact.mail} + try: + collection.delete_one(to_delete) + except Exception as error: + self.logger.info(error) if __name__ == '__main__': db_manager = MongoDbManager() + print(db_manager.get_captcha_error_contacts_for_current_day()) diff --git a/main.py b/main.py index 2886742..b42cfc9 100644 --- a/main.py +++ b/main.py @@ -34,10 +34,13 @@ def start_book(start_number, end_number, store_choose_state=0, max_workers=10, p proxy) -def recheck_the_captcha_error_contacts(store_type=0, mode: ModeEnum = ModeEnum.MANUAL): +def recheck_the_captcha_error_contacts(on_no_contact_found, store_type=0, mode: ModeEnum = ModeEnum.MANUAL, + max_workers=10): # get all the contacts in captcha error - contact_list = params.local_db_manager.get_all_captcha_error_contacts() - with ThreadPoolExecutor(max_workers=10) as executor: + contact_list = params.mongo_store_manager.get_captcha_error_contacts_for_current_day() + if len(contact_list) == 0: + on_no_contact_found() + with ThreadPoolExecutor(max_workers=max_workers) as executor: for errorContact in contact_list: contact = ContactPojo.get_contact_from_error_contact(errorContact) proxy = get_proxy() @@ -54,4 +57,4 @@ def get_proxy(proxy_type=ProxyType.BRIGHT_DATA): if __name__ == '__main__': # 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机，传入1的时候是总店 # start_book(16, 16, store_choose_state=0, proxy_type=0) - recheck_the_captcha_error_contacts(0, mode=ModeEnum.AUTOMATIC) + recheck_the_captcha_error_contacts(store_type=0, mode=ModeEnum.AUTOMATIC, on_no_contact_found=lambda: None) diff --git a/workers/commandor_page.py b/workers/commandor_page.py index 6fc6d2c..6b0f15a 100644 --- a/workers/commandor_page.py +++ b/workers/commandor_page.py @@ -127,9 +127,9 @@ class CommandorPage: self.logger.info("填充信息: " + str(self.contact.phone)) self._set_name(self.contact.last_name, self.contact.first_name) self._setPhoneCountryAndStore() - self._setPhoneNumber("0" + str(self.contact.phone)) + self._set_phone_number("0" + str(self.contact.phone)) self._set_email(self.contact.mail) - self.setIdNumber(self.contact.passport) + self._set_id_number(self.contact.passport) self._checkCgu() if self.appointment_mode == ModeEnum.AUTOMATIC: self.resolve_captcha() @@ -205,7 +205,7 @@ class CommandorPage: except Exception as error: self.logger.error(error) - def _setPhoneNumber(self, phoneNumber): + def _set_phone_number(self, phoneNumber): time.sleep(get_random_wait_time()) try: self.page.evaluate("""(phoneNumber)=>document.getElementById("phone_number").value =phoneNumber""", @@ -264,9 +264,7 @@ class CommandorPage: self.is_captcha_in_error = True if not self.is_finished: # save the error to database with contact info - params.local_db_manager.insert_or_update( - get_captcha_error_contact_from_contact(self.contact, ERROR_TYPE_CAPTCHA)) - params.oracle_log_sender.send_captcha_error(self.contact) + self.handle_captcha_error() self.is_finished = True # no need to retry captcha, if retry ,will generate DOUBLE_REQUEST_ERROR_MESSAGE self.termine() @@ -283,7 +281,7 @@ class CommandorPage: except Exception as error: self.logger.error(error) - def setIdNumber(self, id): + def _set_id_number(self, id): time.sleep(get_random_wait_time()) try: self.page.evaluate(""" (id) =>{ @@ -332,6 +330,7 @@ class CommandorPage: result.store_type = self.store_type params.firebase_store_manager.save(result) params.local_db_manager.handle_success(result) + params.mongo_store_manager.delete_captcha_error_contact_for_current_day(self.contact) if status is PublishType.SUCCESS: self.on_success(result) time.sleep(2) @@ -360,13 +359,11 @@ class CommandorPage: self.logger.error(error) self.page.reload(timeout=PAGE_TIMEOUT) - -def get_random_id_number() -> str: - S = 8 # number of characters in the string. - ran = ''.join(random.choices(string.digits, k=S)) - id_number = "57" + str(ran) - print("The randomly generated string is : 94" + str(ran)) # print the random data - return id_number + def handle_captcha_error(self): + params.local_db_manager.insert_or_update( + get_captcha_error_contact_from_contact(self.contact, ERROR_TYPE_CAPTCHA)) + params.mongo_store_manager.insert_captcha_error_contact(self.contact) + params.oracle_log_sender.send_captcha_error(self.contact) def on_success(result: ReserveResultPojo): @@ -374,12 +371,11 @@ def on_success(result: ReserveResultPojo): def launch_page(): - passport_number = get_random_id_number() - contact = ContactPojo(phone_number="+33758912245", passport_number=passport_number, last_name="XU", + contact = ContactPojo(phone_number="+33758912245", passport_number="82546975", last_name="XU", first_name="xingzhen", mail="ColbyPatel653@gmail.com", ccid="", position=0) page = CommandorPage(contact, store_type=1) - return page.start_page(params.get_proxy()) + return page.start_page(params.get_proxy(ProxyType.BRIGHT_DATA)) def wait_for_otp(event: threading.Event, commandor: CommandorPage):