diff --git a/check_results.py b/check_results.py index 8d15d92..0bcdd26 100644 --- a/check_results.py +++ b/check_results.py @@ -41,7 +41,7 @@ class ResultChecker: device = random.choice(params.DEVICES) self.logger.info("模拟设备: " + device) pixel_2 = self.tls.playwright.devices[device] - context = self.browser.new_context(**pixel_2, locale='en-GB') + context = self.browser.new_context(**pixel_2, locale='fr-FR') self.page = context.new_page() # hide webdriver information self.page.add_init_script("""() => { @@ -66,9 +66,6 @@ class ResultChecker: # url_to_check = url + "?lang=fr" print("url is " + url) content = None - random_id_number = str(phone_number)[1:len(str(phone_number))] - proxy_username = "panleicim-res-fr-" + random_id_number - print("proxy_username is " + proxy_username) proxy = { "server": params.BRIGHT_DATA_PROXY_SERVER, "username": params.BRIGHT_DATA_PROXY_USERNAME, @@ -123,7 +120,7 @@ if __name__ == '__main__': reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) result_list.append(reserve_pojo) - with ThreadPoolExecutor(max_workers=10) as executor: + with ThreadPoolExecutor(max_workers=15) as executor: for reserve in result_list: count = count + 1 if reserve.accepted is None or ResultEnum.ACCEPTED.value == reserve.accepted: diff --git a/db/local_db_manager.py b/db/local_db_manager.py index 25c075d..be79267 100644 --- a/db/local_db_manager.py +++ b/db/local_db_manager.py @@ -1,9 +1,14 @@ +from builtins import list + import sqlalchemy from sqlalchemy import MetaData, Column, String, Integer, DateTime, Table from sqlalchemy.orm import Session from sqlalchemy_utils import database_exists, create_database -from pojo.captcha_error_contact_pojo import ContactInErrorPojo +import definitions +import params +from pojo.ReserveResultPojo import ReserveResultPojo +from pojo.captcha_error_contact_pojo import ContactInErrorPojo, ERROR_TYPE_CAPTCHA class LocalDbManager: @@ -12,7 +17,7 @@ class LocalDbManager: self.session = Session(self.init_db(path)) def init_db(self, path: str): - uri_for_db = "sqlite:///{}/{}.db".format(path, "appointment") + uri_for_db = "sqlite:///{}/{}.db?check_same_thread=false".format(path, "appointment") print(uri_for_db) # 2.-Turn on database engine db_engine = sqlalchemy.create_engine(uri_for_db) # ensure this is the correct path for the sqlite file. @@ -43,3 +48,16 @@ class LocalDbManager: def insert_or_update(self, instance: ContactInErrorPojo): self.session.merge(instance) self.session.commit() + + def get_all_captcha_error_contacts(self) -> list: + return self.session.query(ContactInErrorPojo).filter(ContactInErrorPojo.error_type == ERROR_TYPE_CAPTCHA).all() + + def handle_success(self, reservePojo: ReserveResultPojo): + # delete the contact from table + self.session.query(ContactInErrorPojo).filter(ContactInErrorPojo.mail == reservePojo.email).delete() + + +if __name__ == '__main__': + list = params.local_db_manager.get_all_captcha_error_contacts() + for item in list: + print(item.mail) diff --git a/definitions.py b/definitions.py index d6a91f3..447ef16 100644 --- a/definitions.py +++ b/definitions.py @@ -2,9 +2,6 @@ import configparser import getpass import os from pathlib import Path - -from db.local_db_manager import LocalDbManager - home = str(Path.home()) config = configparser.ConfigParser() print("home path: " + home) @@ -17,5 +14,4 @@ FIREBASE_CONFIG_FILE = config['DEFAULT']['firebase_config_file'] LOGS_DIR = config['DEFAULT']['LOGS_DIR'] username = getpass.getuser() LOG_SOURCE = username -local_db_manager = LocalDbManager(home) ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/main.py b/main.py index 1f1b0a9..f36a574 100644 --- a/main.py +++ b/main.py @@ -5,6 +5,7 @@ from concurrent.futures import ThreadPoolExecutor import params from logs.AppLogging import init_logger from pojo.ModeEnum import ModeEnum +from pojo.contact_pojo import ContactPojo from utils.excel_reader import ExcelHelper from workers.commandor_page import CommandorPage @@ -25,26 +26,31 @@ def start_book(start_number, end_number, store_choose_state=0, max_workers=10, p logger.info(contacts) with ThreadPoolExecutor(max_workers=max_workers) as executor: for contact in contacts: - proxy = get_proxy(contact.phone, proxy_type) + proxy = get_proxy(proxy_type) # start the task in thread executor.submit( CommandorPage(contact, store_type=store_choose_state, proxy_type=proxy_type, mode=mode).start_page, proxy) -def get_proxy(phone_number, proxy_type=0): - # random_id_number = str(phone_number)[1:len(str(phone_number))] - random_id_number = params.get_random_id_number_for_proxy() - proxy_username = params.get_proxy_name_prefix(proxy_type) + random_id_number - logger.info("proxy_username is " + proxy_username) - proxy = { - "server": params.BRIGHT_DATA_PROXY_SERVER, - "username": params.BRIGHT_DATA_PROXY_USERNAME, - "password": params.BRIGHT_DATA_PROXY_PASSWORD - } - return proxy +def recheck_the_captcha_error_contacts(store_type=0, mode: ModeEnum = ModeEnum.MANUAL): + # get all the contacts in captcha error + list = params.local_db_manager.get_all_captcha_error_contacts() + with ThreadPoolExecutor(max_workers=10) as executor: + for errorContact in list: + contact = ContactPojo.get_contact_from_error_contact(errorContact) + proxy = get_proxy() + # start the task in thread + executor.submit( + CommandorPage(contact, store_type=store_type, mode=mode).start_page, + proxy) + + +def get_proxy(proxy_type=0): + return params.get_proxy(proxy_type) if __name__ == '__main__': # 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机，传入1的时候是总店 - start_book(16, 16, store_choose_state=0, proxy_type=0) + # start_book(16, 16, store_choose_state=0, proxy_type=0) + recheck_the_captcha_error_contacts(0, mode=ModeEnum.AUTOMATIC) diff --git a/params.py b/params.py index 50ebce8..c717864 100644 --- a/params.py +++ b/params.py @@ -1,7 +1,9 @@ import random import string +import definitions from db.DbManager import DataManager +from db.local_db_manager import LocalDbManager from logs.LogSender import LogSender firebase_store_manager = DataManager() @@ -25,6 +27,24 @@ def get_proxy_name_prefix(proxy_type=0) -> str: return PROXY_NAME_PREFIX_CC +def get_proxy(proxy_type=0): + if proxy_type == 0: + random_id_number = get_random_id_number_for_proxy() + proxy_username = get_proxy_name_prefix(proxy_type) + random_id_number + proxy = { + "server": PROXY_SERVER, + "username": proxy_username, + "password": PROXY_PASSWORD + } + else: + proxy = { + "server": BRIGHT_DATA_PROXY_SERVER, + "username": BRIGHT_DATA_PROXY_USERNAME, + "password": BRIGHT_DATA_PROXY_PASSWORD + } + return proxy + + def get_random_id_number_for_proxy() -> str: S = 8 # number of characters in the string. ran = ''.join(random.choices(string.digits, k=S)) @@ -33,6 +53,8 @@ def get_random_id_number_for_proxy() -> str: return id_number +local_db_manager = LocalDbManager(definitions.home) + DEVICES = ['iPad (gen 6)', 'iPad (gen 6) landscape', 'iPad (gen 7)', 'iPad (gen 7) landscape', 'iPad Mini', 'iPad Mini landscape', 'iPad Pro 11', 'iPad Pro 11 landscape', 'iPhone 6', 'iPhone 6 landscape', 'iPhone 6 Plus', 'iPhone 6 Plus landscape', 'iPhone 7', 'iPhone 7 landscape', 'iPhone 7 Plus', diff --git a/pojo/captcha_error_contact_pojo.py b/pojo/captcha_error_contact_pojo.py index e11f5c9..1b155d8 100644 --- a/pojo/captcha_error_contact_pojo.py +++ b/pojo/captcha_error_contact_pojo.py @@ -1,8 +1,6 @@ from sqlalchemy import Column, String, Integer, DateTime, func from sqlalchemy.orm import declarative_base -import definitions -from pojo.contact_pojo import ContactPojo Base = declarative_base() @@ -22,23 +20,4 @@ class ContactInErrorPojo(Base): position: int = Column(Integer) error_type = Column(Integer) update_at = Column(DateTime, onupdate=func.now()) - create_at = Column(DateTime, default=func.now()) - - -def get_captcha_error_contact_from_contact(contact: ContactPojo, error_type: int) -> ContactInErrorPojo: - captcha_error = ContactInErrorPojo() - captcha_error.mail = contact.mail - captcha_error.ccid = contact.ccid - captcha_error.phone = contact.phone - captcha_error.passport = contact.passport - captcha_error.first_name = contact.first_name - captcha_error.last_name = contact.last_name - captcha_error.position = contact.position - captcha_error.error_type = error_type - return captcha_error - - -if __name__ == '__main__': - contact = ContactPojo(mail="panleici3m@gmail.com", phone_number="649114592", ccid="", position=0, - passport_number="3322111", first_name="Lei", last_name="PAAaN") - definitions.local_db_manager.insert_or_update(get_captcha_error_contact_from_contact(contact, ERROR_TYPE_CAPTCHA)) + create_at = Column(DateTime, default=func.now()) \ No newline at end of file diff --git a/pojo/contact_pojo.py b/pojo/contact_pojo.py index 77ef414..de491c8 100644 --- a/pojo/contact_pojo.py +++ b/pojo/contact_pojo.py @@ -1,5 +1,7 @@ from dataclasses import dataclass +from pojo.captcha_error_contact_pojo import ContactInErrorPojo + @dataclass class ContactPojo: @@ -35,6 +37,12 @@ class ContactPojo: return dest + @staticmethod + def get_contact_from_error_contact(errorContact: ContactInErrorPojo): + return ContactPojo(phone_number=errorContact.phone, mail=errorContact.mail, ccid=errorContact.ccid, + last_name=errorContact.last_name, first_name=errorContact.first_name, + position=errorContact.position, passport_number=errorContact.passport) + @staticmethod def from_firestore_dict(source): ccid = source['ccid'] diff --git a/utils/generate_random_passport_id.py b/utils/generate_random_passport_id.py index a892600..66f5d2c 100644 --- a/utils/generate_random_passport_id.py +++ b/utils/generate_random_passport_id.py @@ -1,6 +1,9 @@ import random import string +from pojo.captcha_error_contact_pojo import ContactInErrorPojo +from pojo.contact_pojo import ContactPojo + letters = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'P', 'Q', 'R', 'S', 'T', '1', '2', '3', '4', '5', '6', '7', '8', '9'] @@ -19,6 +22,18 @@ def get_random_passport_id_number() -> str: return id_number +def get_captcha_error_contact_from_contact(contact: ContactPojo, error_type: int) -> ContactInErrorPojo: + captcha_error = ContactInErrorPojo() + captcha_error.mail = contact.mail + captcha_error.ccid = contact.ccid + captcha_error.phone = contact.phone + captcha_error.passport = contact.passport + captcha_error.first_name = contact.first_name + captcha_error.last_name = contact.last_name + captcha_error.position = contact.position + captcha_error.error_type = error_type + return captcha_error + if __name__ == '__main__': # for i in range(1,200): # print(get_random_id_number()) diff --git a/workers/commandor_page.py b/workers/commandor_page.py index d206aa7..ac7f9f8 100644 --- a/workers/commandor_page.py +++ b/workers/commandor_page.py @@ -12,9 +12,10 @@ import definitions import params from pojo.ModeEnum import ModeEnum from pojo.ReserveResultPojo import ReserveResultPojo, PublishType -from pojo.captcha_error_contact_pojo import get_captcha_error_contact_from_contact, ERROR_TYPE_CAPTCHA, \ +from pojo.captcha_error_contact_pojo import ERROR_TYPE_CAPTCHA, \ TOO_MANY_REQUEST_ERROR from pojo.contact_pojo import ContactPojo +from utils.generate_random_passport_id import get_captcha_error_contact_from_contact from workers.SolveCaptch import SolveCaptcha RDV_URL = "https://rendezvousparis.hermes.com/client/register" @@ -32,7 +33,7 @@ CONFIRMED_MESSAGE_FR = "Votre demande de rendez-vous Maroquinerie a bien été e DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today." DOUBLE_REQUEST_ERROR_MESSAGE_FR = "Une demande avec les données saisies a déjà été validée aujourd’hui." TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests" -TOO_MANY_REQUEST_ERROR_MESSAGE_FR = "Suite à un trop grand nombre de demandes aujourd’hui," +TOO_MANY_REQUEST_ERROR_MESSAGE_FR = "Suite à un trop grand nombre de demandes" CAPTCHA_ERROR_MESSAGE = "Error verifying captcha, please try again" CAPTCHA_ERROR_MESSAGE_FR = "La vérification du captcha a échoué" TIME_OUT = 400000 @@ -98,11 +99,7 @@ class CommandorPage: first_page = None while first_page is None: first_page = self.start_browser(proxy, self.tls.playwright, devices) - proxy = { - "server": params.BRIGHT_DATA_PROXY_SERVER, - "username": params.BRIGHT_DATA_PROXY_USERNAME, - "password": params.BRIGHT_DATA_PROXY_PASSWORD - } + proxy = params.get_proxy(self.proxy_type) # wait for sms_code field # self.clickOnValidBtn() self.thread_event = e @@ -132,7 +129,7 @@ class CommandorPage: self.logger.info("填充信息: " + str(self.contact.phone)) self._set_name(self.contact.last_name, self.contact.first_name) self._setPhoneCountryAndStore() - self._setPhoneNumber(self.contact.phone) + self._setPhoneNumber("0" + str(self.contact.phone)) self._set_email(self.contact.mail) self.setIdNumber(self.contact.passport) self._checkCgu() @@ -255,7 +252,7 @@ class CommandorPage: elif TOO_MANY_REQUEST_ERROR_MESSAGE in erro_content or TOO_MANY_REQUEST_ERROR_MESSAGE_FR in erro_content: # this email has been already used if not self.is_finished: - definitions.local_db_manager.insert_or_update( + params.local_db_manager.insert_or_update( get_captcha_error_contact_from_contact(self.contact, TOO_MANY_REQUEST_ERROR)) params.oracle_log_sender.send_too_many_error(self.contact) self.is_finished = True @@ -265,7 +262,7 @@ class CommandorPage: self.is_captcha_in_error = True if not self.is_finished: # save the error to database with contact info - definitions.local_db_manager.insert_or_update( + params.local_db_manager.insert_or_update( get_captcha_error_contact_from_contact(self.contact, ERROR_TYPE_CAPTCHA)) params.oracle_log_sender.send_captcha_error(self.contact) self.is_finished = True @@ -331,6 +328,7 @@ class CommandorPage: passport=contact.passport, ccid=contact.ccid) result.id = id params.firebase_store_manager.save(result) + params.local_db_manager.handle_success(result) if status is PublishType.SUCCESS: self.on_success(result) time.sleep(2)