diff --git a/appointment.py b/appointment.py index c505dbf..9e07a1a 100644 --- a/appointment.py +++ b/appointment.py @@ -2,6 +2,7 @@ import PySimpleGUI as sg # First the window layout in 2 columns from main import start_book +from pojo.ModeEnum import ModeEnum KEY_CHOOSE_STORE = "CHOOSE_STORE" KEY_START_NUMBER = "KEY_START_NUMBER" @@ -13,6 +14,9 @@ KEY_PROXY_CC = "KEY_PROXY_CC" KEY_FAUBOURG = "KEY_FAUBOURG" KEY_GEORGE = "KEY_GEORGE" KEY_SEVRES = "KEY_SEVRES" +KEY_AUTOMATIC = "KEY_AUTOMATIC" +KEY_MANUAL = "KEY_MANUAL" +GROUP_MODE = "GROUP_MODE" GROUP_STORE = "STORE" GROUP_PROXY = "GROUP_PROXY" @@ -36,12 +40,19 @@ proxy_settings_column = [ [sg.Radio('res(速度)', group_id=GROUP_PROXY, key=KEY_PROXY_RES, default=True)], [sg.Radio('cc(稳定)', group_id=GROUP_PROXY, key=KEY_PROXY_CC, default=False)], ] + +mode_settings_column = [ + [sg.Text("约会模式")], + [sg.Radio('手动', group_id=GROUP_MODE, key=KEY_MANUAL, default=True)], + [sg.Radio('自动', group_id=GROUP_MODE, key=KEY_AUTOMATIC, default=False)], +] # ----- Full layout ----- layout = [ [ sg.Column(file_list_column), sg.Column(store_settings_column), - sg.Column(proxy_settings_column) + sg.Column(proxy_settings_column), + sg.Column(mode_settings_column) ] ] @@ -57,6 +68,7 @@ while True: end_line = int(values[KEY_END_NUMBER]) max_workers = int(values[KEY_MAX_WORKERS]) store_type = 0 + mode = ModeEnum.MANUAL if values[KEY_FAUBOURG]: store_type = 1 elif values[KEY_GEORGE]: @@ -69,7 +81,11 @@ while True: proxy_type = 1 elif values[KEY_PROXY_RES]: proxy_type = 0 - start_book(start_line, end_line, store_choose_state=store_type, max_workers=max_workers, proxy_type=proxy_type) + + if values[KEY_AUTOMATIC]: + mode = ModeEnum.AUTOMATIC + start_book(start_line, end_line, store_choose_state=store_type, max_workers=max_workers, proxy_type=proxy_type, + mode=mode) # except Exception as error: # print("Not Integer: ") # print(error) diff --git a/check_results.py b/check_results.py index 39ad769..80f0041 100644 --- a/check_results.py +++ b/check_results.py @@ -21,7 +21,7 @@ BLANK_URL = "about:blank" class TlsPlaywright(threading.local): def __init__(self) -> None: self.playwright = sync_playwright().start() - print("Create playwright instance in Thread", threading.current_thread().name) + print("创建浏览器实例，线程: ", threading.current_thread().name) class ResultChecker: @@ -108,7 +108,7 @@ if __name__ == '__main__': # get the list params.oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS) db_manager = params.firebase_store_manager - # collection = db_manager.get_all_successful_items_for_day("2022-05-14", "landd") + # collection = db_manager.get_successful_item_for_day_by_status("2022-05-17", ResultEnum.ACCEPTED) collection = db_manager.get_all_successful_items() count = 0 result_list = [] @@ -116,12 +116,12 @@ if __name__ == '__main__': reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) result_list.append(reserve_pojo) - with ThreadPoolExecutor(max_workers=10) as executor: + + with ThreadPoolExecutor(max_workers=25) as executor: for reserve in result_list: count = count + 1 - if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted: - if reserve.url != BLANK_URL: - executor.submit(ResultChecker().run, reserve, collection) + if reserve.accepted is None or ResultEnum.ACCEPTED.value == reserve.accepted: + executor.submit(ResultChecker().run, reserve, collection) else: print("status is " + reserve.accepted) diff --git a/db/DbManager.py b/db/DbManager.py index c448813..711eb19 100644 --- a/db/DbManager.py +++ b/db/DbManager.py @@ -5,6 +5,7 @@ import firebase_admin from firebase_admin import credentials, firestore import definitions +from pojo import ResultEnum from pojo.MailPojo import MailPojo from pojo.ReserveResultPojo import ReserveResultPojo from pojo.SimInfoPojo import SimInfoPojo @@ -38,6 +39,12 @@ class DataManager: doc_ref.where(u'source_from', u'==', source_from) return doc_ref + def get_successful_item_for_day_by_status(self, day, status: ResultEnum): + doc_ref = self._db.collection(day) + if status is not None: + doc_ref.where(u'accepted', u'==', status.value) + return doc_ref + def save_sim_info(self, sim_info: SimInfoPojo): doc_ref = self._db.collection(SIM_INFOS).document(sim_info.phone) doc_ref.set(sim_info.to_firestore_dict()) diff --git a/docs/rdv.html b/docs/rdv.html new file mode 100644 index 0000000..c6d47ee --- /dev/null +++ b/docs/rdv.html @@ -0,0 +1,254 @@ + + + + + reCAPTCHA + + + + + + +
+ + +
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+ +
PrivacyTerms
+
+
+ + + \ No newline at end of file diff --git a/logs/LogSender.py b/logs/LogSender.py index 477cf38..6e4f614 100644 --- a/logs/LogSender.py +++ b/logs/LogSender.py @@ -25,6 +25,7 @@ LOG_ERROR = "ERROR" LOG_TYPE_INFO = "INFO" LOG_ERROR_TYPE_DOUBLE_DATA = "DOUBLE_REQUEST_FOR_SAME_DATA" LOG_ERROR_TOO_MANY_REQUEST_TODAY = "TOO_MANY_REQUEST_TODAY" +LOG_ERROR_CAPTCHA_ERROR_MESSAGE = "CAPTCHA_ERROR" LOG_SUBJECT_ERROR = "ERROR" LOG_APPOINTMENT_ERROR = "APPOINTMENT_ERROR" LOG_APPOINTMENT_TIMEOUT = "TIMEOUT" @@ -78,6 +79,10 @@ class LogSender: error_msg = contact.mail self.send_log(msg=error_msg, type=LOG_ERROR_TOO_MANY_REQUEST_TODAY, subject=LOG_SUBJECT_ERROR) + def send_captcha_error(self, contact: ContactPojo): + error_msg = contact.mail + self.send_log(msg=error_msg, type=LOG_ERROR_CAPTCHA_ERROR_MESSAGE, subject=LOG_SUBJECT_ERROR) + def send_appoint_result(self, result: ReserveResultPojo): if result.type == PublishType.SUCCESS: # get id diff --git a/main.py b/main.py index 28397d6..e05c530 100644 --- a/main.py +++ b/main.py @@ -4,6 +4,7 @@ from concurrent.futures import ThreadPoolExecutor import params from logs.AppLogging import init_logger +from pojo.ModeEnum import ModeEnum from utils.excel_reader import ExcelHelper from workers.commandor_page import CommandorPage @@ -13,7 +14,8 @@ logger = logging.getLogger() logger.addHandler(logging.StreamHandler(stream=sys.stdout)) -def start_book(start_number, end_number, store_choose_state=0, max_workers=10, proxy_type=0): +def start_book(start_number, end_number, store_choose_state=0, max_workers=10, proxy_type=0, + mode: ModeEnum = ModeEnum.MANUAL): # read the contact, and contact the 2 objects together excel_reader = ExcelHelper() all_contacts = excel_reader.read_contacts() @@ -25,8 +27,9 @@ def start_book(start_number, end_number, store_choose_state=0, max_workers=10, p for contact in contacts: proxy = get_proxy(contact.phone, proxy_type) # start the task in thread - executor.submit(CommandorPage(contact, store_type=store_choose_state, proxy_type=proxy_type).start_page, - proxy) + executor.submit( + CommandorPage(contact, store_type=store_choose_state, proxy_type=proxy_type, mode=mode).start_page, + proxy) def get_proxy(phone_number, proxy_type=0): diff --git a/params.py b/params.py index 0fc1bf0..c3f32f7 100644 --- a/params.py +++ b/params.py @@ -1,9 +1,6 @@ -import configparser import random import string -from playwright.sync_api import sync_playwright - from db.DbManager import DataManager from logs.LogSender import LogSender @@ -22,6 +19,7 @@ def get_proxy_name_prefix(proxy_type = 0) -> str: else: return PROXY_NAME_PREFIX_CC + def get_random_id_number_for_proxy() -> str: S = 8 # number of characters in the string. ran = ''.join(random.choices(string.digits, k=S)) diff --git a/pojo/ModeEnum.py b/pojo/ModeEnum.py new file mode 100644 index 0000000..4a37bf2 --- /dev/null +++ b/pojo/ModeEnum.py @@ -0,0 +1,6 @@ +from enum import Enum + + +class ModeEnum(Enum): + AUTOMATIC = "AUTOMATIC" + MANUAL = "MANUAL" diff --git a/requirements.txt b/requirements.txt index b694e4b..da21495 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,5 @@ XlsxWriter~=3.0.3 boto3~=1.21.13 openpyxl==3.0.9 google-cloud-firestore==2.4.0 -PySimpleGUI==4.60.0 \ No newline at end of file +PySimpleGUI==4.60.0 +SpeechRecognition==3.8.1 \ No newline at end of file diff --git a/workers/SolveCaptch.py b/workers/SolveCaptch.py new file mode 100644 index 0000000..d8f9023 --- /dev/null +++ b/workers/SolveCaptch.py @@ -0,0 +1,54 @@ +# for recaptcha +import logging +import random +import re +import time +import requests + +CAPCHA_NOT_READY = "CAPCHA_NOT_READY" +REGEX_DATA_SITE_KEY = "data-sitekey=[\"a-z0-9A-Z]+" +API_KEY = "d66aaf490d8aa424a5175e1fbd1aadea" + + +class SolveCaptcha: + def __init__(self, page): + self.page = page + self.logger = logging.getLogger("SolveCaptcha") + self.main_frame = None + self.recaptcha = None + + def delay(self): + self.page.wait_for_timeout(random.randint(1, 3) * 1000) + + def start(self, handle_solution_received): + self.logger.info("start to resolve captcha") + content = self.page.content() + data_sitekey = re.findall(REGEX_DATA_SITE_KEY, content) + self.logger.info(data_sitekey) + if len(data_sitekey) == 1: + key_with_comma = data_sitekey[0].split("=")[-1] + key = key_with_comma.replace("\"", '') + self.logger.info("key is : " + key) + self.solve_captcha(key, handle_solution_received) + + def solve_captcha(self, google_key: str, handle_solution_received): + self.logger.info("solve_captcha()") + url_get = "http://2captcha.com/in.php?key={}&method=userrecaptcha&googlekey={}&pageurl={}".format(API_KEY, + google_key, + self.page.url) + res = requests.get(url_get) + self.logger.info(res.text) + results = res.text.split("|") + self.captcha_id = results[-1] + # wait for 15 seconds + time.sleep(15) + # get result of the captcha + url_response = "http://2captcha.com/res.php?key={}&action=get&id={}".format(API_KEY, + self.captcha_id) + solution = CAPCHA_NOT_READY + while solution == CAPCHA_NOT_READY: + solution_res = requests.get(url_response) + time.sleep(5) + solution = solution_res.text + print(solution) + handle_solution_received(solution.split("|")[-1]) diff --git a/workers/commandor_page.py b/workers/commandor_page.py index 626434a..70314ed 100644 --- a/workers/commandor_page.py +++ b/workers/commandor_page.py @@ -10,8 +10,10 @@ from playwright.sync_api import sync_playwright import params from params import PROXY_SERVER, PROXY_PASSWORD +from pojo.ModeEnum import ModeEnum from pojo.ReserveResultPojo import ReserveResultPojo, PublishType from pojo.contact_pojo import ContactPojo +from workers.SolveCaptch import SolveCaptcha RDV_URL = "https://rendezvousparis.hermes.com/client/register" @@ -26,9 +28,10 @@ MESSAGE_FIELD_CLASS = ".message" CONFIRMED_MESSAGE = "Your request for a Leather Goods appointment has been registered" DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today." TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests" +CAPTCHA_ERROR_MESSAGE = "Error verifying captcha, please try again" TIME_OUT = 400000 OTP_TIMEOUT = 240 -PAGE_TIMEOUT = 30000 +PAGE_TIMEOUT = 40000 def get_random_wait_time() -> float: @@ -44,12 +47,16 @@ class Tls(threading.local): class CommandorPage: tls = Tls() - def __init__(self, contact: ContactPojo, store_type=0, proxy_type=0): + def __init__(self, contact: ContactPojo, store_type=0, proxy_type=0, mode: ModeEnum = ModeEnum.MANUAL): self.otp_value = None - self.logger = logging.getLogger("CommandorPage") + self.logger = logging.getLogger("CommandorPage:" + str(contact.phone)) self.is_finished = False self.contact = contact self.proxy_type = proxy_type + self.is_event_sent = False + self.is_captcha_in_error = False + self.is_filling_fields = False + self.appointment_mode = mode # 0: random # 1: faubourg # 2: George @@ -63,9 +70,12 @@ class CommandorPage: def on_success(self, result: ReserveResultPojo): self.logger.info("on_success called.") - self.logger.info(result) - params.oracle_log_sender.send_appoint_result(result) self.is_finished = True + if not self.is_event_sent: + self.logger.info("will send successful event") + self.logger.info(result) + params.oracle_log_sender.send_appoint_result(result) + self.is_event_sent = True def timeout_occurred(self): params.oracle_log_sender.send_timeout_log(self.contact) @@ -89,7 +99,6 @@ class CommandorPage: "username": proxy_username, "password": params.PROXY_PASSWORD } - self.fill_fields() # wait for sms_code field # self.clickOnValidBtn() self.thread_event = e @@ -103,7 +112,6 @@ class CommandorPage: self.clickOnValidBtn() otp_sent = self.page.locator(MESSAGE_FIELD_CLASS) otp_sent.wait_for(state='visible', timeout=TIME_OUT) - # print("message is:" + message) time.sleep(get_random_wait_time()) message = self.page.content() if CONFIRMED_MESSAGE in message: @@ -115,20 +123,25 @@ class CommandorPage: self.termine() def fill_fields(self): - self._set_name(self.contact.last_name, self.contact.first_name) - self._setPhoneCountryAndStore() - self._setPhoneNumber(self.contact.phone) - self._set_email(self.contact.mail) - self.setIdNumber(self.contact.passport) - # - self._checkCgu() + if not self.is_filling_fields: + self.is_filling_fields = True + self.logger.info("填充信息: " + str(self.contact.phone)) + self._set_name(self.contact.last_name, self.contact.first_name) + self._setPhoneCountryAndStore() + self._setPhoneNumber(self.contact.phone) + self._set_email(self.contact.mail) + self.setIdNumber(self.contact.passport) + self._checkCgu() + if self.appointment_mode == ModeEnum.AUTOMATIC: + self.resolve_captcha() + self.is_filling_fields = False def start_browser(self, proxy, pwright, device) -> Union[str, None]: try: self.browser = pwright.webkit.launch(headless=False, timeout=PAGE_TIMEOUT, proxy=proxy) self.logger.info("模拟设备: " + device) pixel_2 = pwright.devices[device] - context = self.browser.new_context(**pixel_2, locale='en-GB') + context = self.browser.new_context(**pixel_2, locale='fr-FR') self.page = context.new_page() # hide webdriver information self.page.add_init_script("""() => { @@ -154,7 +167,7 @@ class CommandorPage: pattern = re.compile(REGEX_RDV_URL) if pattern.match(response.url): self.logger.info("result url found: " + response.url) - self.publish_message_to_queue(self.contact, PublishType.PENDING, response.url) + # self.publish_message_to_queue(self.contact, PublishType.PENDING, response.url) def start_page(self, proxy): e = threading.Event() @@ -166,26 +179,29 @@ class CommandorPage: self.logger.info("url is " + self.page.url) if self.page.url == RDV_URL: self.fill_fields() - message = self.page.content() - if CONFIRMED_MESSAGE in message: - # publish the successful message - self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url) - self.get_errors() + try: + message = self.page.content() + if CONFIRMED_MESSAGE in message: + # publish the successful message + self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url) + self.get_errors() + except Exception as error: + self.logger.error(error) def on_document_loaded(self): - print("on_document_loaded called") + self.logger.info("on_document_loaded called") def _setPhoneCountryAndStore(self): try: if self.store_type == 0: self.page.evaluate("""()=>{ - document.getElementById("phone_country").focus(); + //document.getElementById("phone_country").focus(); document.getElementById("phone_country").value = \"FR\" }""") else: store_to_choose = self.store_map[self.store_type] self.page.evaluate("""(store_to_choose)=>{ document.getElementById("prefer").value = store_to_choose; - document.getElementById("phone_country").focus(); + //document.getElementById("phone_country").focus(); document.getElementById("phone_country").value = \"FR\" }""", store_to_choose) except Exception as error: self.logger.error(error) @@ -204,7 +220,7 @@ class CommandorPage: self.page.evaluate("""(name)=> { let surname = document.getElementById("surname"); if(surname.value.length == 0){ - surname.focus(); + // surname.focus(); surname.value = name.lastName; document.getElementById("name").focus(); document.getElementById("name").value = name.firstName @@ -220,7 +236,7 @@ class CommandorPage: items = self.page.query_selector("div.alert") if items: erro_content = items.inner_html() - print("错误:" + erro_content) + self.logger.info("错误:" + erro_content) self._handle_errors(erro_content) except Exception as ext: self.logger.error(ext) @@ -228,16 +244,25 @@ class CommandorPage: def _handle_errors(self, erro_content: str): if DOUBLE_REQUEST_ERROR_MESSAGE in erro_content: # this email has been already used - params.oracle_log_sender.send_double_data_error(self.contact) - # close browser - time.sleep(2) - self.browser.close() + if not self.is_finished: + params.oracle_log_sender.send_double_data_error(self.contact) + self.is_finished = True + self.termine() elif TOO_MANY_REQUEST_ERROR_MESSAGE in erro_content: # this email has been already used - params.oracle_log_sender.send_too_many_error(self.contact) - # close browser - time.sleep(2) - self.browser.close() + if not self.is_finished: + params.oracle_log_sender.send_too_many_error(self.contact) + self.is_finished = True + self.termine() + elif CAPTCHA_ERROR_MESSAGE in erro_content: + # this email has been already used + self.is_captcha_in_error = True + if not self.is_finished: + params.oracle_log_sender.send_captcha_error(self.contact) + self.is_finished = True + # no need to retry captcha, if retry ,will generate DOUBLE_REQUEST_ERROR_MESSAGE + self.termine() + # self.resolve_captcha() def _set_email(self, email): time.sleep(get_random_wait_time()) @@ -260,7 +285,6 @@ class CommandorPage: self.logger.error(error) def _checkCgu(self): - # self.page.mouse.wheel(0, random.randint(200, 600)) try: self.page.evaluate(""" document.getElementById("cgu").focus(); @@ -290,6 +314,7 @@ class CommandorPage: def termine(self): self.logger.info("will close browser") + time.sleep(1) self.browser.close() def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str): @@ -305,6 +330,29 @@ class CommandorPage: time.sleep(2) self.browser.close() + def resolve_captcha(self): + self.captcha_solver = SolveCaptcha(self.page) + self.captcha_solver.start(self.fill_captcha_solution) + + def fill_captcha_solution(self, solution): + self.logger.info("will input solution: " + solution) + try: + self.page.evaluate("""(solution)=>{ + document.getElementById("g-recaptcha-response").innerHTML=solution;}""", solution) + self.logger.info("will click on valid btn") + self.clickOnValidBtn() + # wait for 20s + time.sleep(20) + if not self.is_finished: + if not self.is_captcha_in_error: + self.clickOnValidBtn() + else: + self.is_captcha_in_error = False + + except Exception as error: + self.logger.error(error) + self.page.reload(timeout=PAGE_TIMEOUT) + def get_random_id_number() -> str: S = 8 # number of characters in the string.