Merge branch 'feature/automatic'

appointment.py

@@ -2,6 +2,7 @@ import PySimpleGUI as sg
 
 # First the window layout in 2 columns
 from main import start_book
+from pojo.ModeEnum import ModeEnum
 
 KEY_CHOOSE_STORE = "CHOOSE_STORE"
 KEY_START_NUMBER = "KEY_START_NUMBER"
@@ -13,6 +14,9 @@ KEY_PROXY_CC = "KEY_PROXY_CC"
 KEY_FAUBOURG = "KEY_FAUBOURG"
 KEY_GEORGE = "KEY_GEORGE"
 KEY_SEVRES = "KEY_SEVRES"
+KEY_AUTOMATIC = "KEY_AUTOMATIC"
+KEY_MANUAL = "KEY_MANUAL"
+GROUP_MODE = "GROUP_MODE"
 GROUP_STORE = "STORE"
 GROUP_PROXY = "GROUP_PROXY"
 
@@ -36,12 +40,19 @@ proxy_settings_column = [
     [sg.Radio('res(速度)', group_id=GROUP_PROXY, key=KEY_PROXY_RES, default=True)],
     [sg.Radio('cc(稳定)', group_id=GROUP_PROXY, key=KEY_PROXY_CC, default=False)],
 ]
+
+mode_settings_column = [
+    [sg.Text("约会模式")],
+    [sg.Radio('手动', group_id=GROUP_MODE, key=KEY_MANUAL, default=True)],
+    [sg.Radio('自动', group_id=GROUP_MODE, key=KEY_AUTOMATIC, default=False)],
+]
 # ----- Full layout -----
 layout = [
     [
         sg.Column(file_list_column),
         sg.Column(store_settings_column),
-        sg.Column(proxy_settings_column)
+        sg.Column(proxy_settings_column),
+        sg.Column(mode_settings_column)
     ]
 ]
 
@@ -57,6 +68,7 @@ while True:
         end_line = int(values[KEY_END_NUMBER])
         max_workers = int(values[KEY_MAX_WORKERS])
         store_type = 0
+        mode = ModeEnum.MANUAL
         if values[KEY_FAUBOURG]:
             store_type = 1
         elif values[KEY_GEORGE]:
@@ -69,7 +81,11 @@ while True:
             proxy_type = 1
         elif values[KEY_PROXY_RES]:
             proxy_type = 0
-        start_book(start_line, end_line, store_choose_state=store_type, max_workers=max_workers, proxy_type=proxy_type)
+
+        if values[KEY_AUTOMATIC]:
+            mode = ModeEnum.AUTOMATIC
+        start_book(start_line, end_line, store_choose_state=store_type, max_workers=max_workers, proxy_type=proxy_type,
+                   mode=mode)
         # except Exception as error:
         #     print("Not Integer: ")
         #     print(error)

check_results.py

@@ -21,7 +21,7 @@ BLANK_URL = "about:blank"
 class TlsPlaywright(threading.local):
     def __init__(self) -> None:
         self.playwright = sync_playwright().start()
-        print("Create playwright instance in Thread", threading.current_thread().name)
+        print("创建浏览器实例，线程: ", threading.current_thread().name)
 
 
 class ResultChecker:
@@ -108,7 +108,7 @@ if __name__ == '__main__':
     # get the list
     params.oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS)
     db_manager = params.firebase_store_manager
-    # collection = db_manager.get_all_successful_items_for_day("2022-05-14", "landd")
+    # collection = db_manager.get_successful_item_for_day_by_status("2022-05-17", ResultEnum.ACCEPTED)
     collection = db_manager.get_all_successful_items()
     count = 0
     result_list = []
@@ -116,12 +116,12 @@ if __name__ == '__main__':
         reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict())
         result_list.append(reserve_pojo)
 
-    with ThreadPoolExecutor(max_workers=10) as executor:
+
+    with ThreadPoolExecutor(max_workers=25) as executor:
         for reserve in result_list:
             count = count + 1
-            if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted:
-                if reserve.url != BLANK_URL:
-                    executor.submit(ResultChecker().run, reserve, collection)
+            if reserve.accepted is None or ResultEnum.ACCEPTED.value == reserve.accepted:
+                executor.submit(ResultChecker().run, reserve, collection)
             else:
                 print("status is " + reserve.accepted)
 

db/DbManager.py

@@ -5,6 +5,7 @@ import firebase_admin
 from firebase_admin import credentials, firestore
 
 import definitions
+from pojo import ResultEnum
 from pojo.MailPojo import MailPojo
 from pojo.ReserveResultPojo import ReserveResultPojo
 from pojo.SimInfoPojo import SimInfoPojo
@@ -38,6 +39,12 @@ class DataManager:
             doc_ref.where(u'source_from', u'==', source_from)
         return doc_ref
 
+    def get_successful_item_for_day_by_status(self, day, status: ResultEnum):
+        doc_ref = self._db.collection(day)
+        if status is not None:
+            doc_ref.where(u'accepted', u'==', status.value)
+        return doc_ref
+
     def save_sim_info(self, sim_info: SimInfoPojo):
         doc_ref = self._db.collection(SIM_INFOS).document(sim_info.phone)
         doc_ref.set(sim_info.to_firestore_dict())

logs/LogSender.py

@@ -25,6 +25,7 @@ LOG_ERROR = "ERROR"
 LOG_TYPE_INFO = "INFO"
 LOG_ERROR_TYPE_DOUBLE_DATA = "DOUBLE_REQUEST_FOR_SAME_DATA"
 LOG_ERROR_TOO_MANY_REQUEST_TODAY = "TOO_MANY_REQUEST_TODAY"
+LOG_ERROR_CAPTCHA_ERROR_MESSAGE = "CAPTCHA_ERROR"
 LOG_SUBJECT_ERROR = "ERROR"
 LOG_APPOINTMENT_ERROR = "APPOINTMENT_ERROR"
 LOG_APPOINTMENT_TIMEOUT = "TIMEOUT"
@@ -78,6 +79,10 @@ class LogSender:
         error_msg = contact.mail
         self.send_log(msg=error_msg, type=LOG_ERROR_TOO_MANY_REQUEST_TODAY, subject=LOG_SUBJECT_ERROR)
 
+    def send_captcha_error(self, contact: ContactPojo):
+        error_msg = contact.mail
+        self.send_log(msg=error_msg, type=LOG_ERROR_CAPTCHA_ERROR_MESSAGE, subject=LOG_SUBJECT_ERROR)
+
     def send_appoint_result(self, result: ReserveResultPojo):
         if result.type == PublishType.SUCCESS:
             # get id

main.py

@@ -4,6 +4,7 @@ from concurrent.futures import ThreadPoolExecutor
 
 import params
 from logs.AppLogging import init_logger
+from pojo.ModeEnum import ModeEnum
 from utils.excel_reader import ExcelHelper
 from workers.commandor_page import CommandorPage
 
@@ -13,7 +14,8 @@ logger = logging.getLogger()
 logger.addHandler(logging.StreamHandler(stream=sys.stdout))
 
 
-def start_book(start_number, end_number, store_choose_state=0, max_workers=10, proxy_type=0):
+def start_book(start_number, end_number, store_choose_state=0, max_workers=10, proxy_type=0,
+               mode: ModeEnum = ModeEnum.MANUAL):
     # read the contact, and contact the 2 objects together
     excel_reader = ExcelHelper()
     all_contacts = excel_reader.read_contacts()
@@ -25,8 +27,9 @@ def start_book(start_number, end_number, store_choose_state=0, max_workers=10, p
         for contact in contacts:
             proxy = get_proxy(contact.phone, proxy_type)
             # start the task in thread
-            executor.submit(CommandorPage(contact, store_type=store_choose_state, proxy_type=proxy_type).start_page,
-                            proxy)
+            executor.submit(
+                CommandorPage(contact, store_type=store_choose_state, proxy_type=proxy_type, mode=mode).start_page,
+                proxy)
 
 
 def get_proxy(phone_number, proxy_type=0):

params.py

@@ -1,9 +1,6 @@
-import configparser
 import random
 import string
 
-from playwright.sync_api import sync_playwright
-
 from db.DbManager import DataManager
 from logs.LogSender import LogSender
 
@@ -22,6 +19,7 @@ def get_proxy_name_prefix(proxy_type = 0) -> str:
     else:
         return PROXY_NAME_PREFIX_CC
 
+
 def get_random_id_number_for_proxy() -> str:
     S = 8  # number of characters in the string.
     ran = ''.join(random.choices(string.digits, k=S))

pojo/ModeEnum.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class ModeEnum(Enum):
+    AUTOMATIC = "AUTOMATIC"
+    MANUAL = "MANUAL"

requirements.txt

@@ -9,3 +9,4 @@ boto3~=1.21.13
 openpyxl==3.0.9
 google-cloud-firestore==2.4.0
 PySimpleGUI==4.60.0
+SpeechRecognition==3.8.1

workers/SolveCaptch.py

@@ -0,0 +1,54 @@
+# for recaptcha
+import logging
+import random
+import re
+import time
+import requests
+
+CAPCHA_NOT_READY = "CAPCHA_NOT_READY"
+REGEX_DATA_SITE_KEY = "data-sitekey=[\"a-z0-9A-Z]+"
+API_KEY = "d66aaf490d8aa424a5175e1fbd1aadea"
+
+
+class SolveCaptcha:
+    def __init__(self, page):
+        self.page = page
+        self.logger = logging.getLogger("SolveCaptcha")
+        self.main_frame = None
+        self.recaptcha = None
+
+    def delay(self):
+        self.page.wait_for_timeout(random.randint(1, 3) * 1000)
+
+    def start(self, handle_solution_received):
+        self.logger.info("start to resolve captcha")
+        content = self.page.content()
+        data_sitekey = re.findall(REGEX_DATA_SITE_KEY, content)
+        self.logger.info(data_sitekey)
+        if len(data_sitekey) == 1:
+            key_with_comma = data_sitekey[0].split("=")[-1]
+            key = key_with_comma.replace("\"", '')
+            self.logger.info("key is : " + key)
+            self.solve_captcha(key, handle_solution_received)
+
+    def solve_captcha(self, google_key: str, handle_solution_received):
+        self.logger.info("solve_captcha()")
+        url_get = "http://2captcha.com/in.php?key={}&method=userrecaptcha&googlekey={}&pageurl={}".format(API_KEY,
+                                                                                                          google_key,
+                                                                                                          self.page.url)
+        res = requests.get(url_get)
+        self.logger.info(res.text)
+        results = res.text.split("|")
+        self.captcha_id = results[-1]
+        #       wait for 15 seconds
+        time.sleep(15)
+        #         get result of the captcha
+        url_response = "http://2captcha.com/res.php?key={}&action=get&id={}".format(API_KEY,
+                                                                                    self.captcha_id)
+        solution = CAPCHA_NOT_READY
+        while solution == CAPCHA_NOT_READY:
+            solution_res = requests.get(url_response)
+            time.sleep(5)
+            solution = solution_res.text
+        print(solution)
+        handle_solution_received(solution.split("|")[-1])

workers/commandor_page.py

@@ -10,8 +10,10 @@ from playwright.sync_api import sync_playwright
 
 import params
 from params import PROXY_SERVER, PROXY_PASSWORD
+from pojo.ModeEnum import ModeEnum
 from pojo.ReserveResultPojo import ReserveResultPojo, PublishType
 from pojo.contact_pojo import ContactPojo
+from workers.SolveCaptch import SolveCaptcha
 
 RDV_URL = "https://rendezvousparis.hermes.com/client/register"
 
@@ -26,9 +28,10 @@ MESSAGE_FIELD_CLASS = ".message"
 CONFIRMED_MESSAGE = "Your request for a Leather Goods appointment has been registered"
 DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today."
 TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests"
+CAPTCHA_ERROR_MESSAGE = "Error verifying captcha, please try again"
 TIME_OUT = 400000
 OTP_TIMEOUT = 240
-PAGE_TIMEOUT = 30000
+PAGE_TIMEOUT = 40000
 
 
 def get_random_wait_time() -> float:
@@ -44,12 +47,16 @@ class Tls(threading.local):
 class CommandorPage:
     tls = Tls()
 
-    def __init__(self, contact: ContactPojo, store_type=0, proxy_type=0):
+    def __init__(self, contact: ContactPojo, store_type=0, proxy_type=0, mode: ModeEnum = ModeEnum.MANUAL):
         self.otp_value = None
-        self.logger = logging.getLogger("CommandorPage")
+        self.logger = logging.getLogger("CommandorPage:" + str(contact.phone))
         self.is_finished = False
         self.contact = contact
         self.proxy_type = proxy_type
+        self.is_event_sent = False
+        self.is_captcha_in_error = False
+        self.is_filling_fields = False
+        self.appointment_mode = mode
         # 0: random
         # 1: faubourg
         # 2: George
@@ -63,9 +70,12 @@ class CommandorPage:
 
     def on_success(self, result: ReserveResultPojo):
         self.logger.info("on_success called.")
-        self.logger.info(result)
-        params.oracle_log_sender.send_appoint_result(result)
         self.is_finished = True
+        if not self.is_event_sent:
+            self.logger.info("will send successful event")
+            self.logger.info(result)
+            params.oracle_log_sender.send_appoint_result(result)
+            self.is_event_sent = True
 
     def timeout_occurred(self):
         params.oracle_log_sender.send_timeout_log(self.contact)
@@ -89,7 +99,6 @@ class CommandorPage:
                 "username": proxy_username,
                 "password": params.PROXY_PASSWORD
             }
-        self.fill_fields()
         # wait for sms_code field
         # self.clickOnValidBtn()
         self.thread_event = e
@@ -103,7 +112,6 @@ class CommandorPage:
             self.clickOnValidBtn()
             otp_sent = self.page.locator(MESSAGE_FIELD_CLASS)
             otp_sent.wait_for(state='visible', timeout=TIME_OUT)
-        # print("message is:" + message)
         time.sleep(get_random_wait_time())
         message = self.page.content()
         if CONFIRMED_MESSAGE in message:
@@ -115,20 +123,25 @@ class CommandorPage:
             self.termine()
 
     def fill_fields(self):
-        self._set_name(self.contact.last_name, self.contact.first_name)
-        self._setPhoneCountryAndStore()
-        self._setPhoneNumber(self.contact.phone)
-        self._set_email(self.contact.mail)
-        self.setIdNumber(self.contact.passport)
-        #
-        self._checkCgu()
+        if not self.is_filling_fields:
+            self.is_filling_fields = True
+            self.logger.info("填充信息: " + str(self.contact.phone))
+            self._set_name(self.contact.last_name, self.contact.first_name)
+            self._setPhoneCountryAndStore()
+            self._setPhoneNumber(self.contact.phone)
+            self._set_email(self.contact.mail)
+            self.setIdNumber(self.contact.passport)
+            self._checkCgu()
+            if self.appointment_mode == ModeEnum.AUTOMATIC:
+                self.resolve_captcha()
+            self.is_filling_fields = False
 
     def start_browser(self, proxy, pwright, device) -> Union[str, None]:
         try:
             self.browser = pwright.webkit.launch(headless=False, timeout=PAGE_TIMEOUT, proxy=proxy)
             self.logger.info("模拟设备: " + device)
             pixel_2 = pwright.devices[device]
-            context = self.browser.new_context(**pixel_2, locale='en-GB')
+            context = self.browser.new_context(**pixel_2, locale='fr-FR')
             self.page = context.new_page()
             # hide webdriver information
             self.page.add_init_script("""() => {
@@ -154,7 +167,7 @@ class CommandorPage:
         pattern = re.compile(REGEX_RDV_URL)
         if pattern.match(response.url):
             self.logger.info("result url found: " + response.url)
-            self.publish_message_to_queue(self.contact, PublishType.PENDING, response.url)
+            # self.publish_message_to_queue(self.contact, PublishType.PENDING, response.url)
 
     def start_page(self, proxy):
         e = threading.Event()
@@ -166,26 +179,29 @@ class CommandorPage:
         self.logger.info("url is " + self.page.url)
         if self.page.url == RDV_URL:
             self.fill_fields()
-        message = self.page.content()
-        if CONFIRMED_MESSAGE in message:
-            # publish the successful message
-            self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url)
-        self.get_errors()
+        try:
+            message = self.page.content()
+            if CONFIRMED_MESSAGE in message:
+                # publish the successful message
+                self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url)
+            self.get_errors()
+        except Exception as error:
+            self.logger.error(error)
 
     def on_document_loaded(self):
-        print("on_document_loaded called")
+        self.logger.info("on_document_loaded called")
 
     def _setPhoneCountryAndStore(self):
         try:
             if self.store_type == 0:
                 self.page.evaluate("""()=>{
-                document.getElementById("phone_country").focus();
+                //document.getElementById("phone_country").focus();
                 document.getElementById("phone_country").value = \"FR\" }""")
             else:
                 store_to_choose = self.store_map[self.store_type]
                 self.page.evaluate("""(store_to_choose)=>{
                 document.getElementById("prefer").value = store_to_choose;
-                document.getElementById("phone_country").focus();
+                //document.getElementById("phone_country").focus();
                 document.getElementById("phone_country").value = \"FR\" }""", store_to_choose)
         except Exception as error:
             self.logger.error(error)
@@ -204,7 +220,7 @@ class CommandorPage:
             self.page.evaluate("""(name)=>   {
             let surname = document.getElementById("surname");
             if(surname.value.length == 0){
-                        surname.focus();
+                        // surname.focus();
             surname.value = name.lastName;
                             document.getElementById("name").focus();
             document.getElementById("name").value = name.firstName
@@ -220,7 +236,7 @@ class CommandorPage:
             items = self.page.query_selector("div.alert")
             if items:
                 erro_content = items.inner_html()
-                print("错误:" + erro_content)
+                self.logger.info("错误:" + erro_content)
                 self._handle_errors(erro_content)
         except Exception as ext:
             self.logger.error(ext)
@@ -228,16 +244,25 @@ class CommandorPage:
     def _handle_errors(self, erro_content: str):
         if DOUBLE_REQUEST_ERROR_MESSAGE in erro_content:
             # this email has been already used
-            params.oracle_log_sender.send_double_data_error(self.contact)
-            #         close browser
-            time.sleep(2)
-            self.browser.close()
+            if not self.is_finished:
+                params.oracle_log_sender.send_double_data_error(self.contact)
+            self.is_finished = True
+            self.termine()
         elif TOO_MANY_REQUEST_ERROR_MESSAGE in erro_content:
             # this email has been already used
-            params.oracle_log_sender.send_too_many_error(self.contact)
-            #         close browser
-            time.sleep(2)
-            self.browser.close()
+            if not self.is_finished:
+                params.oracle_log_sender.send_too_many_error(self.contact)
+            self.is_finished = True
+            self.termine()
+        elif CAPTCHA_ERROR_MESSAGE in erro_content:
+            # this email has been already used
+            self.is_captcha_in_error = True
+            if not self.is_finished:
+                params.oracle_log_sender.send_captcha_error(self.contact)
+            self.is_finished = True
+            # no need to retry captcha, if retry ,will generate DOUBLE_REQUEST_ERROR_MESSAGE
+            self.termine()
+            # self.resolve_captcha()
 
     def _set_email(self, email):
         time.sleep(get_random_wait_time())
@@ -260,7 +285,6 @@ class CommandorPage:
             self.logger.error(error)
 
     def _checkCgu(self):
-        # self.page.mouse.wheel(0, random.randint(200, 600))
         try:
             self.page.evaluate("""
                     document.getElementById("cgu").focus();
@@ -290,6 +314,7 @@ class CommandorPage:
 
     def termine(self):
         self.logger.info("will close browser")
+        time.sleep(1)
         self.browser.close()
 
     def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str):
@@ -305,6 +330,29 @@ class CommandorPage:
             time.sleep(2)
             self.browser.close()
 
+    def resolve_captcha(self):
+        self.captcha_solver = SolveCaptcha(self.page)
+        self.captcha_solver.start(self.fill_captcha_solution)
+
+    def fill_captcha_solution(self, solution):
+        self.logger.info("will input solution: " + solution)
+        try:
+            self.page.evaluate("""(solution)=>{
+               document.getElementById("g-recaptcha-response").innerHTML=solution;}""", solution)
+            self.logger.info("will click on valid btn")
+            self.clickOnValidBtn()
+            #     wait for 20s
+            time.sleep(20)
+            if not self.is_finished:
+                if not self.is_captcha_in_error:
+                    self.clickOnValidBtn()
+                else:
+                    self.is_captcha_in_error = False
+
+        except Exception as error:
+            self.logger.error(error)
+            self.page.reload(timeout=PAGE_TIMEOUT)
+
 
 def get_random_id_number() -> str:
     S = 8  # number of characters in the string.