optimize local db

check_results.py

@@ -41,7 +41,7 @@ class ResultChecker:
             device = random.choice(params.DEVICES)
             self.logger.info("模拟设备: " + device)
             pixel_2 = self.tls.playwright.devices[device]
-            context = self.browser.new_context(**pixel_2, locale='en-GB')
+            context = self.browser.new_context(**pixel_2, locale='fr-FR')
             self.page = context.new_page()
             # hide webdriver information
             self.page.add_init_script("""() => {
@@ -66,9 +66,6 @@ class ResultChecker:
         # url_to_check = url + "?lang=fr"
         print("url is " + url)
         content = None
-        random_id_number = str(phone_number)[1:len(str(phone_number))]
-        proxy_username = "panleicim-res-fr-" + random_id_number
-        print("proxy_username is " + proxy_username)
         proxy = {
             "server": params.BRIGHT_DATA_PROXY_SERVER,
             "username": params.BRIGHT_DATA_PROXY_USERNAME,
@@ -123,7 +120,7 @@ if __name__ == '__main__':
         reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict())
         result_list.append(reserve_pojo)
 
-    with ThreadPoolExecutor(max_workers=10) as executor:
+    with ThreadPoolExecutor(max_workers=15) as executor:
         for reserve in result_list:
             count = count + 1
             if reserve.accepted is None or ResultEnum.ACCEPTED.value == reserve.accepted:

db/local_db_manager.py

@@ -1,9 +1,14 @@
+from builtins import list
+
 import sqlalchemy
 from sqlalchemy import MetaData, Column, String, Integer, DateTime, Table
 from sqlalchemy.orm import Session
 from sqlalchemy_utils import database_exists, create_database
 
-from pojo.captcha_error_contact_pojo import ContactInErrorPojo
+import definitions
+import params
+from pojo.ReserveResultPojo import ReserveResultPojo
+from pojo.captcha_error_contact_pojo import ContactInErrorPojo, ERROR_TYPE_CAPTCHA
 
 
 class LocalDbManager:
@@ -12,7 +17,7 @@ class LocalDbManager:
         self.session = Session(self.init_db(path))
 
     def init_db(self, path: str):
-        uri_for_db = "sqlite:///{}/{}.db".format(path, "appointment")
+        uri_for_db = "sqlite:///{}/{}.db?check_same_thread=false".format(path, "appointment")
         print(uri_for_db)
         # 2.-Turn on database engine
         db_engine = sqlalchemy.create_engine(uri_for_db)  # ensure this is the correct path for the sqlite file.
@@ -43,3 +48,16 @@ class LocalDbManager:
     def insert_or_update(self, instance: ContactInErrorPojo):
         self.session.merge(instance)
         self.session.commit()
+
+    def get_all_captcha_error_contacts(self) -> list:
+        return self.session.query(ContactInErrorPojo).filter(ContactInErrorPojo.error_type == ERROR_TYPE_CAPTCHA).all()
+
+    def handle_success(self, reservePojo: ReserveResultPojo):
+        #         delete the contact from table
+        self.session.query(ContactInErrorPojo).filter(ContactInErrorPojo.mail == reservePojo.email).delete()
+
+
+if __name__ == '__main__':
+    list = params.local_db_manager.get_all_captcha_error_contacts()
+    for item in list:
+        print(item.mail)

definitions.py

@@ -2,9 +2,6 @@ import configparser
 import getpass
 import os
 from pathlib import Path
-
-from db.local_db_manager import LocalDbManager
-
 home = str(Path.home())
 config = configparser.ConfigParser()
 print("home path: " + home)
@@ -17,5 +14,4 @@ FIREBASE_CONFIG_FILE = config['DEFAULT']['firebase_config_file']
 LOGS_DIR = config['DEFAULT']['LOGS_DIR']
 username = getpass.getuser()
 LOG_SOURCE = username
-local_db_manager = LocalDbManager(home)
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))

main.py

@@ -5,6 +5,7 @@ from concurrent.futures import ThreadPoolExecutor
 import params
 from logs.AppLogging import init_logger
 from pojo.ModeEnum import ModeEnum
+from pojo.contact_pojo import ContactPojo
 from utils.excel_reader import ExcelHelper
 from workers.commandor_page import CommandorPage
 
@@ -25,26 +26,31 @@ def start_book(start_number, end_number, store_choose_state=0, max_workers=10, p
     logger.info(contacts)
     with ThreadPoolExecutor(max_workers=max_workers) as executor:
         for contact in contacts:
-            proxy = get_proxy(contact.phone, proxy_type)
+            proxy = get_proxy(proxy_type)
             # start the task in thread
             executor.submit(
                 CommandorPage(contact, store_type=store_choose_state, proxy_type=proxy_type, mode=mode).start_page,
                 proxy)
 
 
-def get_proxy(phone_number, proxy_type=0):
+def recheck_the_captcha_error_contacts(store_type=0, mode: ModeEnum = ModeEnum.MANUAL):
-    # random_id_number = str(phone_number)[1:len(str(phone_number))]
+    # get all the contacts in captcha error
-    random_id_number = params.get_random_id_number_for_proxy()
+    list = params.local_db_manager.get_all_captcha_error_contacts()
-    proxy_username = params.get_proxy_name_prefix(proxy_type) + random_id_number
+    with ThreadPoolExecutor(max_workers=10) as executor:
-    logger.info("proxy_username is " + proxy_username)
+        for errorContact in list:
-    proxy = {
+            contact = ContactPojo.get_contact_from_error_contact(errorContact)
-        "server": params.BRIGHT_DATA_PROXY_SERVER,
+            proxy = get_proxy()
-        "username": params.BRIGHT_DATA_PROXY_USERNAME,
+            # start the task in thread
-        "password": params.BRIGHT_DATA_PROXY_PASSWORD
+            executor.submit(
-    }
+                CommandorPage(contact, store_type=store_type, mode=mode).start_page,
-    return proxy
+                proxy)
+
+
+def get_proxy(proxy_type=0):
+    return params.get_proxy(proxy_type)
 
 
 if __name__ == '__main__':
     # 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机，传入1的时候是总店
-    start_book(16, 16, store_choose_state=0, proxy_type=0)
+    # start_book(16, 16, store_choose_state=0, proxy_type=0)
+    recheck_the_captcha_error_contacts(0, mode=ModeEnum.AUTOMATIC)

params.py

@@ -1,7 +1,9 @@
 import random
 import string
 
+import definitions
 from db.DbManager import DataManager
+from db.local_db_manager import LocalDbManager
 from logs.LogSender import LogSender
 
 firebase_store_manager = DataManager()
@@ -25,6 +27,24 @@ def get_proxy_name_prefix(proxy_type=0) -> str:
         return PROXY_NAME_PREFIX_CC
 
 
+def get_proxy(proxy_type=0):
+    if proxy_type == 0:
+        random_id_number = get_random_id_number_for_proxy()
+        proxy_username = get_proxy_name_prefix(proxy_type) + random_id_number
+        proxy = {
+            "server": PROXY_SERVER,
+            "username": proxy_username,
+            "password": PROXY_PASSWORD
+        }
+    else:
+        proxy = {
+            "server": BRIGHT_DATA_PROXY_SERVER,
+            "username": BRIGHT_DATA_PROXY_USERNAME,
+            "password": BRIGHT_DATA_PROXY_PASSWORD
+        }
+    return proxy
+
+
 def get_random_id_number_for_proxy() -> str:
     S = 8  # number of characters in the string.
     ran = ''.join(random.choices(string.digits, k=S))
@@ -33,6 +53,8 @@ def get_random_id_number_for_proxy() -> str:
     return id_number
 
 
+local_db_manager = LocalDbManager(definitions.home)
+
 DEVICES = ['iPad (gen 6)', 'iPad (gen 6) landscape', 'iPad (gen 7)', 'iPad (gen 7) landscape', 'iPad Mini',
            'iPad Mini landscape', 'iPad Pro 11', 'iPad Pro 11 landscape', 'iPhone 6', 'iPhone 6 landscape',
            'iPhone 6 Plus', 'iPhone 6 Plus landscape', 'iPhone 7', 'iPhone 7 landscape', 'iPhone 7 Plus',

pojo/captcha_error_contact_pojo.py

@@ -1,8 +1,6 @@
 from sqlalchemy import Column, String, Integer, DateTime, func
 from sqlalchemy.orm import declarative_base
 
-import definitions
-from pojo.contact_pojo import ContactPojo
 
 Base = declarative_base()
 
@@ -22,23 +20,4 @@ class ContactInErrorPojo(Base):
     position: int = Column(Integer)
     error_type = Column(Integer)
     update_at = Column(DateTime, onupdate=func.now())
-    create_at = Column(DateTime, default=func.now())
+    create_at = Column(DateTime, default=func.now())
-
-
-def get_captcha_error_contact_from_contact(contact: ContactPojo, error_type: int) -> ContactInErrorPojo:
-    captcha_error = ContactInErrorPojo()
-    captcha_error.mail = contact.mail
-    captcha_error.ccid = contact.ccid
-    captcha_error.phone = contact.phone
-    captcha_error.passport = contact.passport
-    captcha_error.first_name = contact.first_name
-    captcha_error.last_name = contact.last_name
-    captcha_error.position = contact.position
-    captcha_error.error_type = error_type
-    return captcha_error
-
-
-if __name__ == '__main__':
-    contact = ContactPojo(mail="panleici3m@gmail.com", phone_number="649114592", ccid="", position=0,
-                          passport_number="3322111", first_name="Lei", last_name="PAAaN")
-    definitions.local_db_manager.insert_or_update(get_captcha_error_contact_from_contact(contact, ERROR_TYPE_CAPTCHA))

pojo/contact_pojo.py

@@ -1,5 +1,7 @@
 from dataclasses import dataclass
 
+from pojo.captcha_error_contact_pojo import ContactInErrorPojo
+
 
 @dataclass
 class ContactPojo:
@@ -35,6 +37,12 @@ class ContactPojo:
 
         return dest
 
+    @staticmethod
+    def get_contact_from_error_contact(errorContact: ContactInErrorPojo):
+        return ContactPojo(phone_number=errorContact.phone, mail=errorContact.mail, ccid=errorContact.ccid,
+                           last_name=errorContact.last_name, first_name=errorContact.first_name,
+                           position=errorContact.position, passport_number=errorContact.passport)
+
     @staticmethod
     def from_firestore_dict(source):
         ccid = source['ccid']

utils/generate_random_passport_id.py

@@ -1,6 +1,9 @@
 import random
 import string
 
+from pojo.captcha_error_contact_pojo import ContactInErrorPojo
+from pojo.contact_pojo import ContactPojo
+
 letters = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'P', 'Q', 'R', 'S', 'T', '1', '2', '3', '4', '5', '6', '7',
            '8', '9']
 
@@ -19,6 +22,18 @@ def get_random_passport_id_number() -> str:
     return id_number
 
 
+def get_captcha_error_contact_from_contact(contact: ContactPojo, error_type: int) -> ContactInErrorPojo:
+    captcha_error = ContactInErrorPojo()
+    captcha_error.mail = contact.mail
+    captcha_error.ccid = contact.ccid
+    captcha_error.phone = contact.phone
+    captcha_error.passport = contact.passport
+    captcha_error.first_name = contact.first_name
+    captcha_error.last_name = contact.last_name
+    captcha_error.position = contact.position
+    captcha_error.error_type = error_type
+    return captcha_error
+
 if __name__ == '__main__':
     # for i in range(1,200):
     #     print(get_random_id_number())

workers/commandor_page.py

@@ -12,9 +12,10 @@ import definitions
 import params
 from pojo.ModeEnum import ModeEnum
 from pojo.ReserveResultPojo import ReserveResultPojo, PublishType
-from pojo.captcha_error_contact_pojo import get_captcha_error_contact_from_contact, ERROR_TYPE_CAPTCHA, \
+from pojo.captcha_error_contact_pojo import ERROR_TYPE_CAPTCHA, \
     TOO_MANY_REQUEST_ERROR
 from pojo.contact_pojo import ContactPojo
+from utils.generate_random_passport_id import get_captcha_error_contact_from_contact
 from workers.SolveCaptch import SolveCaptcha
 
 RDV_URL = "https://rendezvousparis.hermes.com/client/register"
@@ -32,7 +33,7 @@ CONFIRMED_MESSAGE_FR = "Votre demande de rendez-vous Maroquinerie a bien été e
 DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today."
 DOUBLE_REQUEST_ERROR_MESSAGE_FR = "Une demande avec les données saisies a déjà été validée aujourd’hui."
 TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests"
-TOO_MANY_REQUEST_ERROR_MESSAGE_FR = "Suite à un trop grand nombre de demandes aujourd’hui,"
+TOO_MANY_REQUEST_ERROR_MESSAGE_FR = "Suite à un trop grand nombre de demandes"
 CAPTCHA_ERROR_MESSAGE = "Error verifying captcha, please try again"
 CAPTCHA_ERROR_MESSAGE_FR = "La vérification du captcha a échoué"
 TIME_OUT = 400000
@@ -98,11 +99,7 @@ class CommandorPage:
         first_page = None
         while first_page is None:
             first_page = self.start_browser(proxy, self.tls.playwright, devices)
-            proxy = {
+            proxy = params.get_proxy(self.proxy_type)
-                "server": params.BRIGHT_DATA_PROXY_SERVER,
-                "username": params.BRIGHT_DATA_PROXY_USERNAME,
-                "password": params.BRIGHT_DATA_PROXY_PASSWORD
-            }
         # wait for sms_code field
         # self.clickOnValidBtn()
         self.thread_event = e
@@ -132,7 +129,7 @@ class CommandorPage:
             self.logger.info("填充信息: " + str(self.contact.phone))
             self._set_name(self.contact.last_name, self.contact.first_name)
             self._setPhoneCountryAndStore()
-            self._setPhoneNumber(self.contact.phone)
+            self._setPhoneNumber("0" + str(self.contact.phone))
             self._set_email(self.contact.mail)
             self.setIdNumber(self.contact.passport)
             self._checkCgu()
@@ -255,7 +252,7 @@ class CommandorPage:
         elif TOO_MANY_REQUEST_ERROR_MESSAGE in erro_content or TOO_MANY_REQUEST_ERROR_MESSAGE_FR in erro_content:
             # this email has been already used
             if not self.is_finished:
-                definitions.local_db_manager.insert_or_update(
+                params.local_db_manager.insert_or_update(
                     get_captcha_error_contact_from_contact(self.contact, TOO_MANY_REQUEST_ERROR))
                 params.oracle_log_sender.send_too_many_error(self.contact)
             self.is_finished = True
@@ -265,7 +262,7 @@ class CommandorPage:
             self.is_captcha_in_error = True
             if not self.is_finished:
                 # save the error to database with contact info
-                definitions.local_db_manager.insert_or_update(
+                params.local_db_manager.insert_or_update(
                     get_captcha_error_contact_from_contact(self.contact, ERROR_TYPE_CAPTCHA))
                 params.oracle_log_sender.send_captcha_error(self.contact)
             self.is_finished = True
@@ -331,6 +328,7 @@ class CommandorPage:
                                    passport=contact.passport, ccid=contact.ccid)
         result.id = id
         params.firebase_store_manager.save(result)
+        params.local_db_manager.handle_success(result)
         if status is PublishType.SUCCESS:
             self.on_success(result)
             time.sleep(2)