panleicim/appointment_tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimize local db

Browse Source
This commit is contained in:
Lei PAN
2022-06-08 21:39:45 +02:00
parent 80d362b174
commit 20c84b2a14
9 changed files with 95 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
check_results.py
+2 -5
View File
@@ -41,7 +41,7 @@ class ResultChecker:
device = random.choice(params.DEVICES)
self.logger.info("模拟设备: " + device)
pixel_2 = self.tls.playwright.devices[device]
context = self.browser.new_context(**pixel_2, locale='en-GB')
context = self.browser.new_context(**pixel_2, locale='fr-FR')
self.page = context.new_page()
# hide webdriver information
self.page.add_init_script("""() => {
@@ -66,9 +66,6 @@ class ResultChecker:
# url_to_check = url + "?lang=fr"
print("url is " + url)
content = None
random_id_number = str(phone_number)[1:len(str(phone_number))]
proxy_username = "panleicim-res-fr-" + random_id_number
print("proxy_username is " + proxy_username)
proxy = {
"server": params.BRIGHT_DATA_PROXY_SERVER,
"username": params.BRIGHT_DATA_PROXY_USERNAME,
@@ -123,7 +120,7 @@ if __name__ == '__main__':
reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict())
result_list.append(reserve_pojo)
with ThreadPoolExecutor(max_workers=10) as executor:
with ThreadPoolExecutor(max_workers=15) as executor:
for reserve in result_list:
count = count + 1
if reserve.accepted is None or ResultEnum.ACCEPTED.value == reserve.accepted:
db/local_db_manager.py
+20 -2
View File
@@ -1,9 +1,14 @@
from builtins import list
import sqlalchemy
from sqlalchemy import MetaData, Column, String, Integer, DateTime, Table
from sqlalchemy.orm import Session
from sqlalchemy_utils import database_exists, create_database
from pojo.captcha_error_contact_pojo import ContactInErrorPojo
import definitions
import params
from pojo.ReserveResultPojo import ReserveResultPojo
from pojo.captcha_error_contact_pojo import ContactInErrorPojo, ERROR_TYPE_CAPTCHA
class LocalDbManager:
@@ -12,7 +17,7 @@ class LocalDbManager:
self.session = Session(self.init_db(path))
def init_db(self, path: str):
uri_for_db = "sqlite:///{}/{}.db".format(path, "appointment")
uri_for_db = "sqlite:///{}/{}.db?check_same_thread=false".format(path, "appointment")
print(uri_for_db)
# 2.-Turn on database engine
db_engine = sqlalchemy.create_engine(uri_for_db) # ensure this is the correct path for the sqlite file.
@@ -43,3 +48,16 @@ class LocalDbManager:
def insert_or_update(self, instance: ContactInErrorPojo):
self.session.merge(instance)
self.session.commit()
def get_all_captcha_error_contacts(self) -> list:
return self.session.query(ContactInErrorPojo).filter(ContactInErrorPojo.error_type == ERROR_TYPE_CAPTCHA).all()
def handle_success(self, reservePojo: ReserveResultPojo):
# delete the contact from table
self.session.query(ContactInErrorPojo).filter(ContactInErrorPojo.mail == reservePojo.email).delete()
if __name__ == '__main__':
list = params.local_db_manager.get_all_captcha_error_contacts()
for item in list:
print(item.mail)
definitions.py
-4
View File
@@ -2,9 +2,6 @@ import configparser
import getpass
import os
from pathlib import Path
from db.local_db_manager import LocalDbManager
home = str(Path.home())
config = configparser.ConfigParser()
print("home path: " + home)
@@ -17,5 +14,4 @@ FIREBASE_CONFIG_FILE = config['DEFAULT']['firebase_config_file']
LOGS_DIR = config['DEFAULT']['LOGS_DIR']
username = getpass.getuser()
LOG_SOURCE = username
local_db_manager = LocalDbManager(home)
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
main.py
+19 -13
View File
@@ -5,6 +5,7 @@ from concurrent.futures import ThreadPoolExecutor
import params
from logs.AppLogging import init_logger
from pojo.ModeEnum import ModeEnum
from pojo.contact_pojo import ContactPojo
from utils.excel_reader import ExcelHelper
from workers.commandor_page import CommandorPage
@@ -25,26 +26,31 @@ def start_book(start_number, end_number, store_choose_state=0, max_workers=10, p
logger.info(contacts)
with ThreadPoolExecutor(max_workers=max_workers) as executor:
for contact in contacts:
proxy = get_proxy(contact.phone, proxy_type)
proxy = get_proxy(proxy_type)
# start the task in thread
executor.submit(
CommandorPage(contact, store_type=store_choose_state, proxy_type=proxy_type, mode=mode).start_page,
proxy)
def get_proxy(phone_number, proxy_type=0):
# random_id_number = str(phone_number)[1:len(str(phone_number))]
random_id_number = params.get_random_id_number_for_proxy()
proxy_username = params.get_proxy_name_prefix(proxy_type) + random_id_number
logger.info("proxy_username is " + proxy_username)
proxy = {
"server": params.BRIGHT_DATA_PROXY_SERVER,
"username": params.BRIGHT_DATA_PROXY_USERNAME,
"password": params.BRIGHT_DATA_PROXY_PASSWORD
}
return proxy
def recheck_the_captcha_error_contacts(store_type=0, mode: ModeEnum = ModeEnum.MANUAL):
# get all the contacts in captcha error
list = params.local_db_manager.get_all_captcha_error_contacts()
with ThreadPoolExecutor(max_workers=10) as executor:
for errorContact in list:
contact = ContactPojo.get_contact_from_error_contact(errorContact)
proxy = get_proxy()
# start the task in thread
executor.submit(
CommandorPage(contact, store_type=store_type, mode=mode).start_page,
proxy)
def get_proxy(proxy_type=0):
return params.get_proxy(proxy_type)
if __name__ == '__main__':
# 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机，传入1的时候是总店
start_book(16, 16, store_choose_state=0, proxy_type=0)
# start_book(16, 16, store_choose_state=0, proxy_type=0)
recheck_the_captcha_error_contacts(0, mode=ModeEnum.AUTOMATIC)
params.py
+22
View File
@@ -1,7 +1,9 @@
import random
import string
import definitions
from db.DbManager import DataManager
from db.local_db_manager import LocalDbManager
from logs.LogSender import LogSender
firebase_store_manager = DataManager()
@@ -25,6 +27,24 @@ def get_proxy_name_prefix(proxy_type=0) -> str:
return PROXY_NAME_PREFIX_CC
def get_proxy(proxy_type=0):
if proxy_type == 0:
random_id_number = get_random_id_number_for_proxy()
proxy_username = get_proxy_name_prefix(proxy_type) + random_id_number
proxy = {
"server": PROXY_SERVER,
"username": proxy_username,
"password": PROXY_PASSWORD
}
else:
proxy = {
"server": BRIGHT_DATA_PROXY_SERVER,
"username": BRIGHT_DATA_PROXY_USERNAME,
"password": BRIGHT_DATA_PROXY_PASSWORD
}
return proxy
def get_random_id_number_for_proxy() -> str:
S = 8 # number of characters in the string.
ran = ''.join(random.choices(string.digits, k=S))
@@ -33,6 +53,8 @@ def get_random_id_number_for_proxy() -> str:
return id_number
local_db_manager = LocalDbManager(definitions.home)
DEVICES = ['iPad (gen 6)', 'iPad (gen 6) landscape', 'iPad (gen 7)', 'iPad (gen 7) landscape', 'iPad Mini',
'iPad Mini landscape', 'iPad Pro 11', 'iPad Pro 11 landscape', 'iPhone 6', 'iPhone 6 landscape',
'iPhone 6 Plus', 'iPhone 6 Plus landscape', 'iPhone 7', 'iPhone 7 landscape', 'iPhone 7 Plus',
pojo/captcha_error_contact_pojo.py
+1 -22
View File
@@ -1,8 +1,6 @@
from sqlalchemy import Column, String, Integer, DateTime, func
from sqlalchemy.orm import declarative_base
import definitions
from pojo.contact_pojo import ContactPojo
Base = declarative_base()
@@ -22,23 +20,4 @@ class ContactInErrorPojo(Base):
position: int = Column(Integer)
error_type = Column(Integer)
update_at = Column(DateTime, onupdate=func.now())
create_at = Column(DateTime, default=func.now())
def get_captcha_error_contact_from_contact(contact: ContactPojo, error_type: int) -> ContactInErrorPojo:
captcha_error = ContactInErrorPojo()
captcha_error.mail = contact.mail
captcha_error.ccid = contact.ccid
captcha_error.phone = contact.phone
captcha_error.passport = contact.passport
captcha_error.first_name = contact.first_name
captcha_error.last_name = contact.last_name
captcha_error.position = contact.position
captcha_error.error_type = error_type
return captcha_error
if __name__ == '__main__':
contact = ContactPojo(mail="panleici3m@gmail.com", phone_number="649114592", ccid="", position=0,
passport_number="3322111", first_name="Lei", last_name="PAAaN")
definitions.local_db_manager.insert_or_update(get_captcha_error_contact_from_contact(contact, ERROR_TYPE_CAPTCHA))
create_at = Column(DateTime, default=func.now())
pojo/contact_pojo.py
+8
View File
@@ -1,5 +1,7 @@
from dataclasses import dataclass
from pojo.captcha_error_contact_pojo import ContactInErrorPojo
@dataclass
class ContactPojo:
@@ -35,6 +37,12 @@ class ContactPojo:
return dest
@staticmethod
def get_contact_from_error_contact(errorContact: ContactInErrorPojo):
return ContactPojo(phone_number=errorContact.phone, mail=errorContact.mail, ccid=errorContact.ccid,
last_name=errorContact.last_name, first_name=errorContact.first_name,
position=errorContact.position, passport_number=errorContact.passport)
@staticmethod
def from_firestore_dict(source):
ccid = source['ccid']
utils/generate_random_passport_id.py
+15
View File
@@ -1,6 +1,9 @@
import random
import string
from pojo.captcha_error_contact_pojo import ContactInErrorPojo
from pojo.contact_pojo import ContactPojo
letters = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'P', 'Q', 'R', 'S', 'T', '1', '2', '3', '4', '5', '6', '7',
'8', '9']
@@ -19,6 +22,18 @@ def get_random_passport_id_number() -> str:
return id_number
def get_captcha_error_contact_from_contact(contact: ContactPojo, error_type: int) -> ContactInErrorPojo:
captcha_error = ContactInErrorPojo()
captcha_error.mail = contact.mail
captcha_error.ccid = contact.ccid
captcha_error.phone = contact.phone
captcha_error.passport = contact.passport
captcha_error.first_name = contact.first_name
captcha_error.last_name = contact.last_name
captcha_error.position = contact.position
captcha_error.error_type = error_type
return captcha_error
if __name__ == '__main__':
# for i in range(1,200):
# print(get_random_id_number())
workers/commandor_page.py
+8 -10
View File
@@ -12,9 +12,10 @@ import definitions
import params
from pojo.ModeEnum import ModeEnum
from pojo.ReserveResultPojo import ReserveResultPojo, PublishType
from pojo.captcha_error_contact_pojo import get_captcha_error_contact_from_contact, ERROR_TYPE_CAPTCHA, \
from pojo.captcha_error_contact_pojo import ERROR_TYPE_CAPTCHA, \
TOO_MANY_REQUEST_ERROR
from pojo.contact_pojo import ContactPojo
from utils.generate_random_passport_id import get_captcha_error_contact_from_contact
from workers.SolveCaptch import SolveCaptcha
RDV_URL = "https://rendezvousparis.hermes.com/client/register"
@@ -32,7 +33,7 @@ CONFIRMED_MESSAGE_FR = "Votre demande de rendez-vous Maroquinerie a bien été e
DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today."
DOUBLE_REQUEST_ERROR_MESSAGE_FR = "Une demande avec les données saisies a déjà été validée aujourd’hui."
TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests"
TOO_MANY_REQUEST_ERROR_MESSAGE_FR = "Suite à un trop grand nombre de demandes aujourd’hui,"
TOO_MANY_REQUEST_ERROR_MESSAGE_FR = "Suite à un trop grand nombre de demandes"
CAPTCHA_ERROR_MESSAGE = "Error verifying captcha, please try again"
CAPTCHA_ERROR_MESSAGE_FR = "La vérification du captcha a échoué"
TIME_OUT = 400000
@@ -98,11 +99,7 @@ class CommandorPage:
first_page = None
while first_page is None:
first_page = self.start_browser(proxy, self.tls.playwright, devices)
proxy = {
"server": params.BRIGHT_DATA_PROXY_SERVER,
"username": params.BRIGHT_DATA_PROXY_USERNAME,
"password": params.BRIGHT_DATA_PROXY_PASSWORD
}
proxy = params.get_proxy(self.proxy_type)
# wait for sms_code field
# self.clickOnValidBtn()
self.thread_event = e
@@ -132,7 +129,7 @@ class CommandorPage:
self.logger.info("填充信息: " + str(self.contact.phone))
self._set_name(self.contact.last_name, self.contact.first_name)
self._setPhoneCountryAndStore()
self._setPhoneNumber(self.contact.phone)
self._setPhoneNumber("0" + str(self.contact.phone))
self._set_email(self.contact.mail)
self.setIdNumber(self.contact.passport)
self._checkCgu()
@@ -255,7 +252,7 @@ class CommandorPage:
elif TOO_MANY_REQUEST_ERROR_MESSAGE in erro_content or TOO_MANY_REQUEST_ERROR_MESSAGE_FR in erro_content:
# this email has been already used
if not self.is_finished:
definitions.local_db_manager.insert_or_update(
params.local_db_manager.insert_or_update(
get_captcha_error_contact_from_contact(self.contact, TOO_MANY_REQUEST_ERROR))
params.oracle_log_sender.send_too_many_error(self.contact)
self.is_finished = True
@@ -265,7 +262,7 @@ class CommandorPage:
self.is_captcha_in_error = True
if not self.is_finished:
# save the error to database with contact info
definitions.local_db_manager.insert_or_update(
params.local_db_manager.insert_or_update(
get_captcha_error_contact_from_contact(self.contact, ERROR_TYPE_CAPTCHA))
params.oracle_log_sender.send_captcha_error(self.contact)
self.is_finished = True
@@ -331,6 +328,7 @@ class CommandorPage:
passport=contact.passport, ccid=contact.ccid)
result.id = id
params.firebase_store_manager.save(result)
params.local_db_manager.handle_success(result)
if status is PublishType.SUCCESS:
self.on_success(result)
time.sleep(2)