panleicim/appointment_tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/automatic'

Browse Source
This commit is contained in:
Lei PAN
2022-06-04 09:59:51 +02:00
parent 08318eef74 f8167dd2d0
commit 0a8c8e6330
11 changed files with 442 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
appointment.py
+18 -2
View File
@@ -2,6 +2,7 @@ import PySimpleGUI as sg
# First the window layout in 2 columns
from main import start_book
from pojo.ModeEnum import ModeEnum
KEY_CHOOSE_STORE = "CHOOSE_STORE"
KEY_START_NUMBER = "KEY_START_NUMBER"
@@ -13,6 +14,9 @@ KEY_PROXY_CC = "KEY_PROXY_CC"
KEY_FAUBOURG = "KEY_FAUBOURG"
KEY_GEORGE = "KEY_GEORGE"
KEY_SEVRES = "KEY_SEVRES"
KEY_AUTOMATIC = "KEY_AUTOMATIC"
KEY_MANUAL = "KEY_MANUAL"
GROUP_MODE = "GROUP_MODE"
GROUP_STORE = "STORE"
GROUP_PROXY = "GROUP_PROXY"
@@ -36,12 +40,19 @@ proxy_settings_column = [
[sg.Radio('res(速度)', group_id=GROUP_PROXY, key=KEY_PROXY_RES, default=True)],
[sg.Radio('cc(稳定)', group_id=GROUP_PROXY, key=KEY_PROXY_CC, default=False)],
]
mode_settings_column = [
[sg.Text("约会模式")],
[sg.Radio('手动', group_id=GROUP_MODE, key=KEY_MANUAL, default=True)],
[sg.Radio('自动', group_id=GROUP_MODE, key=KEY_AUTOMATIC, default=False)],
]
# ----- Full layout -----
layout = [
[
sg.Column(file_list_column),
sg.Column(store_settings_column),
sg.Column(proxy_settings_column)
sg.Column(proxy_settings_column),
sg.Column(mode_settings_column)
]
]
@@ -57,6 +68,7 @@ while True:
end_line = int(values[KEY_END_NUMBER])
max_workers = int(values[KEY_MAX_WORKERS])
store_type = 0
mode = ModeEnum.MANUAL
if values[KEY_FAUBOURG]:
store_type = 1
elif values[KEY_GEORGE]:
@@ -69,7 +81,11 @@ while True:
proxy_type = 1
elif values[KEY_PROXY_RES]:
proxy_type = 0
start_book(start_line, end_line, store_choose_state=store_type, max_workers=max_workers, proxy_type=proxy_type)
if values[KEY_AUTOMATIC]:
mode = ModeEnum.AUTOMATIC
start_book(start_line, end_line, store_choose_state=store_type, max_workers=max_workers, proxy_type=proxy_type,
mode=mode)
# except Exception as error:
# print("Not Integer: ")
# print(error)
check_results.py
+6 -6
View File
@@ -21,7 +21,7 @@ BLANK_URL = "about:blank"
class TlsPlaywright(threading.local):
def __init__(self) -> None:
self.playwright = sync_playwright().start()
print("Create playwright instance in Thread", threading.current_thread().name)
print("创建浏览器实例，线程: ", threading.current_thread().name)
class ResultChecker:
@@ -108,7 +108,7 @@ if __name__ == '__main__':
# get the list
params.oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS)
db_manager = params.firebase_store_manager
# collection = db_manager.get_all_successful_items_for_day("2022-05-14", "landd")
# collection = db_manager.get_successful_item_for_day_by_status("2022-05-17", ResultEnum.ACCEPTED)
collection = db_manager.get_all_successful_items()
count = 0
result_list = []
@@ -116,12 +116,12 @@ if __name__ == '__main__':
reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict())
result_list.append(reserve_pojo)
with ThreadPoolExecutor(max_workers=10) as executor:
with ThreadPoolExecutor(max_workers=25) as executor:
for reserve in result_list:
count = count + 1
if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted:
if reserve.url != BLANK_URL:
executor.submit(ResultChecker().run, reserve, collection)
if reserve.accepted is None or ResultEnum.ACCEPTED.value == reserve.accepted:
executor.submit(ResultChecker().run, reserve, collection)
else:
print("status is " + reserve.accepted)
db/DbManager.py
+7
View File
@@ -5,6 +5,7 @@ import firebase_admin
from firebase_admin import credentials, firestore
import definitions
from pojo import ResultEnum
from pojo.MailPojo import MailPojo
from pojo.ReserveResultPojo import ReserveResultPojo
from pojo.SimInfoPojo import SimInfoPojo
@@ -38,6 +39,12 @@ class DataManager:
doc_ref.where(u'source_from', u'==', source_from)
return doc_ref
def get_successful_item_for_day_by_status(self, day, status: ResultEnum):
doc_ref = self._db.collection(day)
if status is not None:
doc_ref.where(u'accepted', u'==', status.value)
return doc_ref
def save_sim_info(self, sim_info: SimInfoPojo):
doc_ref = self._db.collection(SIM_INFOS).document(sim_info.phone)
doc_ref.set(sim_info.to_firestore_dict())
docs/rdv.html
+254
View File
File diff suppressed because one or more lines are too long
logs/LogSender.py
+5
View File
@@ -25,6 +25,7 @@ LOG_ERROR = "ERROR"
LOG_TYPE_INFO = "INFO"
LOG_ERROR_TYPE_DOUBLE_DATA = "DOUBLE_REQUEST_FOR_SAME_DATA"
LOG_ERROR_TOO_MANY_REQUEST_TODAY = "TOO_MANY_REQUEST_TODAY"
LOG_ERROR_CAPTCHA_ERROR_MESSAGE = "CAPTCHA_ERROR"
LOG_SUBJECT_ERROR = "ERROR"
LOG_APPOINTMENT_ERROR = "APPOINTMENT_ERROR"
LOG_APPOINTMENT_TIMEOUT = "TIMEOUT"
@@ -78,6 +79,10 @@ class LogSender:
error_msg = contact.mail
self.send_log(msg=error_msg, type=LOG_ERROR_TOO_MANY_REQUEST_TODAY, subject=LOG_SUBJECT_ERROR)
def send_captcha_error(self, contact: ContactPojo):
error_msg = contact.mail
self.send_log(msg=error_msg, type=LOG_ERROR_CAPTCHA_ERROR_MESSAGE, subject=LOG_SUBJECT_ERROR)
def send_appoint_result(self, result: ReserveResultPojo):
if result.type == PublishType.SUCCESS:
# get id
main.py
+6 -3
View File
@@ -4,6 +4,7 @@ from concurrent.futures import ThreadPoolExecutor
import params
from logs.AppLogging import init_logger
from pojo.ModeEnum import ModeEnum
from utils.excel_reader import ExcelHelper
from workers.commandor_page import CommandorPage
@@ -13,7 +14,8 @@ logger = logging.getLogger()
logger.addHandler(logging.StreamHandler(stream=sys.stdout))
def start_book(start_number, end_number, store_choose_state=0, max_workers=10, proxy_type=0):
def start_book(start_number, end_number, store_choose_state=0, max_workers=10, proxy_type=0,
mode: ModeEnum = ModeEnum.MANUAL):
# read the contact, and contact the 2 objects together
excel_reader = ExcelHelper()
all_contacts = excel_reader.read_contacts()
@@ -25,8 +27,9 @@ def start_book(start_number, end_number, store_choose_state=0, max_workers=10, p
for contact in contacts:
proxy = get_proxy(contact.phone, proxy_type)
# start the task in thread
executor.submit(CommandorPage(contact, store_type=store_choose_state, proxy_type=proxy_type).start_page,
proxy)
executor.submit(
CommandorPage(contact, store_type=store_choose_state, proxy_type=proxy_type, mode=mode).start_page,
proxy)
def get_proxy(phone_number, proxy_type=0):
params.py
+1 -3
View File
@@ -1,9 +1,6 @@
import configparser
import random
import string
from playwright.sync_api import sync_playwright
from db.DbManager import DataManager
from logs.LogSender import LogSender
@@ -22,6 +19,7 @@ def get_proxy_name_prefix(proxy_type = 0) -> str:
else:
return PROXY_NAME_PREFIX_CC
def get_random_id_number_for_proxy() -> str:
S = 8 # number of characters in the string.
ran = ''.join(random.choices(string.digits, k=S))
pojo/ModeEnum.py
+6
View File
@@ -0,0 +1,6 @@
from enum import Enum
class ModeEnum(Enum):
AUTOMATIC = "AUTOMATIC"
MANUAL = "MANUAL"
requirements.txt
+2 -1
View File
@@ -8,4 +8,5 @@ XlsxWriter~=3.0.3
boto3~=1.21.13
openpyxl==3.0.9
google-cloud-firestore==2.4.0
PySimpleGUI==4.60.0
PySimpleGUI==4.60.0
SpeechRecognition==3.8.1
workers/SolveCaptch.py
+54
View File
@@ -0,0 +1,54 @@
# for recaptcha
import logging
import random
import re
import time
import requests
CAPCHA_NOT_READY = "CAPCHA_NOT_READY"
REGEX_DATA_SITE_KEY = "data-sitekey=[\"a-z0-9A-Z]+"
API_KEY = "d66aaf490d8aa424a5175e1fbd1aadea"
class SolveCaptcha:
def __init__(self, page):
self.page = page
self.logger = logging.getLogger("SolveCaptcha")
self.main_frame = None
self.recaptcha = None
def delay(self):
self.page.wait_for_timeout(random.randint(1, 3) * 1000)
def start(self, handle_solution_received):
self.logger.info("start to resolve captcha")
content = self.page.content()
data_sitekey = re.findall(REGEX_DATA_SITE_KEY, content)
self.logger.info(data_sitekey)
if len(data_sitekey) == 1:
key_with_comma = data_sitekey[0].split("=")[-1]
key = key_with_comma.replace("\"", '')
self.logger.info("key is : " + key)
self.solve_captcha(key, handle_solution_received)
def solve_captcha(self, google_key: str, handle_solution_received):
self.logger.info("solve_captcha()")
url_get = "http://2captcha.com/in.php?key={}&method=userrecaptcha&googlekey={}&pageurl={}".format(API_KEY,
google_key,
self.page.url)
res = requests.get(url_get)
self.logger.info(res.text)
results = res.text.split("|")
self.captcha_id = results[-1]
# wait for 15 seconds
time.sleep(15)
# get result of the captcha
url_response = "http://2captcha.com/res.php?key={}&action=get&id={}".format(API_KEY,
self.captcha_id)
solution = CAPCHA_NOT_READY
while solution == CAPCHA_NOT_READY:
solution_res = requests.get(url_response)
time.sleep(5)
solution = solution_res.text
print(solution)
handle_solution_received(solution.split("|")[-1])
workers/commandor_page.py
+83 -35
View File
@@ -10,8 +10,10 @@ from playwright.sync_api import sync_playwright
import params
from params import PROXY_SERVER, PROXY_PASSWORD
from pojo.ModeEnum import ModeEnum
from pojo.ReserveResultPojo import ReserveResultPojo, PublishType
from pojo.contact_pojo import ContactPojo
from workers.SolveCaptch import SolveCaptcha
RDV_URL = "https://rendezvousparis.hermes.com/client/register"
@@ -26,9 +28,10 @@ MESSAGE_FIELD_CLASS = ".message"
CONFIRMED_MESSAGE = "Your request for a Leather Goods appointment has been registered"
DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today."
TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests"
CAPTCHA_ERROR_MESSAGE = "Error verifying captcha, please try again"
TIME_OUT = 400000
OTP_TIMEOUT = 240
PAGE_TIMEOUT = 30000
PAGE_TIMEOUT = 40000
def get_random_wait_time() -> float:
@@ -44,12 +47,16 @@ class Tls(threading.local):
class CommandorPage:
tls = Tls()
def __init__(self, contact: ContactPojo, store_type=0, proxy_type=0):
def __init__(self, contact: ContactPojo, store_type=0, proxy_type=0, mode: ModeEnum = ModeEnum.MANUAL):
self.otp_value = None
self.logger = logging.getLogger("CommandorPage")
self.logger = logging.getLogger("CommandorPage:" + str(contact.phone))
self.is_finished = False
self.contact = contact
self.proxy_type = proxy_type
self.is_event_sent = False
self.is_captcha_in_error = False
self.is_filling_fields = False
self.appointment_mode = mode
# 0: random
# 1: faubourg
# 2: George
@@ -63,9 +70,12 @@ class CommandorPage:
def on_success(self, result: ReserveResultPojo):
self.logger.info("on_success called.")
self.logger.info(result)
params.oracle_log_sender.send_appoint_result(result)
self.is_finished = True
if not self.is_event_sent:
self.logger.info("will send successful event")
self.logger.info(result)
params.oracle_log_sender.send_appoint_result(result)
self.is_event_sent = True
def timeout_occurred(self):
params.oracle_log_sender.send_timeout_log(self.contact)
@@ -89,7 +99,6 @@ class CommandorPage:
"username": proxy_username,
"password": params.PROXY_PASSWORD
}
self.fill_fields()
# wait for sms_code field
# self.clickOnValidBtn()
self.thread_event = e
@@ -103,7 +112,6 @@ class CommandorPage:
self.clickOnValidBtn()
otp_sent = self.page.locator(MESSAGE_FIELD_CLASS)
otp_sent.wait_for(state='visible', timeout=TIME_OUT)
# print("message is:" + message)
time.sleep(get_random_wait_time())
message = self.page.content()
if CONFIRMED_MESSAGE in message:
@@ -115,20 +123,25 @@ class CommandorPage:
self.termine()
def fill_fields(self):
self._set_name(self.contact.last_name, self.contact.first_name)
self._setPhoneCountryAndStore()
self._setPhoneNumber(self.contact.phone)
self._set_email(self.contact.mail)
self.setIdNumber(self.contact.passport)
#
self._checkCgu()
if not self.is_filling_fields:
self.is_filling_fields = True
self.logger.info("填充信息: " + str(self.contact.phone))
self._set_name(self.contact.last_name, self.contact.first_name)
self._setPhoneCountryAndStore()
self._setPhoneNumber(self.contact.phone)
self._set_email(self.contact.mail)
self.setIdNumber(self.contact.passport)
self._checkCgu()
if self.appointment_mode == ModeEnum.AUTOMATIC:
self.resolve_captcha()
self.is_filling_fields = False
def start_browser(self, proxy, pwright, device) -> Union[str, None]:
try:
self.browser = pwright.webkit.launch(headless=False, timeout=PAGE_TIMEOUT, proxy=proxy)
self.logger.info("模拟设备: " + device)
pixel_2 = pwright.devices[device]
context = self.browser.new_context(**pixel_2, locale='en-GB')
context = self.browser.new_context(**pixel_2, locale='fr-FR')
self.page = context.new_page()
# hide webdriver information
self.page.add_init_script("""() => {
@@ -154,7 +167,7 @@ class CommandorPage:
pattern = re.compile(REGEX_RDV_URL)
if pattern.match(response.url):
self.logger.info("result url found: " + response.url)
self.publish_message_to_queue(self.contact, PublishType.PENDING, response.url)
# self.publish_message_to_queue(self.contact, PublishType.PENDING, response.url)
def start_page(self, proxy):
e = threading.Event()
@@ -166,26 +179,29 @@ class CommandorPage:
self.logger.info("url is " + self.page.url)
if self.page.url == RDV_URL:
self.fill_fields()
message = self.page.content()
if CONFIRMED_MESSAGE in message:
# publish the successful message
self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url)
self.get_errors()
try:
message = self.page.content()
if CONFIRMED_MESSAGE in message:
# publish the successful message
self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url)
self.get_errors()
except Exception as error:
self.logger.error(error)
def on_document_loaded(self):
print("on_document_loaded called")
self.logger.info("on_document_loaded called")
def _setPhoneCountryAndStore(self):
try:
if self.store_type == 0:
self.page.evaluate("""()=>{
document.getElementById("phone_country").focus();
//document.getElementById("phone_country").focus();
document.getElementById("phone_country").value = \"FR\" }""")
else:
store_to_choose = self.store_map[self.store_type]
self.page.evaluate("""(store_to_choose)=>{
document.getElementById("prefer").value = store_to_choose;
document.getElementById("phone_country").focus();
//document.getElementById("phone_country").focus();
document.getElementById("phone_country").value = \"FR\" }""", store_to_choose)
except Exception as error:
self.logger.error(error)
@@ -204,7 +220,7 @@ class CommandorPage:
self.page.evaluate("""(name)=> {
let surname = document.getElementById("surname");
if(surname.value.length == 0){
surname.focus();
// surname.focus();
surname.value = name.lastName;
document.getElementById("name").focus();
document.getElementById("name").value = name.firstName
@@ -220,7 +236,7 @@ class CommandorPage:
items = self.page.query_selector("div.alert")
if items:
erro_content = items.inner_html()
print("错误:" + erro_content)
self.logger.info("错误:" + erro_content)
self._handle_errors(erro_content)
except Exception as ext:
self.logger.error(ext)
@@ -228,16 +244,25 @@ class CommandorPage:
def _handle_errors(self, erro_content: str):
if DOUBLE_REQUEST_ERROR_MESSAGE in erro_content:
# this email has been already used
params.oracle_log_sender.send_double_data_error(self.contact)
# close browser
time.sleep(2)
self.browser.close()
if not self.is_finished:
params.oracle_log_sender.send_double_data_error(self.contact)
self.is_finished = True
self.termine()
elif TOO_MANY_REQUEST_ERROR_MESSAGE in erro_content:
# this email has been already used
params.oracle_log_sender.send_too_many_error(self.contact)
# close browser
time.sleep(2)
self.browser.close()
if not self.is_finished:
params.oracle_log_sender.send_too_many_error(self.contact)
self.is_finished = True
self.termine()
elif CAPTCHA_ERROR_MESSAGE in erro_content:
# this email has been already used
self.is_captcha_in_error = True
if not self.is_finished:
params.oracle_log_sender.send_captcha_error(self.contact)
self.is_finished = True
# no need to retry captcha, if retry ,will generate DOUBLE_REQUEST_ERROR_MESSAGE
self.termine()
# self.resolve_captcha()
def _set_email(self, email):
time.sleep(get_random_wait_time())
@@ -260,7 +285,6 @@ class CommandorPage:
self.logger.error(error)
def _checkCgu(self):
# self.page.mouse.wheel(0, random.randint(200, 600))
try:
self.page.evaluate("""
document.getElementById("cgu").focus();
@@ -290,6 +314,7 @@ class CommandorPage:
def termine(self):
self.logger.info("will close browser")
time.sleep(1)
self.browser.close()
def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str):
@@ -305,6 +330,29 @@ class CommandorPage:
time.sleep(2)
self.browser.close()
def resolve_captcha(self):
self.captcha_solver = SolveCaptcha(self.page)
self.captcha_solver.start(self.fill_captcha_solution)
def fill_captcha_solution(self, solution):
self.logger.info("will input solution: " + solution)
try:
self.page.evaluate("""(solution)=>{
document.getElementById("g-recaptcha-response").innerHTML=solution;}""", solution)
self.logger.info("will click on valid btn")
self.clickOnValidBtn()
# wait for 20s
time.sleep(20)
if not self.is_finished:
if not self.is_captcha_in_error:
self.clickOnValidBtn()
else:
self.is_captcha_in_error = False
except Exception as error:
self.logger.error(error)
self.page.reload(timeout=PAGE_TIMEOUT)
def get_random_id_number() -> str:
S = 8 # number of characters in the string.