panleicim/appointment_tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

correction on the check_results

Browse Source
This commit is contained in:
Lei PAN
2022-07-07 09:42:05 +02:00
parent c3204b48e5
commit 3f08f7d3fd
9 changed files with 38 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.py
+2 -2
View File
@@ -2,7 +2,7 @@ import logging
import sys
from concurrent.futures import ThreadPoolExecutor
from src import params
from src import params, definitions
from src.logs.AppLogging import init_logger
from src.pojo.ModeEnum import ModeEnum
from src.pojo.contact_pojo import ContactPojo
@@ -40,7 +40,7 @@ def start_book(start_number, end_number, store_choose_state=0, max_workers=10, p
def recheck_the_captcha_error_contacts(on_no_contact_found, store_type=0, mode: ModeEnum = ModeEnum.MANUAL,
max_workers=10):
# get all the contacts in captcha error
contact_list = params.mongo_store_manager.get_captcha_error_contacts_for_current_day()
contact_list = definitions.mongo_store_manager.get_captcha_error_contacts_for_current_day()
if len(contact_list) == 0:
on_no_contact_found()
with ThreadPoolExecutor(max_workers=max_workers) as executor:
src/check_results.py
+9 -7
View File
@@ -13,6 +13,7 @@ from notification.mailer import Mailer
from pojo.ReserveResultPojo import ReserveResultPojo
from pojo.ResultEnum import ResultEnum
from proxy.proxy_type import ProxyType
from src import definitions
SORRY_SENTENCE_FR = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci"
SORRY_SENTENCE_EN = "we are extremely sorry that we were not able to fulfill"
@@ -66,10 +67,10 @@ class ResultChecker:
url = reserve_pojo.url
print("url is " + url)
content = None
proxy = params.get_proxy(ProxyType.NETNUT)
proxy = params.get_proxy(ProxyType.BRIGHT_DATA)
while content is None:
content = self.load_page(self.tls.playwright, proxy, url)
proxy = params.get_proxy(ProxyType.NETNUT)
proxy = params.get_proxy(ProxyType.BRIGHT_DATA)
print(content)
self.browser.close()
print("Stopped worker in ", threading.current_thread().name)
@@ -97,25 +98,26 @@ class ResultChecker:
mailer.send_email(get_accepted_result_from(reserve_pojo))
except Exception as err:
print(err)
reserve_pojo.accepted = status
firestore_collection.document(reserve_pojo.id).update({u'accepted': status.name})
params.mongo_store_manager.update_reserve_result(reserve_pojo.id, status)
definitions.mongo_store_manager.update_reserve_result(reserve_pojo.id, status)
def check_results():
# get the list
params.oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS)
db_manager = params.firebase_store_manager
db_manager = definitions.firebase_store_manager
firestore_collection = db_manager.get_all_successful_items()
reserve_list = params.mongo_store_manager.get_all_successful_items_for_day()
reserve_list = definitions.mongo_store_manager.get_all_successful_items_for_day()
print("size is " + str(len(reserve_list)))
start_check(reserve_list, firestore_collection, False)
reserve_list = params.mongo_store_manager.get_all_successful_items_for_day()
reserve_list = definitions.mongo_store_manager.get_all_successful_items_for_day()
start_check(reserve_list, firestore_collection, True)
def start_check(reserve_list, firestore_collection, need_send_email: bool):
count = 0
with ThreadPoolExecutor(max_workers=15) as executor:
with ThreadPoolExecutor(max_workers=20) as executor:
for reserve in reserve_list:
count = count + 1
if reserve.accepted is None or ResultEnum.ACCEPTED.value == reserve.accepted:
src/db/DbManager.py
-1
View File
@@ -60,7 +60,6 @@ class DataManager:
collection_name = str(datetime.date.today())
doc_ref = self._db.collection(collection_name).document(result.id)
doc_ref.set(result.to_firestore_dict())
params.mongo_store_manager.insert_reserve_result(collection_name=collection_name, reserve=result)
def find_appointment_detail_via_phone(self, day, phone) -> ReserveResultPojo:
params.oracle_log_sender.send_read_db_event("find_appointment_detail_via_phone")
src/definitions.py
+9 -2
View File
@@ -4,9 +4,8 @@ import os
from pathlib import Path
home = str(Path.home())
config = configparser.ConfigParser()
print("home path: " + home)
# check the config file exsistence
# check the config file existence
config_file_path = home + "/config.ini"
config.read(config_file_path)
CONTACT_LIST_FILE = config['DEFAULT']['contact_list_file']
@@ -15,3 +14,11 @@ LOGS_DIR = config['DEFAULT']['LOGS_DIR']
username = getpass.getuser()
LOG_SOURCE = username
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
from src.db.mongo_manager import MongoDbManager
mongo_store_manager = MongoDbManager()
from src.db.DbManager import DataManager
firebase_store_manager = DataManager()
src/notification/mailer.py
+3 -3
View File
@@ -4,7 +4,7 @@ from mako.template import Template
import boto3
import definitions
from src import definitions
from src.notification import AcceptedResultPojo
TEMPLATE_NAME = 'scrapy_template_email'
@@ -26,8 +26,8 @@ class Mailer:
aws_secret_access_key=secret)
def send_email(self, result: AcceptedResultPojo):
recipients = ['panleicim@gmail.com', 'kamenonly@gmail.com', 'tangliang0411@gmail.com']
# recipients = ['panleicim@gmail.com']
# recipients = ['panleicim@gmail.com', 'kamenonly@gmail.com', 'tangliang0411@gmail.com']
recipients = ['panleicim@gmail.com']
mytemplate = Template(filename=definitions.ROOT_DIR + "/templates/appointment_results.html")
self.logger.info("send email to " + str(recipients))
src/params.py
-5
View File
@@ -2,13 +2,10 @@ import random
import string
from src import definitions
from src.db.DbManager import DataManager
from src.db.local_db_manager import LocalDbManager
from src.db.mongo_manager import MongoDbManager
from src.logs.LogSender import LogSender
from src.proxy.proxy_type import ProxyType
firebase_store_manager = DataManager()
oracle_log_sender = LogSender()
# proxy
@@ -23,8 +20,6 @@ BRIGHT_DATA_MOBILE_PROXY_PASSWORD = "fk5f7c2z2c19"
PROXY_NAME_PREFIX_RES = "panleicim-res-fr-"
PROXY_NAME_PREFIX_CC = "panleicim-cc-fr-"
mongo_store_manager = MongoDbManager()
def get_proxy_name_prefix() -> str:
return PROXY_NAME_PREFIX_RES
src/utils/black_list_checker.py
+2 -2
View File
@@ -1,13 +1,13 @@
import time
from src import params
from src import params, definitions
from src.pojo.contact_pojo import ContactPojo
SEVEN_DAYS_IN_S = 7 * 24 * 3600
def can_send_request(contact: ContactPojo) -> bool:
black_list = params.mongo_store_manager.get_blacklist_contacts()
black_list = definitions.mongo_store_manager.get_blacklist_contacts()
for black_contact in black_list:
if contact.mail == black_contact.mail:
# check date
src/utils/export_valid_profiles.py
+4 -4
View File
@@ -3,9 +3,9 @@ import string
import xlsxwriter
import params
from pojo.ReserveResultPojo import ReserveResultPojo
from pojo.contact_pojo import ContactPojo
from src import definitions
from src.pojo.ReserveResultPojo import ReserveResultPojo
from src.pojo.contact_pojo import ContactPojo
def get_random_id_number() -> str:
@@ -23,7 +23,7 @@ def write_the_valid_profiles_to_excel():
collection = []
for day in day_list:
collection.extend(params.firebase_store_manager.get_all_successful_items_for_day(day, source_from=None).stream())
collection.extend(definitions.firebase_store_manager.get_all_successful_items_for_day(day, source_from=None).stream())
valid_contacts = []
# exist_contacts = ExcelHelper().read_contacts()
src/workers/commandor_page.py
+9 -6
View File
@@ -1,3 +1,4 @@
import datetime
import logging
import random
import re
@@ -9,7 +10,7 @@ from typing import Union
from playwright.sync_api import sync_playwright
from src import params
from src import params, definitions
from src.pojo.ModeEnum import ModeEnum
from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType
from src.pojo.captcha_error_contact_pojo import TOO_MANY_REQUEST_ERROR, ERROR_TYPE_CAPTCHA
@@ -257,7 +258,7 @@ class CommandorPage:
params.local_db_manager.insert_or_update(
get_captcha_error_contact_from_contact(self.contact, TOO_MANY_REQUEST_ERROR))
params.oracle_log_sender.send_too_many_error(self.contact)
params.mongo_store_manager.insert_blacklist_contact(self.contact)
definitions.mongo_store_manager.insert_blacklist_contact(self.contact)
self.is_finished = True
self.termine()
elif CAPTCHA_ERROR_MESSAGE in erro_content or CAPTCHA_ERROR_MESSAGE_FR in erro_content:
@@ -329,10 +330,12 @@ class CommandorPage:
passport=contact.passport, ccid=contact.ccid)
result.id = id
result.store_type = self.store_type
params.firebase_store_manager.save(result)
definitions.firebase_store_manager.save(result)
collection_name = str(datetime.date.today())
definitions.mongo_store_manager.insert_reserve_result(collection_name=collection_name, reserve=result)
params.local_db_manager.handle_success(result)
params.mongo_store_manager.delete_captcha_error_contact_for_current_day(self.contact)
params.mongo_store_manager.remove_contact_from_black_list(self.contact)
definitions.mongo_store_manager.delete_captcha_error_contact_for_current_day(self.contact)
definitions.mongo_store_manager.remove_contact_from_black_list(self.contact)
if status is PublishType.SUCCESS:
self.on_success(result)
time.sleep(2)
@@ -364,7 +367,7 @@ class CommandorPage:
def handle_captcha_error(self):
params.local_db_manager.insert_or_update(
get_captcha_error_contact_from_contact(self.contact, ERROR_TYPE_CAPTCHA))
params.mongo_store_manager.insert_captcha_error_contact(self.contact)
definitions.mongo_store_manager.insert_captcha_error_contact(self.contact)
params.oracle_log_sender.send_captcha_error(self.contact)