diff --git a/main.py b/main.py index 03e204b..099b97e 100644 --- a/main.py +++ b/main.py @@ -2,7 +2,7 @@ import logging import sys from concurrent.futures import ThreadPoolExecutor -from src import params +from src import params, definitions from src.logs.AppLogging import init_logger from src.pojo.ModeEnum import ModeEnum from src.pojo.contact_pojo import ContactPojo @@ -40,7 +40,7 @@ def start_book(start_number, end_number, store_choose_state=0, max_workers=10, p def recheck_the_captcha_error_contacts(on_no_contact_found, store_type=0, mode: ModeEnum = ModeEnum.MANUAL, max_workers=10): # get all the contacts in captcha error - contact_list = params.mongo_store_manager.get_captcha_error_contacts_for_current_day() + contact_list = definitions.mongo_store_manager.get_captcha_error_contacts_for_current_day() if len(contact_list) == 0: on_no_contact_found() with ThreadPoolExecutor(max_workers=max_workers) as executor: diff --git a/src/check_results.py b/src/check_results.py index 1fa5c47..9c637fa 100644 --- a/src/check_results.py +++ b/src/check_results.py @@ -13,6 +13,7 @@ from notification.mailer import Mailer from pojo.ReserveResultPojo import ReserveResultPojo from pojo.ResultEnum import ResultEnum from proxy.proxy_type import ProxyType +from src import definitions SORRY_SENTENCE_FR = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci" SORRY_SENTENCE_EN = "we are extremely sorry that we were not able to fulfill" @@ -66,10 +67,10 @@ class ResultChecker: url = reserve_pojo.url print("url is " + url) content = None - proxy = params.get_proxy(ProxyType.NETNUT) + proxy = params.get_proxy(ProxyType.BRIGHT_DATA) while content is None: content = self.load_page(self.tls.playwright, proxy, url) - proxy = params.get_proxy(ProxyType.NETNUT) + proxy = params.get_proxy(ProxyType.BRIGHT_DATA) print(content) self.browser.close() print("Stopped worker in ", threading.current_thread().name) @@ -97,25 +98,26 @@ class ResultChecker: mailer.send_email(get_accepted_result_from(reserve_pojo)) except Exception as err: print(err) + reserve_pojo.accepted = status firestore_collection.document(reserve_pojo.id).update({u'accepted': status.name}) - params.mongo_store_manager.update_reserve_result(reserve_pojo.id, status) + definitions.mongo_store_manager.update_reserve_result(reserve_pojo.id, status) def check_results(): # get the list params.oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS) - db_manager = params.firebase_store_manager + db_manager = definitions.firebase_store_manager firestore_collection = db_manager.get_all_successful_items() - reserve_list = params.mongo_store_manager.get_all_successful_items_for_day() + reserve_list = definitions.mongo_store_manager.get_all_successful_items_for_day() print("size is " + str(len(reserve_list))) start_check(reserve_list, firestore_collection, False) - reserve_list = params.mongo_store_manager.get_all_successful_items_for_day() + reserve_list = definitions.mongo_store_manager.get_all_successful_items_for_day() start_check(reserve_list, firestore_collection, True) def start_check(reserve_list, firestore_collection, need_send_email: bool): count = 0 - with ThreadPoolExecutor(max_workers=15) as executor: + with ThreadPoolExecutor(max_workers=20) as executor: for reserve in reserve_list: count = count + 1 if reserve.accepted is None or ResultEnum.ACCEPTED.value == reserve.accepted: diff --git a/src/db/DbManager.py b/src/db/DbManager.py index 37971d4..483da21 100644 --- a/src/db/DbManager.py +++ b/src/db/DbManager.py @@ -60,7 +60,6 @@ class DataManager: collection_name = str(datetime.date.today()) doc_ref = self._db.collection(collection_name).document(result.id) doc_ref.set(result.to_firestore_dict()) - params.mongo_store_manager.insert_reserve_result(collection_name=collection_name, reserve=result) def find_appointment_detail_via_phone(self, day, phone) -> ReserveResultPojo: params.oracle_log_sender.send_read_db_event("find_appointment_detail_via_phone") diff --git a/src/definitions.py b/src/definitions.py index 447ef16..4957f45 100644 --- a/src/definitions.py +++ b/src/definitions.py @@ -4,9 +4,8 @@ import os from pathlib import Path home = str(Path.home()) config = configparser.ConfigParser() -print("home path: " + home) -# check the config file exsistence +# check the config file existence config_file_path = home + "/config.ini" config.read(config_file_path) CONTACT_LIST_FILE = config['DEFAULT']['contact_list_file'] @@ -15,3 +14,11 @@ LOGS_DIR = config['DEFAULT']['LOGS_DIR'] username = getpass.getuser() LOG_SOURCE = username ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) + +from src.db.mongo_manager import MongoDbManager + +mongo_store_manager = MongoDbManager() + +from src.db.DbManager import DataManager + +firebase_store_manager = DataManager() diff --git a/src/notification/mailer.py b/src/notification/mailer.py index 9fb6a72..01e3d97 100644 --- a/src/notification/mailer.py +++ b/src/notification/mailer.py @@ -4,7 +4,7 @@ from mako.template import Template import boto3 -import definitions +from src import definitions from src.notification import AcceptedResultPojo TEMPLATE_NAME = 'scrapy_template_email' @@ -26,8 +26,8 @@ class Mailer: aws_secret_access_key=secret) def send_email(self, result: AcceptedResultPojo): - recipients = ['panleicim@gmail.com', 'kamenonly@gmail.com', 'tangliang0411@gmail.com'] - # recipients = ['panleicim@gmail.com'] + # recipients = ['panleicim@gmail.com', 'kamenonly@gmail.com', 'tangliang0411@gmail.com'] + recipients = ['panleicim@gmail.com'] mytemplate = Template(filename=definitions.ROOT_DIR + "/templates/appointment_results.html") self.logger.info("send email to " + str(recipients)) diff --git a/src/params.py b/src/params.py index 525b7d2..672f8a5 100644 --- a/src/params.py +++ b/src/params.py @@ -2,13 +2,10 @@ import random import string from src import definitions -from src.db.DbManager import DataManager from src.db.local_db_manager import LocalDbManager -from src.db.mongo_manager import MongoDbManager from src.logs.LogSender import LogSender from src.proxy.proxy_type import ProxyType -firebase_store_manager = DataManager() oracle_log_sender = LogSender() # proxy @@ -23,8 +20,6 @@ BRIGHT_DATA_MOBILE_PROXY_PASSWORD = "fk5f7c2z2c19" PROXY_NAME_PREFIX_RES = "panleicim-res-fr-" PROXY_NAME_PREFIX_CC = "panleicim-cc-fr-" -mongo_store_manager = MongoDbManager() - def get_proxy_name_prefix() -> str: return PROXY_NAME_PREFIX_RES diff --git a/src/utils/black_list_checker.py b/src/utils/black_list_checker.py index 6914028..d369f50 100644 --- a/src/utils/black_list_checker.py +++ b/src/utils/black_list_checker.py @@ -1,13 +1,13 @@ import time -from src import params +from src import params, definitions from src.pojo.contact_pojo import ContactPojo SEVEN_DAYS_IN_S = 7 * 24 * 3600 def can_send_request(contact: ContactPojo) -> bool: - black_list = params.mongo_store_manager.get_blacklist_contacts() + black_list = definitions.mongo_store_manager.get_blacklist_contacts() for black_contact in black_list: if contact.mail == black_contact.mail: # check date diff --git a/src/utils/export_valid_profiles.py b/src/utils/export_valid_profiles.py index c8d3112..4b8f3dd 100644 --- a/src/utils/export_valid_profiles.py +++ b/src/utils/export_valid_profiles.py @@ -3,9 +3,9 @@ import string import xlsxwriter -import params -from pojo.ReserveResultPojo import ReserveResultPojo -from pojo.contact_pojo import ContactPojo +from src import definitions +from src.pojo.ReserveResultPojo import ReserveResultPojo +from src.pojo.contact_pojo import ContactPojo def get_random_id_number() -> str: @@ -23,7 +23,7 @@ def write_the_valid_profiles_to_excel(): collection = [] for day in day_list: - collection.extend(params.firebase_store_manager.get_all_successful_items_for_day(day, source_from=None).stream()) + collection.extend(definitions.firebase_store_manager.get_all_successful_items_for_day(day, source_from=None).stream()) valid_contacts = [] # exist_contacts = ExcelHelper().read_contacts() diff --git a/src/workers/commandor_page.py b/src/workers/commandor_page.py index 51af8be..d8c35ab 100644 --- a/src/workers/commandor_page.py +++ b/src/workers/commandor_page.py @@ -1,3 +1,4 @@ +import datetime import logging import random import re @@ -9,7 +10,7 @@ from typing import Union from playwright.sync_api import sync_playwright -from src import params +from src import params, definitions from src.pojo.ModeEnum import ModeEnum from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType from src.pojo.captcha_error_contact_pojo import TOO_MANY_REQUEST_ERROR, ERROR_TYPE_CAPTCHA @@ -257,7 +258,7 @@ class CommandorPage: params.local_db_manager.insert_or_update( get_captcha_error_contact_from_contact(self.contact, TOO_MANY_REQUEST_ERROR)) params.oracle_log_sender.send_too_many_error(self.contact) - params.mongo_store_manager.insert_blacklist_contact(self.contact) + definitions.mongo_store_manager.insert_blacklist_contact(self.contact) self.is_finished = True self.termine() elif CAPTCHA_ERROR_MESSAGE in erro_content or CAPTCHA_ERROR_MESSAGE_FR in erro_content: @@ -329,10 +330,12 @@ class CommandorPage: passport=contact.passport, ccid=contact.ccid) result.id = id result.store_type = self.store_type - params.firebase_store_manager.save(result) + definitions.firebase_store_manager.save(result) + collection_name = str(datetime.date.today()) + definitions.mongo_store_manager.insert_reserve_result(collection_name=collection_name, reserve=result) params.local_db_manager.handle_success(result) - params.mongo_store_manager.delete_captcha_error_contact_for_current_day(self.contact) - params.mongo_store_manager.remove_contact_from_black_list(self.contact) + definitions.mongo_store_manager.delete_captcha_error_contact_for_current_day(self.contact) + definitions.mongo_store_manager.remove_contact_from_black_list(self.contact) if status is PublishType.SUCCESS: self.on_success(result) time.sleep(2) @@ -364,7 +367,7 @@ class CommandorPage: def handle_captcha_error(self): params.local_db_manager.insert_or_update( get_captcha_error_contact_from_contact(self.contact, ERROR_TYPE_CAPTCHA)) - params.mongo_store_manager.insert_captcha_error_contact(self.contact) + definitions.mongo_store_manager.insert_captcha_error_contact(self.contact) params.oracle_log_sender.send_captcha_error(self.contact)