From fe56c47d50d32fff38041a7359882046ec109568 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Fri, 8 Jul 2022 09:04:45 +0200 Subject: [PATCH] remove local db --- main.py | 3 +- scheduler.py | 3 +- src/check_results.py | 16 +++--- src/config.py | 16 ++++++ src/db/DbManager.py | 4 +- src/db/local_db_manager.py | 54 ------------------- src/db/mongo_manager.py | 4 +- src/definitions.py | 21 +------- src/logs/AppLogging.py | 2 +- src/logs/LogSender.py | 4 +- src/notification/mailer.py | 10 ++-- src/params.py | 4 -- src/pojo/ReserveResultPojo.py | 4 +- .../templates}/appointment_results.html | 0 src/utils/black_list_checker.py | 3 +- src/utils/excel_reader.py | 2 +- src/workers/SolveCaptch.py | 3 +- src/workers/commandor_page.py | 15 +++--- 18 files changed, 57 insertions(+), 111 deletions(-) create mode 100644 src/config.py delete mode 100644 src/db/local_db_manager.py rename {templates => src/templates}/appointment_results.html (100%) diff --git a/main.py b/main.py index 099b97e..6021e3f 100644 --- a/main.py +++ b/main.py @@ -3,6 +3,7 @@ import sys from concurrent.futures import ThreadPoolExecutor from src import params, definitions +from src.db.mongo_manager import MONGO_STORE_MANAGER from src.logs.AppLogging import init_logger from src.pojo.ModeEnum import ModeEnum from src.pojo.contact_pojo import ContactPojo @@ -40,7 +41,7 @@ def start_book(start_number, end_number, store_choose_state=0, max_workers=10, p def recheck_the_captcha_error_contacts(on_no_contact_found, store_type=0, mode: ModeEnum = ModeEnum.MANUAL, max_workers=10): # get all the contacts in captcha error - contact_list = definitions.mongo_store_manager.get_captcha_error_contacts_for_current_day() + contact_list = MONGO_STORE_MANAGER.get_captcha_error_contacts_for_current_day() if len(contact_list) == 0: on_no_contact_found() with ThreadPoolExecutor(max_workers=max_workers) as executor: diff --git a/scheduler.py b/scheduler.py index 08b27ca..bbad1b5 100644 --- a/scheduler.py +++ b/scheduler.py @@ -1,12 +1,13 @@ from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor from apscheduler.schedulers.blocking import BlockingScheduler -from check_results import check_results +from src.check_results import check_results def check_results_job(): check_results() + def start_check_results_job(sched): # sched.add_job(start_book_appointment, 'cron', day_of_week='mon-sat', hour='10', # minute='30', diff --git a/src/check_results.py b/src/check_results.py index 9c637fa..31e6657 100644 --- a/src/check_results.py +++ b/src/check_results.py @@ -6,14 +6,15 @@ from typing import Union from playwright.sync_api import sync_playwright -import params -from logs.LogSender import TYPE_EVENT_CHECK_RESULTS, LOG_SUBJECT_EVENT +from src.db.mongo_manager import MONGO_STORE_MANAGER +from src.definitions import LOG_SUBJECT_EVENT, TYPE_EVENT_CHECK_RESULTS +from src.logs.LogSender import LogSender from notification.AcceptedResultPojo import get_accepted_result_from from notification.mailer import Mailer from pojo.ReserveResultPojo import ReserveResultPojo from pojo.ResultEnum import ResultEnum from proxy.proxy_type import ProxyType -from src import definitions +from src import definitions, params SORRY_SENTENCE_FR = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci" SORRY_SENTENCE_EN = "we are extremely sorry that we were not able to fulfill" @@ -25,6 +26,7 @@ BLANK_URL = "about:blank" WELCOME_URL = "https://rendezvousparis.hermes.com/client/welcome" mailer = Mailer() +oracle_log_sender = LogSender() class TlsPlaywright(threading.local): @@ -100,18 +102,18 @@ class ResultChecker: print(err) reserve_pojo.accepted = status firestore_collection.document(reserve_pojo.id).update({u'accepted': status.name}) - definitions.mongo_store_manager.update_reserve_result(reserve_pojo.id, status) + MONGO_STORE_MANAGER.update_reserve_result(reserve_pojo.id, status) def check_results(): # get the list - params.oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS) + oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS) db_manager = definitions.firebase_store_manager firestore_collection = db_manager.get_all_successful_items() - reserve_list = definitions.mongo_store_manager.get_all_successful_items_for_day() + reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day() print("size is " + str(len(reserve_list))) start_check(reserve_list, firestore_collection, False) - reserve_list = definitions.mongo_store_manager.get_all_successful_items_for_day() + reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day() start_check(reserve_list, firestore_collection, True) diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..8632ba1 --- /dev/null +++ b/src/config.py @@ -0,0 +1,16 @@ +import configparser +import getpass +import os +from pathlib import Path +home = str(Path.home()) +config = configparser.ConfigParser() + +# check the config file existence +config_file_path = home + "/config.ini" +config.read(config_file_path) +CONTACT_LIST_FILE = config['DEFAULT']['contact_list_file'] +FIREBASE_CONFIG_FILE = config['DEFAULT']['firebase_config_file'] +LOGS_DIR = config['DEFAULT']['LOGS_DIR'] +username = getpass.getuser() +LOG_SOURCE = username +ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) \ No newline at end of file diff --git a/src/db/DbManager.py b/src/db/DbManager.py index 483da21..774e3c1 100644 --- a/src/db/DbManager.py +++ b/src/db/DbManager.py @@ -4,7 +4,7 @@ from typing import Union import firebase_admin from firebase_admin import credentials, firestore -from src import definitions, params +from src import params, config from src.pojo.MailPojo import MailPojo from src.pojo.ReserveResultPojo import ReserveResultPojo from src.pojo.ResultEnum import ResultEnum @@ -22,7 +22,7 @@ class DataManager: batch_size = 20 def __init__(self): - cred = credentials.Certificate(definitions.FIREBASE_CONFIG_FILE) + cred = credentials.Certificate(config.FIREBASE_CONFIG_FILE) self._app = firebase_admin.initialize_app(cred) self._db = firestore.client() diff --git a/src/db/local_db_manager.py b/src/db/local_db_manager.py deleted file mode 100644 index 1ba3a7b..0000000 --- a/src/db/local_db_manager.py +++ /dev/null @@ -1,54 +0,0 @@ -from builtins import list - -import sqlalchemy -from sqlalchemy import MetaData, Column, String, Integer, DateTime, Table -from sqlalchemy.orm import Session -from sqlalchemy_utils import database_exists, create_database - -from src.pojo.ReserveResultPojo import ReserveResultPojo -from src.pojo.captcha_error_contact_pojo import ContactInErrorPojo - - -class LocalDbManager: - - def __init__(self, path: str): - self.session = Session(self.init_db(path)) - - def init_db(self, path: str): - uri_for_db = "sqlite:///{}/{}.db?check_same_thread=false".format(path, "appointment") - print(uri_for_db) - # 2.-Turn on database engine - db_engine = sqlalchemy.create_engine(uri_for_db) # ensure this is the correct path for the sqlite file. - if not database_exists(uri_for_db): - create_database(uri_for_db) - connextion = db_engine.connect() - if not db_engine.dialect.has_table(connextion, - ContactInErrorPojo.__tablename__): # If table don't exist, Create. - metadata = MetaData(db_engine) - # Create a table with the appropriate Columns - Table(ContactInErrorPojo.__tablename__, metadata, - Column('id', Integer, primary_key=True, autoincrement=True), - Column('mail', String), - Column('phone', String), - Column('passport', String), - Column('last_name', String), - Column('first_name', String), - Column('ccid', String), - Column('position', Integer), - Column('error_type', Integer), - Column('update_at', DateTime), - Column('create_at', DateTime)) - # Implement the creation - metadata.create_all() - - return db_engine - - def insert_or_update(self, instance: ContactInErrorPojo): - self.session.merge(instance) - self.session.commit() - - def handle_success(self, reservePojo: ReserveResultPojo): - # delete the contact from table - self.session.query(ContactInErrorPojo).filter(ContactInErrorPojo.mail == reservePojo.email).delete() - - diff --git a/src/db/mongo_manager.py b/src/db/mongo_manager.py index 2894968..4cc49e1 100644 --- a/src/db/mongo_manager.py +++ b/src/db/mongo_manager.py @@ -117,8 +117,10 @@ class MongoDbManager: self.logger.info(error) +MONGO_STORE_MANAGER = MongoDbManager() + if __name__ == '__main__': db_manager = MongoDbManager() black_list = db_manager.get_blacklist_contacts() for contact in black_list: - print(contact) \ No newline at end of file + print(contact) diff --git a/src/definitions.py b/src/definitions.py index 4957f45..f0b072c 100644 --- a/src/definitions.py +++ b/src/definitions.py @@ -1,23 +1,6 @@ -import configparser -import getpass -import os -from pathlib import Path -home = str(Path.home()) -config = configparser.ConfigParser() -# check the config file existence -config_file_path = home + "/config.ini" -config.read(config_file_path) -CONTACT_LIST_FILE = config['DEFAULT']['contact_list_file'] -FIREBASE_CONFIG_FILE = config['DEFAULT']['firebase_config_file'] -LOGS_DIR = config['DEFAULT']['LOGS_DIR'] -username = getpass.getuser() -LOG_SOURCE = username -ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) - -from src.db.mongo_manager import MongoDbManager - -mongo_store_manager = MongoDbManager() +LOG_SUBJECT_EVENT = "EVENT" +TYPE_EVENT_CHECK_RESULTS = "EVENT_CHECK_RESULTS" from src.db.DbManager import DataManager diff --git a/src/logs/AppLogging.py b/src/logs/AppLogging.py index 9ed941c..7805764 100644 --- a/src/logs/AppLogging.py +++ b/src/logs/AppLogging.py @@ -1,7 +1,7 @@ import datetime import logging -from src.definitions import LOGS_DIR +from src.config import LOGS_DIR def init_logger(): diff --git a/src/logs/LogSender.py b/src/logs/LogSender.py index 17b5ad3..3c6a0af 100644 --- a/src/logs/LogSender.py +++ b/src/logs/LogSender.py @@ -6,7 +6,7 @@ import oci from oci.loggingingestion import LoggingClient from oci.loggingingestion.models import PutLogsDetails, LogEntryBatch, LogEntry -from src import definitions +from src import config from src.logs.AppLogging import init_logger from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType from src.pojo.contact_pojo import ContactPojo @@ -96,7 +96,7 @@ class LogSender: def send_read_db_event(self, msg: str): self.send_log(msg=msg, type=TYPE_EVENT_READ_DB, subject=LOG_SUBJECT_EVENT) - def send_log(self, msg: str, source=definitions.LOG_SOURCE, subject="appointment", type: str = "INFO"): + def send_log(self, msg: str, source=config.LOG_SOURCE, subject="appointment", type: str = "INFO"): log_id = "ocid1.log.oc1.eu-frankfurt-1.amaaaaaas4ft22ya3ub6glkltqqbnmkxo3ui7xwq3dxtjd2scdhme4deyu2q" response = self._loggingingestion_client.put_logs( log_id=log_id, diff --git a/src/notification/mailer.py b/src/notification/mailer.py index 01e3d97..3c3bc89 100644 --- a/src/notification/mailer.py +++ b/src/notification/mailer.py @@ -1,10 +1,10 @@ import base64 import logging -from mako.template import Template import boto3 +from mako.template import Template -from src import definitions +from src import config from src.notification import AcceptedResultPojo TEMPLATE_NAME = 'scrapy_template_email' @@ -26,10 +26,10 @@ class Mailer: aws_secret_access_key=secret) def send_email(self, result: AcceptedResultPojo): - # recipients = ['panleicim@gmail.com', 'kamenonly@gmail.com', 'tangliang0411@gmail.com'] - recipients = ['panleicim@gmail.com'] + recipients = ['panleicim@gmail.com', 'kamenonly@gmail.com', 'tangliang0411@gmail.com'] + # recipients = ['panleicim@gmail.com'] - mytemplate = Template(filename=definitions.ROOT_DIR + "/templates/appointment_results.html") + mytemplate = Template(filename=config.ROOT_DIR + "/templates/appointment_results.html") self.logger.info("send email to " + str(recipients)) self.ses_client.send_email( FromEmailAddress="noreply@lpaconsulting.fr", diff --git a/src/params.py b/src/params.py index 672f8a5..6543800 100644 --- a/src/params.py +++ b/src/params.py @@ -1,8 +1,6 @@ import random import string -from src import definitions -from src.db.local_db_manager import LocalDbManager from src.logs.LogSender import LogSender from src.proxy.proxy_type import ProxyType @@ -51,8 +49,6 @@ def get_random_id_number_for_proxy() -> str: return id_number -local_db_manager = LocalDbManager(definitions.home) - DEVICES = ['iPad (gen 6)', 'iPad (gen 6) landscape', 'iPad (gen 7)', 'iPad (gen 7) landscape', 'iPad Mini', 'iPad Mini landscape', 'iPad Pro 11', 'iPad Pro 11 landscape', 'iPhone 6', 'iPhone 6 landscape', 'iPhone 6 Plus', 'iPhone 6 Plus landscape', 'iPhone 7', 'iPhone 7 landscape', 'iPhone 7 Plus', diff --git a/src/pojo/ReserveResultPojo.py b/src/pojo/ReserveResultPojo.py index e4a1ef2..ca69069 100644 --- a/src/pojo/ReserveResultPojo.py +++ b/src/pojo/ReserveResultPojo.py @@ -4,7 +4,7 @@ from typing import Union from dataclasses_json import dataclass_json -from src import definitions +from src import config class PublishType(Enum): @@ -29,7 +29,7 @@ class ReserveResultPojo: slot_position = None sim_position = None ccid: str = "" - source_from: str = definitions.LOG_SOURCE + source_from: str = config.LOG_SOURCE store_type = 0 @staticmethod diff --git a/templates/appointment_results.html b/src/templates/appointment_results.html similarity index 100% rename from templates/appointment_results.html rename to src/templates/appointment_results.html diff --git a/src/utils/black_list_checker.py b/src/utils/black_list_checker.py index d369f50..560ef5a 100644 --- a/src/utils/black_list_checker.py +++ b/src/utils/black_list_checker.py @@ -1,13 +1,14 @@ import time from src import params, definitions +from src.db.mongo_manager import MONGO_STORE_MANAGER from src.pojo.contact_pojo import ContactPojo SEVEN_DAYS_IN_S = 7 * 24 * 3600 def can_send_request(contact: ContactPojo) -> bool: - black_list = definitions.mongo_store_manager.get_blacklist_contacts() + black_list = MONGO_STORE_MANAGER.get_blacklist_contacts() for black_contact in black_list: if contact.mail == black_contact.mail: # check date diff --git a/src/utils/excel_reader.py b/src/utils/excel_reader.py index 0c586e9..440daa1 100644 --- a/src/utils/excel_reader.py +++ b/src/utils/excel_reader.py @@ -5,7 +5,7 @@ import string import pandas as pandas import xlsxwriter -from src.definitions import CONTACT_LIST_FILE +from src.config import CONTACT_LIST_FILE from src.pojo.contact_pojo import ContactPojo from src.utils.generate_random_passport_id import get_random_passport_id_number diff --git a/src/workers/SolveCaptch.py b/src/workers/SolveCaptch.py index e32f4fe..326ea97 100644 --- a/src/workers/SolveCaptch.py +++ b/src/workers/SolveCaptch.py @@ -50,5 +50,6 @@ class SolveCaptcha: solution_res = requests.get(url_response) time.sleep(5) solution = solution_res.text - self.logger.info(solution) + self.logger.info("response code: " + str(res.status_code)) + # self.logger.info(solution) handle_solution_received(solution.split("|")[-1]) diff --git a/src/workers/commandor_page.py b/src/workers/commandor_page.py index f46ab63..27351db 100644 --- a/src/workers/commandor_page.py +++ b/src/workers/commandor_page.py @@ -11,6 +11,7 @@ from typing import Union from playwright.sync_api import sync_playwright from src import params, definitions +from src.db.mongo_manager import MONGO_STORE_MANAGER from src.pojo.ModeEnum import ModeEnum from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType from src.pojo.captcha_error_contact_pojo import TOO_MANY_REQUEST_ERROR, ERROR_TYPE_CAPTCHA @@ -255,10 +256,8 @@ class CommandorPage: elif TOO_MANY_REQUEST_ERROR_MESSAGE in erro_content or TOO_MANY_REQUEST_ERROR_MESSAGE_FR in erro_content: # this email is in black list if not self.is_finished: - params.local_db_manager.insert_or_update( - get_captcha_error_contact_from_contact(self.contact, TOO_MANY_REQUEST_ERROR)) params.oracle_log_sender.send_too_many_error(self.contact) - definitions.mongo_store_manager.insert_blacklist_contact(self.contact) + MONGO_STORE_MANAGER.insert_blacklist_contact(self.contact) self.is_finished = True self.termine() elif CAPTCHA_ERROR_MESSAGE in erro_content or CAPTCHA_ERROR_MESSAGE_FR in erro_content: @@ -332,9 +331,9 @@ class CommandorPage: result.store_type = self.store_type definitions.firebase_store_manager.save(result) collection_name = str(datetime.date.today()) - definitions.mongo_store_manager.insert_reserve_result(collection_name=collection_name, reserve=result) - definitions.mongo_store_manager.delete_captcha_error_contact_for_current_day(self.contact) - definitions.mongo_store_manager.remove_contact_from_black_list(self.contact) + MONGO_STORE_MANAGER.insert_reserve_result(collection_name=collection_name, reserve=result) + MONGO_STORE_MANAGER.delete_captcha_error_contact_for_current_day(self.contact) + MONGO_STORE_MANAGER.remove_contact_from_black_list(self.contact) if status is PublishType.SUCCESS: self.on_success(result) time.sleep(2) @@ -364,9 +363,7 @@ class CommandorPage: self.page.reload(timeout=PAGE_TIMEOUT) def handle_captcha_error(self): - params.local_db_manager.insert_or_update( - get_captcha_error_contact_from_contact(self.contact, ERROR_TYPE_CAPTCHA)) - definitions.mongo_store_manager.insert_captcha_error_contact(self.contact) + MONGO_STORE_MANAGER.insert_captcha_error_contact(self.contact) params.oracle_log_sender.send_captcha_error(self.contact)