remove local db

main.py

@@ -3,6 +3,7 @@ import sys
 from concurrent.futures import ThreadPoolExecutor
 
 from src import params, definitions
+from src.db.mongo_manager import MONGO_STORE_MANAGER
 from src.logs.AppLogging import init_logger
 from src.pojo.ModeEnum import ModeEnum
 from src.pojo.contact_pojo import ContactPojo
@@ -40,7 +41,7 @@ def start_book(start_number, end_number, store_choose_state=0, max_workers=10, p
 def recheck_the_captcha_error_contacts(on_no_contact_found, store_type=0, mode: ModeEnum = ModeEnum.MANUAL,
                                        max_workers=10):
     # get all the contacts in captcha error
-    contact_list = definitions.mongo_store_manager.get_captcha_error_contacts_for_current_day()
+    contact_list = MONGO_STORE_MANAGER.get_captcha_error_contacts_for_current_day()
     if len(contact_list) == 0:
         on_no_contact_found()
     with ThreadPoolExecutor(max_workers=max_workers) as executor:

scheduler.py

@@ -1,12 +1,13 @@
 from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor
 from apscheduler.schedulers.blocking import BlockingScheduler
 
-from check_results import check_results
+from src.check_results import check_results
 
 
 def check_results_job():
     check_results()
 
+
 def start_check_results_job(sched):
     # sched.add_job(start_book_appointment, 'cron', day_of_week='mon-sat', hour='10',
     #               minute='30',

src/check_results.py

@@ -6,14 +6,15 @@ from typing import Union
 
 from playwright.sync_api import sync_playwright
 
-import params
+from src.db.mongo_manager import MONGO_STORE_MANAGER
-from logs.LogSender import TYPE_EVENT_CHECK_RESULTS, LOG_SUBJECT_EVENT
+from src.definitions import LOG_SUBJECT_EVENT, TYPE_EVENT_CHECK_RESULTS
+from src.logs.LogSender import LogSender
 from notification.AcceptedResultPojo import get_accepted_result_from
 from notification.mailer import Mailer
 from pojo.ReserveResultPojo import ReserveResultPojo
 from pojo.ResultEnum import ResultEnum
 from proxy.proxy_type import ProxyType
-from src import definitions
+from src import definitions, params
 
 SORRY_SENTENCE_FR = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci"
 SORRY_SENTENCE_EN = "we are extremely sorry that we were not able to fulfill"
@@ -25,6 +26,7 @@ BLANK_URL = "about:blank"
 WELCOME_URL = "https://rendezvousparis.hermes.com/client/welcome"
 
 mailer = Mailer()
+oracle_log_sender = LogSender()
 
 
 class TlsPlaywright(threading.local):
@@ -100,18 +102,18 @@ class ResultChecker:
                     print(err)
         reserve_pojo.accepted = status
         firestore_collection.document(reserve_pojo.id).update({u'accepted': status.name})
-        definitions.mongo_store_manager.update_reserve_result(reserve_pojo.id, status)
+        MONGO_STORE_MANAGER.update_reserve_result(reserve_pojo.id, status)
 
 
 def check_results():
     # get the list
-    params.oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS)
+    oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS)
     db_manager = definitions.firebase_store_manager
     firestore_collection = db_manager.get_all_successful_items()
-    reserve_list = definitions.mongo_store_manager.get_all_successful_items_for_day()
+    reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
     print("size is " + str(len(reserve_list)))
     start_check(reserve_list, firestore_collection, False)
-    reserve_list = definitions.mongo_store_manager.get_all_successful_items_for_day()
+    reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
     start_check(reserve_list, firestore_collection, True)
 
 

src/config.py

@@ -0,0 +1,16 @@
+import configparser
+import getpass
+import os
+from pathlib import Path
+home = str(Path.home())
+config = configparser.ConfigParser()
+
+# check the config file existence
+config_file_path = home + "/config.ini"
+config.read(config_file_path)
+CONTACT_LIST_FILE = config['DEFAULT']['contact_list_file']
+FIREBASE_CONFIG_FILE = config['DEFAULT']['firebase_config_file']
+LOGS_DIR = config['DEFAULT']['LOGS_DIR']
+username = getpass.getuser()
+LOG_SOURCE = username
+ROOT_DIR = os.path.dirname(os.path.abspath(__file__))

src/db/DbManager.py

@@ -4,7 +4,7 @@ from typing import Union
 import firebase_admin
 from firebase_admin import credentials, firestore
 
-from src import definitions, params
+from src import params, config
 from src.pojo.MailPojo import MailPojo
 from src.pojo.ReserveResultPojo import ReserveResultPojo
 from src.pojo.ResultEnum import ResultEnum
@@ -22,7 +22,7 @@ class DataManager:
     batch_size = 20
 
     def __init__(self):
-        cred = credentials.Certificate(definitions.FIREBASE_CONFIG_FILE)
+        cred = credentials.Certificate(config.FIREBASE_CONFIG_FILE)
         self._app = firebase_admin.initialize_app(cred)
         self._db = firestore.client()
 

src/db/local_db_manager.py

@@ -1,54 +0,0 @@
-from builtins import list
-
-import sqlalchemy
-from sqlalchemy import MetaData, Column, String, Integer, DateTime, Table
-from sqlalchemy.orm import Session
-from sqlalchemy_utils import database_exists, create_database
-
-from src.pojo.ReserveResultPojo import ReserveResultPojo
-from src.pojo.captcha_error_contact_pojo import ContactInErrorPojo
-
-
-class LocalDbManager:
-
-    def __init__(self, path: str):
-        self.session = Session(self.init_db(path))
-
-    def init_db(self, path: str):
-        uri_for_db = "sqlite:///{}/{}.db?check_same_thread=false".format(path, "appointment")
-        print(uri_for_db)
-        # 2.-Turn on database engine
-        db_engine = sqlalchemy.create_engine(uri_for_db)  # ensure this is the correct path for the sqlite file.
-        if not database_exists(uri_for_db):
-            create_database(uri_for_db)
-        connextion = db_engine.connect()
-        if not db_engine.dialect.has_table(connextion,
-                                           ContactInErrorPojo.__tablename__):  # If table don't exist, Create.
-            metadata = MetaData(db_engine)
-            # Create a table with the appropriate Columns
-            Table(ContactInErrorPojo.__tablename__, metadata,
-                  Column('id', Integer, primary_key=True, autoincrement=True),
-                  Column('mail', String),
-                  Column('phone', String),
-                  Column('passport', String),
-                  Column('last_name', String),
-                  Column('first_name', String),
-                  Column('ccid', String),
-                  Column('position', Integer),
-                  Column('error_type', Integer),
-                  Column('update_at', DateTime),
-                  Column('create_at', DateTime))
-            # Implement the creation
-            metadata.create_all()
-
-        return db_engine
-
-    def insert_or_update(self, instance: ContactInErrorPojo):
-        self.session.merge(instance)
-        self.session.commit()
-
-    def handle_success(self, reservePojo: ReserveResultPojo):
-        #         delete the contact from table
-        self.session.query(ContactInErrorPojo).filter(ContactInErrorPojo.mail == reservePojo.email).delete()
-
-

src/db/mongo_manager.py

@@ -117,6 +117,8 @@ class MongoDbManager:
             self.logger.info(error)
 
 
+MONGO_STORE_MANAGER = MongoDbManager()
+
 if __name__ == '__main__':
     db_manager = MongoDbManager()
     black_list = db_manager.get_blacklist_contacts()

src/definitions.py

@@ -1,23 +1,6 @@
-import configparser
-import getpass
-import os
-from pathlib import Path
-home = str(Path.home())
-config = configparser.ConfigParser()
 
-# check the config file existence
+LOG_SUBJECT_EVENT = "EVENT"
-config_file_path = home + "/config.ini"
+TYPE_EVENT_CHECK_RESULTS = "EVENT_CHECK_RESULTS"
-config.read(config_file_path)
-CONTACT_LIST_FILE = config['DEFAULT']['contact_list_file']
-FIREBASE_CONFIG_FILE = config['DEFAULT']['firebase_config_file']
-LOGS_DIR = config['DEFAULT']['LOGS_DIR']
-username = getpass.getuser()
-LOG_SOURCE = username
-ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
-
-from src.db.mongo_manager import MongoDbManager
-
-mongo_store_manager = MongoDbManager()
 
 from src.db.DbManager import DataManager
 

src/logs/AppLogging.py

@@ -1,7 +1,7 @@
 import datetime
 import logging
 
-from src.definitions import LOGS_DIR
+from src.config import LOGS_DIR
 
 
 def init_logger():

src/logs/LogSender.py

@@ -6,7 +6,7 @@ import oci
 from oci.loggingingestion import LoggingClient
 from oci.loggingingestion.models import PutLogsDetails, LogEntryBatch, LogEntry
 
-from src import definitions
+from src import config
 from src.logs.AppLogging import init_logger
 from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType
 from src.pojo.contact_pojo import ContactPojo
@@ -96,7 +96,7 @@ class LogSender:
     def send_read_db_event(self, msg: str):
         self.send_log(msg=msg, type=TYPE_EVENT_READ_DB, subject=LOG_SUBJECT_EVENT)
 
-    def send_log(self, msg: str, source=definitions.LOG_SOURCE, subject="appointment", type: str = "INFO"):
+    def send_log(self, msg: str, source=config.LOG_SOURCE, subject="appointment", type: str = "INFO"):
         log_id = "ocid1.log.oc1.eu-frankfurt-1.amaaaaaas4ft22ya3ub6glkltqqbnmkxo3ui7xwq3dxtjd2scdhme4deyu2q"
         response = self._loggingingestion_client.put_logs(
             log_id=log_id,

src/notification/mailer.py

@@ -1,10 +1,10 @@
 import base64
 import logging
-from mako.template import Template
 
 import boto3
+from mako.template import Template
 
-from src import definitions
+from src import config
 from src.notification import AcceptedResultPojo
 
 TEMPLATE_NAME = 'scrapy_template_email'
@@ -26,10 +26,10 @@ class Mailer:
                                        aws_secret_access_key=secret)
 
     def send_email(self, result: AcceptedResultPojo):
-        # recipients = ['panleicim@gmail.com', 'kamenonly@gmail.com', 'tangliang0411@gmail.com']
+        recipients = ['panleicim@gmail.com', 'kamenonly@gmail.com', 'tangliang0411@gmail.com']
-        recipients = ['panleicim@gmail.com']
+        # recipients = ['panleicim@gmail.com']
 
-        mytemplate = Template(filename=definitions.ROOT_DIR + "/templates/appointment_results.html")
+        mytemplate = Template(filename=config.ROOT_DIR + "/templates/appointment_results.html")
         self.logger.info("send email to " + str(recipients))
         self.ses_client.send_email(
             FromEmailAddress="noreply@lpaconsulting.fr",

src/params.py

@@ -1,8 +1,6 @@
 import random
 import string
 
-from src import definitions
-from src.db.local_db_manager import LocalDbManager
 from src.logs.LogSender import LogSender
 from src.proxy.proxy_type import ProxyType
 
@@ -51,8 +49,6 @@ def get_random_id_number_for_proxy() -> str:
     return id_number
 
 
-local_db_manager = LocalDbManager(definitions.home)
-
 DEVICES = ['iPad (gen 6)', 'iPad (gen 6) landscape', 'iPad (gen 7)', 'iPad (gen 7) landscape', 'iPad Mini',
            'iPad Mini landscape', 'iPad Pro 11', 'iPad Pro 11 landscape', 'iPhone 6', 'iPhone 6 landscape',
            'iPhone 6 Plus', 'iPhone 6 Plus landscape', 'iPhone 7', 'iPhone 7 landscape', 'iPhone 7 Plus',

src/pojo/ReserveResultPojo.py

@@ -4,7 +4,7 @@ from typing import Union
 
 from dataclasses_json import dataclass_json
 
-from src import definitions
+from src import config
 
 
 class PublishType(Enum):
@@ -29,7 +29,7 @@ class ReserveResultPojo:
     slot_position = None
     sim_position = None
     ccid: str = ""
-    source_from: str = definitions.LOG_SOURCE
+    source_from: str = config.LOG_SOURCE
     store_type = 0
 
     @staticmethod

src/utils/black_list_checker.py

@@ -1,13 +1,14 @@
 import time
 
 from src import params, definitions
+from src.db.mongo_manager import MONGO_STORE_MANAGER
 from src.pojo.contact_pojo import ContactPojo
 
 SEVEN_DAYS_IN_S = 7 * 24 * 3600
 
 
 def can_send_request(contact: ContactPojo) -> bool:
-    black_list = definitions.mongo_store_manager.get_blacklist_contacts()
+    black_list = MONGO_STORE_MANAGER.get_blacklist_contacts()
     for black_contact in black_list:
         if contact.mail == black_contact.mail:
             # check date

src/utils/excel_reader.py

@@ -5,7 +5,7 @@ import string
 import pandas as pandas
 import xlsxwriter
 
-from src.definitions import CONTACT_LIST_FILE
+from src.config import CONTACT_LIST_FILE
 from src.pojo.contact_pojo import ContactPojo
 from src.utils.generate_random_passport_id import get_random_passport_id_number
 

src/workers/SolveCaptch.py

@@ -50,5 +50,6 @@ class SolveCaptcha:
             solution_res = requests.get(url_response)
             time.sleep(5)
             solution = solution_res.text
-        self.logger.info(solution)
+            self.logger.info("response code: " + str(res.status_code))
+        # self.logger.info(solution)
         handle_solution_received(solution.split("|")[-1])

src/workers/commandor_page.py

@@ -11,6 +11,7 @@ from typing import Union
 from playwright.sync_api import sync_playwright
 
 from src import params, definitions
+from src.db.mongo_manager import MONGO_STORE_MANAGER
 from src.pojo.ModeEnum import ModeEnum
 from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType
 from src.pojo.captcha_error_contact_pojo import TOO_MANY_REQUEST_ERROR, ERROR_TYPE_CAPTCHA
@@ -255,10 +256,8 @@ class CommandorPage:
         elif TOO_MANY_REQUEST_ERROR_MESSAGE in erro_content or TOO_MANY_REQUEST_ERROR_MESSAGE_FR in erro_content:
             # this email is in black list
             if not self.is_finished:
-                params.local_db_manager.insert_or_update(
-                    get_captcha_error_contact_from_contact(self.contact, TOO_MANY_REQUEST_ERROR))
                 params.oracle_log_sender.send_too_many_error(self.contact)
-                definitions.mongo_store_manager.insert_blacklist_contact(self.contact)
+                MONGO_STORE_MANAGER.insert_blacklist_contact(self.contact)
             self.is_finished = True
             self.termine()
         elif CAPTCHA_ERROR_MESSAGE in erro_content or CAPTCHA_ERROR_MESSAGE_FR in erro_content:
@@ -332,9 +331,9 @@ class CommandorPage:
         result.store_type = self.store_type
         definitions.firebase_store_manager.save(result)
         collection_name = str(datetime.date.today())
-        definitions.mongo_store_manager.insert_reserve_result(collection_name=collection_name, reserve=result)
+        MONGO_STORE_MANAGER.insert_reserve_result(collection_name=collection_name, reserve=result)
-        definitions.mongo_store_manager.delete_captcha_error_contact_for_current_day(self.contact)
+        MONGO_STORE_MANAGER.delete_captcha_error_contact_for_current_day(self.contact)
-        definitions.mongo_store_manager.remove_contact_from_black_list(self.contact)
+        MONGO_STORE_MANAGER.remove_contact_from_black_list(self.contact)
         if status is PublishType.SUCCESS:
             self.on_success(result)
             time.sleep(2)
@@ -364,9 +363,7 @@ class CommandorPage:
             self.page.reload(timeout=PAGE_TIMEOUT)
 
     def handle_captcha_error(self):
-        params.local_db_manager.insert_or_update(
+        MONGO_STORE_MANAGER.insert_captcha_error_contact(self.contact)
-            get_captcha_error_contact_from_contact(self.contact, ERROR_TYPE_CAPTCHA))
-        definitions.mongo_store_manager.insert_captcha_error_contact(self.contact)
         params.oracle_log_sender.send_captcha_error(self.contact)