panleicim/appointment_tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove local db

Browse Source
This commit is contained in:
Lei PAN
2022-07-08 09:04:45 +02:00
parent 4662bed9b8
commit fe56c47d50
18 changed files with 57 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/check_results.py
+9 -7
View File
@@ -6,14 +6,15 @@ from typing import Union
from playwright.sync_api import sync_playwright
import params
from logs.LogSender import TYPE_EVENT_CHECK_RESULTS, LOG_SUBJECT_EVENT
from src.db.mongo_manager import MONGO_STORE_MANAGER
from src.definitions import LOG_SUBJECT_EVENT, TYPE_EVENT_CHECK_RESULTS
from src.logs.LogSender import LogSender
from notification.AcceptedResultPojo import get_accepted_result_from
from notification.mailer import Mailer
from pojo.ReserveResultPojo import ReserveResultPojo
from pojo.ResultEnum import ResultEnum
from proxy.proxy_type import ProxyType
from src import definitions
from src import definitions, params
SORRY_SENTENCE_FR = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci"
SORRY_SENTENCE_EN = "we are extremely sorry that we were not able to fulfill"
@@ -25,6 +26,7 @@ BLANK_URL = "about:blank"
WELCOME_URL = "https://rendezvousparis.hermes.com/client/welcome"
mailer = Mailer()
oracle_log_sender = LogSender()
class TlsPlaywright(threading.local):
@@ -100,18 +102,18 @@ class ResultChecker:
print(err)
reserve_pojo.accepted = status
firestore_collection.document(reserve_pojo.id).update({u'accepted': status.name})
definitions.mongo_store_manager.update_reserve_result(reserve_pojo.id, status)
MONGO_STORE_MANAGER.update_reserve_result(reserve_pojo.id, status)
def check_results():
# get the list
params.oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS)
oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS)
db_manager = definitions.firebase_store_manager
firestore_collection = db_manager.get_all_successful_items()
reserve_list = definitions.mongo_store_manager.get_all_successful_items_for_day()
reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
print("size is " + str(len(reserve_list)))
start_check(reserve_list, firestore_collection, False)
reserve_list = definitions.mongo_store_manager.get_all_successful_items_for_day()
reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
start_check(reserve_list, firestore_collection, True)
src/config.py
+16
View File
@@ -0,0 +1,16 @@
import configparser
import getpass
import os
from pathlib import Path
home = str(Path.home())
config = configparser.ConfigParser()
# check the config file existence
config_file_path = home + "/config.ini"
config.read(config_file_path)
CONTACT_LIST_FILE = config['DEFAULT']['contact_list_file']
FIREBASE_CONFIG_FILE = config['DEFAULT']['firebase_config_file']
LOGS_DIR = config['DEFAULT']['LOGS_DIR']
username = getpass.getuser()
LOG_SOURCE = username
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
src/db/DbManager.py
+2 -2
View File
@@ -4,7 +4,7 @@ from typing import Union
import firebase_admin
from firebase_admin import credentials, firestore
from src import definitions, params
from src import params, config
from src.pojo.MailPojo import MailPojo
from src.pojo.ReserveResultPojo import ReserveResultPojo
from src.pojo.ResultEnum import ResultEnum
@@ -22,7 +22,7 @@ class DataManager:
batch_size = 20
def __init__(self):
cred = credentials.Certificate(definitions.FIREBASE_CONFIG_FILE)
cred = credentials.Certificate(config.FIREBASE_CONFIG_FILE)
self._app = firebase_admin.initialize_app(cred)
self._db = firestore.client()
src/db/local_db_manager.py
-54
View File
@@ -1,54 +0,0 @@
from builtins import list
import sqlalchemy
from sqlalchemy import MetaData, Column, String, Integer, DateTime, Table
from sqlalchemy.orm import Session
from sqlalchemy_utils import database_exists, create_database
from src.pojo.ReserveResultPojo import ReserveResultPojo
from src.pojo.captcha_error_contact_pojo import ContactInErrorPojo
class LocalDbManager:
def __init__(self, path: str):
self.session = Session(self.init_db(path))
def init_db(self, path: str):
uri_for_db = "sqlite:///{}/{}.db?check_same_thread=false".format(path, "appointment")
print(uri_for_db)
# 2.-Turn on database engine
db_engine = sqlalchemy.create_engine(uri_for_db) # ensure this is the correct path for the sqlite file.
if not database_exists(uri_for_db):
create_database(uri_for_db)
connextion = db_engine.connect()
if not db_engine.dialect.has_table(connextion,
ContactInErrorPojo.__tablename__): # If table don't exist, Create.
metadata = MetaData(db_engine)
# Create a table with the appropriate Columns
Table(ContactInErrorPojo.__tablename__, metadata,
Column('id', Integer, primary_key=True, autoincrement=True),
Column('mail', String),
Column('phone', String),
Column('passport', String),
Column('last_name', String),
Column('first_name', String),
Column('ccid', String),
Column('position', Integer),
Column('error_type', Integer),
Column('update_at', DateTime),
Column('create_at', DateTime))
# Implement the creation
metadata.create_all()
return db_engine
def insert_or_update(self, instance: ContactInErrorPojo):
self.session.merge(instance)
self.session.commit()
def handle_success(self, reservePojo: ReserveResultPojo):
# delete the contact from table
self.session.query(ContactInErrorPojo).filter(ContactInErrorPojo.mail == reservePojo.email).delete()
src/db/mongo_manager.py
+3 -1
View File
@@ -117,8 +117,10 @@ class MongoDbManager:
self.logger.info(error)
MONGO_STORE_MANAGER = MongoDbManager()
if __name__ == '__main__':
db_manager = MongoDbManager()
black_list = db_manager.get_blacklist_contacts()
for contact in black_list:
print(contact)
print(contact)
src/definitions.py
+2 -19
View File
@@ -1,23 +1,6 @@
import configparser
import getpass
import os
from pathlib import Path
home = str(Path.home())
config = configparser.ConfigParser()
# check the config file existence
config_file_path = home + "/config.ini"
config.read(config_file_path)
CONTACT_LIST_FILE = config['DEFAULT']['contact_list_file']
FIREBASE_CONFIG_FILE = config['DEFAULT']['firebase_config_file']
LOGS_DIR = config['DEFAULT']['LOGS_DIR']
username = getpass.getuser()
LOG_SOURCE = username
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
from src.db.mongo_manager import MongoDbManager
mongo_store_manager = MongoDbManager()
LOG_SUBJECT_EVENT = "EVENT"
TYPE_EVENT_CHECK_RESULTS = "EVENT_CHECK_RESULTS"
from src.db.DbManager import DataManager
src/logs/AppLogging.py
+1 -1
View File
@@ -1,7 +1,7 @@
import datetime
import logging
from src.definitions import LOGS_DIR
from src.config import LOGS_DIR
def init_logger():
src/logs/LogSender.py
+2 -2
View File
@@ -6,7 +6,7 @@ import oci
from oci.loggingingestion import LoggingClient
from oci.loggingingestion.models import PutLogsDetails, LogEntryBatch, LogEntry
from src import definitions
from src import config
from src.logs.AppLogging import init_logger
from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType
from src.pojo.contact_pojo import ContactPojo
@@ -96,7 +96,7 @@ class LogSender:
def send_read_db_event(self, msg: str):
self.send_log(msg=msg, type=TYPE_EVENT_READ_DB, subject=LOG_SUBJECT_EVENT)
def send_log(self, msg: str, source=definitions.LOG_SOURCE, subject="appointment", type: str = "INFO"):
def send_log(self, msg: str, source=config.LOG_SOURCE, subject="appointment", type: str = "INFO"):
log_id = "ocid1.log.oc1.eu-frankfurt-1.amaaaaaas4ft22ya3ub6glkltqqbnmkxo3ui7xwq3dxtjd2scdhme4deyu2q"
response = self._loggingingestion_client.put_logs(
log_id=log_id,
src/notification/mailer.py
+5 -5
View File
@@ -1,10 +1,10 @@
import base64
import logging
from mako.template import Template
import boto3
from mako.template import Template
from src import definitions
from src import config
from src.notification import AcceptedResultPojo
TEMPLATE_NAME = 'scrapy_template_email'
@@ -26,10 +26,10 @@ class Mailer:
aws_secret_access_key=secret)
def send_email(self, result: AcceptedResultPojo):
# recipients = ['panleicim@gmail.com', 'kamenonly@gmail.com', 'tangliang0411@gmail.com']
recipients = ['panleicim@gmail.com']
recipients = ['panleicim@gmail.com', 'kamenonly@gmail.com', 'tangliang0411@gmail.com']
# recipients = ['panleicim@gmail.com']
mytemplate = Template(filename=definitions.ROOT_DIR + "/templates/appointment_results.html")
mytemplate = Template(filename=config.ROOT_DIR + "/templates/appointment_results.html")
self.logger.info("send email to " + str(recipients))
self.ses_client.send_email(
FromEmailAddress="noreply@lpaconsulting.fr",
src/params.py
-4
View File
@@ -1,8 +1,6 @@
import random
import string
from src import definitions
from src.db.local_db_manager import LocalDbManager
from src.logs.LogSender import LogSender
from src.proxy.proxy_type import ProxyType
@@ -51,8 +49,6 @@ def get_random_id_number_for_proxy() -> str:
return id_number
local_db_manager = LocalDbManager(definitions.home)
DEVICES = ['iPad (gen 6)', 'iPad (gen 6) landscape', 'iPad (gen 7)', 'iPad (gen 7) landscape', 'iPad Mini',
'iPad Mini landscape', 'iPad Pro 11', 'iPad Pro 11 landscape', 'iPhone 6', 'iPhone 6 landscape',
'iPhone 6 Plus', 'iPhone 6 Plus landscape', 'iPhone 7', 'iPhone 7 landscape', 'iPhone 7 Plus',
src/pojo/ReserveResultPojo.py
+2 -2
View File
@@ -4,7 +4,7 @@ from typing import Union
from dataclasses_json import dataclass_json
from src import definitions
from src import config
class PublishType(Enum):
@@ -29,7 +29,7 @@ class ReserveResultPojo:
slot_position = None
sim_position = None
ccid: str = ""
source_from: str = definitions.LOG_SOURCE
source_from: str = config.LOG_SOURCE
store_type = 0
@staticmethod
src/templates/appointment_results.html
+15
View File
@@ -0,0 +1,15 @@
姓名:${result.name}
<br/>
<br/>电话: ${result.phone}
<br/>
<br/>邮件: ${result.email},
<br/>
<br/>护照: ${result.passport}
<br/>
<br/>SIM卡位置: 卡池第${result.slot_position}行，第${result.sim_position}列
<br/>
<br/> 短信内容: ${result.sms_msg}
<br/>
<br/> SIM卡CCID: ${result.ccid}
<br/>
<br/> 约会url: ${result.url}
src/utils/black_list_checker.py
+2 -1
View File
@@ -1,13 +1,14 @@
import time
from src import params, definitions
from src.db.mongo_manager import MONGO_STORE_MANAGER
from src.pojo.contact_pojo import ContactPojo
SEVEN_DAYS_IN_S = 7 * 24 * 3600
def can_send_request(contact: ContactPojo) -> bool:
black_list = definitions.mongo_store_manager.get_blacklist_contacts()
black_list = MONGO_STORE_MANAGER.get_blacklist_contacts()
for black_contact in black_list:
if contact.mail == black_contact.mail:
# check date
src/utils/excel_reader.py
+1 -1
View File
@@ -5,7 +5,7 @@ import string
import pandas as pandas
import xlsxwriter
from src.definitions import CONTACT_LIST_FILE
from src.config import CONTACT_LIST_FILE
from src.pojo.contact_pojo import ContactPojo
from src.utils.generate_random_passport_id import get_random_passport_id_number
src/workers/SolveCaptch.py
+2 -1
View File
@@ -50,5 +50,6 @@ class SolveCaptcha:
solution_res = requests.get(url_response)
time.sleep(5)
solution = solution_res.text
self.logger.info(solution)
self.logger.info("response code: " + str(res.status_code))
# self.logger.info(solution)
handle_solution_received(solution.split("|")[-1])
src/workers/commandor_page.py
+6 -9
View File
@@ -11,6 +11,7 @@ from typing import Union
from playwright.sync_api import sync_playwright
from src import params, definitions
from src.db.mongo_manager import MONGO_STORE_MANAGER
from src.pojo.ModeEnum import ModeEnum
from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType
from src.pojo.captcha_error_contact_pojo import TOO_MANY_REQUEST_ERROR, ERROR_TYPE_CAPTCHA
@@ -255,10 +256,8 @@ class CommandorPage:
elif TOO_MANY_REQUEST_ERROR_MESSAGE in erro_content or TOO_MANY_REQUEST_ERROR_MESSAGE_FR in erro_content:
# this email is in black list
if not self.is_finished:
params.local_db_manager.insert_or_update(
get_captcha_error_contact_from_contact(self.contact, TOO_MANY_REQUEST_ERROR))
params.oracle_log_sender.send_too_many_error(self.contact)
definitions.mongo_store_manager.insert_blacklist_contact(self.contact)
MONGO_STORE_MANAGER.insert_blacklist_contact(self.contact)
self.is_finished = True
self.termine()
elif CAPTCHA_ERROR_MESSAGE in erro_content or CAPTCHA_ERROR_MESSAGE_FR in erro_content:
@@ -332,9 +331,9 @@ class CommandorPage:
result.store_type = self.store_type
definitions.firebase_store_manager.save(result)
collection_name = str(datetime.date.today())
definitions.mongo_store_manager.insert_reserve_result(collection_name=collection_name, reserve=result)
definitions.mongo_store_manager.delete_captcha_error_contact_for_current_day(self.contact)
definitions.mongo_store_manager.remove_contact_from_black_list(self.contact)
MONGO_STORE_MANAGER.insert_reserve_result(collection_name=collection_name, reserve=result)
MONGO_STORE_MANAGER.delete_captcha_error_contact_for_current_day(self.contact)
MONGO_STORE_MANAGER.remove_contact_from_black_list(self.contact)
if status is PublishType.SUCCESS:
self.on_success(result)
time.sleep(2)
@@ -364,9 +363,7 @@ class CommandorPage:
self.page.reload(timeout=PAGE_TIMEOUT)
def handle_captcha_error(self):
params.local_db_manager.insert_or_update(
get_captcha_error_contact_from_contact(self.contact, ERROR_TYPE_CAPTCHA))
definitions.mongo_store_manager.insert_captcha_error_contact(self.contact)
MONGO_STORE_MANAGER.insert_captcha_error_contact(self.contact)
params.oracle_log_sender.send_captcha_error(self.contact)