diff --git a/main.py b/main.py index 7c5fd96..811d55c 100644 --- a/main.py +++ b/main.py @@ -18,7 +18,7 @@ logger = logging.getLogger() logger.addHandler(logging.StreamHandler(stream=sys.stdout)) -def start_book(start_number, end_number, store_choose_state=0, max_workers=1, proxy_type=ProxyType.BRIGHT_DATA, +def start_book(start_number, end_number, store_choose_state=0, max_workers=20, proxy_type=ProxyType.BRIGHT_DATA, mode: ModeEnum = ModeEnum.MANUAL, headless=False): # read the contact, and contact the 2 objects together excel_reader = ExcelHelper() @@ -61,5 +61,5 @@ def get_proxy(proxy_type=ProxyType.BRIGHT_DATA): if __name__ == '__main__': # 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机,传入1的时候是总店 - start_book(1000, 1171, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=False) + start_book(1, 20, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=False) # recheck_the_captcha_error_contacts(store_type=1, mode=ModeEnum.AUTOMATIC, on_no_contact_found=lambda: None) diff --git a/src/check_results.py b/src/check_results.py index 2089198..e26c22f 100644 --- a/src/check_results.py +++ b/src/check_results.py @@ -1,3 +1,4 @@ +import datetime import logging import random import threading @@ -6,6 +7,7 @@ from typing import Union from playwright.sync_api import sync_playwright +from src.db.mirgration.migration_tools import migre_accepted_appointment from src.db.mongo_manager import MONGO_STORE_MANAGER from src.definitions import LOG_SUBJECT_EVENT, TYPE_EVENT_CHECK_RESULTS from src.logs.LogSender import LogSender @@ -114,6 +116,8 @@ def check_results(headless=False): start_check(reserve_list, firestore_collection, headless, need_send_email=False) reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day() start_check(reserve_list, firestore_collection, headless, need_send_email=True) + # copy the accepted info to the accepted collection + migre_accepted_appointment(str(datetime.date.today())) def start_check(reserve_list, firestore_collection, headless: bool, need_send_email: bool): diff --git a/src/db/DbManager.py b/src/db/DbManager.py index 774e3c1..d4ae20d 100644 --- a/src/db/DbManager.py +++ b/src/db/DbManager.py @@ -42,6 +42,12 @@ class DataManager: doc_ref.where(u'source_from', u'==', source_from) return doc_ref + def get_all_accepted_items_for_day(self, day): + params.oracle_log_sender.send_read_db_event("get_all_successful_items_for_day for {}".format(day)) + doc_ref = self._db.collection(day) + accepted_ref = doc_ref.where(u'accepted', u'==', "ACCEPTED") + return accepted_ref + def get_successful_item_for_day_by_status(self, day, status: ResultEnum): params.oracle_log_sender.send_read_db_event("get_successful_item_for_day_by_status for {}".format(day)) doc_ref = self._db.collection(day) diff --git a/src/db/mirgration/__init__.py b/src/db/mirgration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/db/mirgration/migration_tools.py b/src/db/mirgration/migration_tools.py new file mode 100644 index 0000000..ac5757b --- /dev/null +++ b/src/db/mirgration/migration_tools.py @@ -0,0 +1,35 @@ +import datetime + +from src import definitions +from src.db.mongo_manager import MONGO_STORE_MANAGER +from src.pojo.ReserveResultPojo import ReserveResultPojo +from src.pojo.ResultEnum import ResultEnum +from src.pojo.accepted_appointment_pojo import AcceptedAppointmentPojo + + +def migre_accepted_item_from_firebase_to_mongo(day: str): + # get successful item from firestore + collection = definitions.firebase_store_manager.get_all_accepted_items_for_day(day).stream() + for valid_appointment in collection: + reserve_pojo = ReserveResultPojo.from_firestore_dict(valid_appointment.to_dict()) + if reserve_pojo.accepted == "ACCEPTED": + MONGO_STORE_MANAGER.update_reserve_result_for_collection(reserve_pojo.id, ResultEnum.ACCEPTED, day) + + +def migre_accepted_appointment(day: str): + # get successful item from firestore + collection = MONGO_STORE_MANAGER.get_accepted_items_for_one_day(day) + for valid_appointment in collection: + accepted_pojo = AcceptedAppointmentPojo.from_reserve(valid_appointment) + accepted_pojo.day = day + accepted_pojo.accepted_at = datetime.datetime.strptime(day, '%Y-%m-%d').timestamp() + MONGO_STORE_MANAGER.insert_accepted_reserve(accepted_pojo) + + +if __name__ == '__main__': + # migre_accepted_item_from_firebase_to_mongo("2022-07-06") + days = ['2022-07-04', '2022-07-05', '2022-07-06', '2022-07-07', '2022-07-08', '2022-07-09', '2022-07-11', + '2022-07-12', '2022-07-13', '2022-07-15', '2022-07-16', '2022-07-18''2022-07-19', '2022-07-20', + '2022-07-21'] + for day in days: + migre_accepted_appointment(day) diff --git a/src/db/mongo_manager.py b/src/db/mongo_manager.py index 4cc49e1..238d9a6 100644 --- a/src/db/mongo_manager.py +++ b/src/db/mongo_manager.py @@ -6,12 +6,14 @@ from pymongo import MongoClient from src import params from src.pojo.ReserveResultPojo import ReserveResultPojo from src.pojo.ResultEnum import ResultEnum +from src.pojo.accepted_appointment_pojo import AcceptedAppointmentPojo from src.pojo.black_contact import BlackContactPojo from src.pojo.contact_pojo import ContactPojo MONGO_DB_URL = "mongo.lpaconsulting.fr" CAPTCHA_ERROR_COLLECTION_PREFIX = "CAPTCHA_ERROR_" BLACK_LIST = "BLACK_LIST" +ACCEPTED_APPOINTMENT_LIST = "ACCEPTED_APPOINTMENT_LIST" class MongoDbManager: @@ -32,6 +34,13 @@ class MongoDbManager: except Exception as Error: self.logger.info(Error) + def insert_accepted_reserve(self, accepted_pojo: AcceptedAppointmentPojo): + try: + collection_to_use = self.db[ACCEPTED_APPOINTMENT_LIST] + collection_to_use.insert_one(accepted_pojo.to_firestore_dict()) + except Exception as Error: + self.logger.info(Error) + def insert_captcha_error_contact(self, contact: ContactPojo): day = str(datetime.date.today()) collection_name = CAPTCHA_ERROR_COLLECTION_PREFIX + day @@ -83,11 +92,26 @@ class MongoDbManager: result_list.append(ReserveResultPojo.from_firestore_dict(document)) return result_list + def get_accepted_items_for_one_day(self, day_in_str: str) -> list: + params.oracle_log_sender.send_read_db_event("get_accepted_items_for_one_day") + collection_name = day_in_str + result_list = [] + cursor = self.db[collection_name] + for document in cursor.find(): + reserve = ReserveResultPojo.from_firestore_dict(document) + if reserve.accepted == ResultEnum.ACCEPTED.name: + result_list.append(reserve) + return result_list + def update_reserve_result(self, reserve_id: str, result: ResultEnum): collection_name = str(datetime.date.today()) collection = self.db[collection_name] collection.find_one_and_update({'_id': reserve_id}, {"$set": {"accepted": result.name}}, upsert=False) + def update_reserve_result_for_collection(self, reserve_id: str, result: ResultEnum, collection_name: str): + collection = self.db[collection_name] + collection.find_one_and_update({'_id': reserve_id}, {"$set": {"accepted": result.name}}, upsert=False) + def get_captcha_error_contacts_for_current_day(self) -> list: day = str(datetime.date.today()) collection_name = CAPTCHA_ERROR_COLLECTION_PREFIX + day diff --git a/src/pojo/ReserveResultPojo.py b/src/pojo/ReserveResultPojo.py index ca69069..317ffe6 100644 --- a/src/pojo/ReserveResultPojo.py +++ b/src/pojo/ReserveResultPojo.py @@ -35,6 +35,8 @@ class ReserveResultPojo: @staticmethod def from_firestore_dict(source): publish_type = source['type'] + if publish_type: + publish_type = PublishType[publish_type] phone = source['phone'] message = source['message'] url = source['url'] @@ -85,6 +87,7 @@ class ReserveResultPojo: u'ccid': self.ccid, u'source_from': self.source_from, u'store_type': self.store_type, + u'accepted': self.accepted, } return dest diff --git a/src/pojo/accepted_appointment_pojo.py b/src/pojo/accepted_appointment_pojo.py new file mode 100644 index 0000000..b8f5d45 --- /dev/null +++ b/src/pojo/accepted_appointment_pojo.py @@ -0,0 +1,54 @@ +import time + +from src.pojo.ReserveResultPojo import ReserveResultPojo + + +class AcceptedAppointmentPojo(ReserveResultPojo): + accepted_at: float + day: str + + def __init__(self, reserve: ReserveResultPojo): + self.id = reserve.id + self.accepted = reserve.accepted + self.passport = reserve.passport + self.url = reserve.url + self.store_type = reserve.store_type + self.lastName = reserve.lastName + self.firstName = reserve.firstName + self.ccid = reserve.ccid + self.email = reserve.email + self.phone = reserve.phone + self.source_from = reserve.source_from + + @staticmethod + def from_reserve(reserve: ReserveResultPojo): + to_return = AcceptedAppointmentPojo(reserve) + to_return.accepted_at = time.time() + return to_return + + def to_firestore_dict(self): + dest = { + u'id': self.id, + u'phone': self.phone, + u'firstName': self.firstName, + u'lastName': self.lastName, + u'email': self.email, + u'passport': self.passport, + u'url': self.url, + u'ccid': self.ccid, + u'source_from': self.source_from, + u'store_type': self.store_type, + u'accepted': self.accepted, + u'day': self.day, + u'accepted_at': self.accepted_at, + } + return dest + + @staticmethod + def from_firestore_dict(source): + result = ReserveResultPojo.from_firestore_dict(source) + accepted_pojo = AcceptedAppointmentPojo.from_reserve(result) + if 'accepted_at' in source: + accepted_at = source['accepted_at'] + accepted_pojo.accepted_at = accepted_at + return accepted_pojo