From 3160fefdab4a40ecb9f71c0b04dc33a1a4d1e06a Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 25 Jul 2022 10:53:29 +0200 Subject: [PATCH 1/4] added check accepted or not before sending request --- src/db/mongo_manager.py | 11 ++++++++ src/pojo/accepted_appointment_pojo.py | 40 +++++++++++++++++++++++++-- src/utils/black_list_checker.py | 20 +++++++++++++- 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/db/mongo_manager.py b/src/db/mongo_manager.py index 238d9a6..aa95578 100644 --- a/src/db/mongo_manager.py +++ b/src/db/mongo_manager.py @@ -41,6 +41,17 @@ class MongoDbManager: except Exception as Error: self.logger.info(Error) + def get_all_accepted_appointments(self) -> list: + collection_name = ACCEPTED_APPOINTMENT_LIST + appointment_list_contacts = [] + try: + collection_to_use = self.db[collection_name] + for document in collection_to_use.find(): + appointment_list_contacts.append(AcceptedAppointmentPojo.from_firestore_dict(document)) + except Exception as error: + self.logger.info(error) + return appointment_list_contacts + def insert_captcha_error_contact(self, contact: ContactPojo): day = str(datetime.date.today()) collection_name = CAPTCHA_ERROR_COLLECTION_PREFIX + day diff --git a/src/pojo/accepted_appointment_pojo.py b/src/pojo/accepted_appointment_pojo.py index b8f5d45..9b752b1 100644 --- a/src/pojo/accepted_appointment_pojo.py +++ b/src/pojo/accepted_appointment_pojo.py @@ -1,6 +1,6 @@ import time -from src.pojo.ReserveResultPojo import ReserveResultPojo +from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType class AcceptedAppointmentPojo(ReserveResultPojo): @@ -46,7 +46,43 @@ class AcceptedAppointmentPojo(ReserveResultPojo): @staticmethod def from_firestore_dict(source): - result = ReserveResultPojo.from_firestore_dict(source) + if 'type' in source: + publish_type = source['type'] + if publish_type: + publish_type = PublishType[publish_type] + else: + publish_type = PublishType.SUCCESS + phone = source['phone'] + url = source['url'] + id = source['id'] + email = source['email'] + lastName = source['lastName'] + firstName = source['firstName'] + result = ReserveResultPojo(type=publish_type, phone=phone, + message="", url=url, email=email, + firstName=firstName, lastName=lastName) + if 'accepted' in source: + accepted = source['accepted'] + result.accepted = accepted + if 'source' in source: + source_from = source['source'] + result.source_from = source_from + if 'sim_position' in source: + sim_position = source['sim_position'] + result.sim_position = sim_position + if 'slot_position' in source: + slot_position = source['slot_position'] + result.slot_position = slot_position + if 'passport' in source: + passport = source['passport'] + result.passport = passport + if 'ccid' in source: + ccid = source['ccid'] + result.ccid = ccid + if 'store_type' in source: + store_type = source['store_type'] + result.store_type = store_type + result.id = id accepted_pojo = AcceptedAppointmentPojo.from_reserve(result) if 'accepted_at' in source: accepted_at = source['accepted_at'] diff --git a/src/utils/black_list_checker.py b/src/utils/black_list_checker.py index 10e5ebf..7a637ae 100644 --- a/src/utils/black_list_checker.py +++ b/src/utils/black_list_checker.py @@ -4,6 +4,7 @@ from src.db.mongo_manager import MONGO_STORE_MANAGER from src.pojo.contact_pojo import ContactPojo SEVEN_DAYS_IN_S = 7 * 24 * 3600 +TWO_WEEKS_IN_S = 2 * SEVEN_DAYS_IN_S def is_already_sent(contact: ContactPojo) -> bool: @@ -24,5 +25,22 @@ def is_in_white_list(contact: ContactPojo) -> bool: return True +def is_in_accepted_list(contact: ContactPojo) -> bool: + accepted_appointment_list = MONGO_STORE_MANAGER.get_all_accepted_appointments() + for accepted_contact in accepted_appointment_list: + if contact.mail == accepted_contact.email: + # check date + return accepted_contact.accepted_at + TWO_WEEKS_IN_S > time.time() + + return False + + def can_send_request(contact: ContactPojo) -> bool: - return is_in_white_list(contact) and (not is_already_sent(contact=contact)) + return is_in_white_list(contact) and (not is_already_sent(contact=contact)) and ( + not is_in_accepted_list(contact=contact)) + + +if __name__ == '__main__': + contact = ContactPojo(phone_number='605764105', passport_number='23023672', last_name='Jia', first_name='Roujin', + mail='alanobierx@hotmail.com') + print(can_send_request(contact)) From f482164c16dc977de51f42f64ba4425c5173ff4b Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 25 Jul 2022 14:21:12 +0200 Subject: [PATCH 2/4] need to test singleton --- main.py | 10 ++++----- src/check_results.py | 7 +----- src/workers/TlsPlaywright.py | 40 +++++++++++++++++++++++++++++++++++ src/workers/commandor_page.py | 8 ++----- 4 files changed, 48 insertions(+), 17 deletions(-) create mode 100644 src/workers/TlsPlaywright.py diff --git a/main.py b/main.py index 03ca4a0..9d810ea 100644 --- a/main.py +++ b/main.py @@ -2,7 +2,7 @@ import logging import sys from concurrent.futures import ThreadPoolExecutor -from src import params, definitions +from src import params from src.db.mongo_manager import MONGO_STORE_MANAGER from src.logs.AppLogging import init_logger from src.pojo.ModeEnum import ModeEnum @@ -40,7 +40,7 @@ def start_book(start_number, end_number, store_choose_state=0, max_workers=20, p def recheck_the_captcha_error_contacts(on_no_contact_found, store_type=0, mode: ModeEnum = ModeEnum.MANUAL, - max_workers=20): + max_workers=20, headless=False): # get all the contacts in captcha error contact_list = MONGO_STORE_MANAGER.get_captcha_error_contacts_for_current_day() if len(contact_list) == 0: @@ -51,7 +51,7 @@ def recheck_the_captcha_error_contacts(on_no_contact_found, store_type=0, mode: proxy = get_proxy() # start the task in thread executor.submit( - CommandorPage(contact, store_type=store_type, mode=mode).start_page, + CommandorPage(contact, store_type=store_type, mode=mode, headless= headless).start_page, proxy) @@ -61,5 +61,5 @@ def get_proxy(proxy_type=ProxyType.BRIGHT_DATA): if __name__ == '__main__': # 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机，传入1的时候是总店 - start_book(901, 1200, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=True) - # recheck_the_captcha_error_contacts(store_type=1, mode=ModeEnum.AUTOMATIC, on_no_contact_found=lambda: None) + # start_book(1, 100, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=True) + recheck_the_captcha_error_contacts(store_type=1, mode=ModeEnum.AUTOMATIC, on_no_contact_found=lambda: None, headless=True) diff --git a/src/check_results.py b/src/check_results.py index e26c22f..f64c4a1 100644 --- a/src/check_results.py +++ b/src/check_results.py @@ -17,6 +17,7 @@ from src.pojo.ReserveResultPojo import ReserveResultPojo from src.pojo.ResultEnum import ResultEnum from src.proxy.proxy_type import ProxyType from src import definitions, params +from src.workers.TlsPlaywright import TlsPlaywright SORRY_SENTENCE_FR = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci" SORRY_SENTENCE_EN = "we are extremely sorry that we were not able to fulfill" @@ -31,12 +32,6 @@ mailer = Mailer() oracle_log_sender = LogSender() -class TlsPlaywright(threading.local): - def __init__(self) -> None: - self.playwright = sync_playwright().start() - print("创建浏览器实例，线程: ", threading.current_thread().name) - - class ResultChecker: tls = TlsPlaywright() diff --git a/src/workers/TlsPlaywright.py b/src/workers/TlsPlaywright.py new file mode 100644 index 0000000..798355f --- /dev/null +++ b/src/workers/TlsPlaywright.py @@ -0,0 +1,40 @@ +import threading + +from playwright.sync_api import sync_playwright + + +class OnlyOne: + class __OnlyOne: + def __init__(self, arg): + self.val = arg + + def __str__(self): + return repr(self) + self.val + + instance = None + + def __init__(self, arg): + if not OnlyOne.instance: + OnlyOne.instance = OnlyOne.__OnlyOne(arg) + else: + OnlyOne.instance.val = arg + + def __getattr__(self, name): + return getattr(self.instance, name) + + +class TlsPlaywright(): + # private nested inner class + class __TlsPlaywrightSingleton(threading.local): + def __init__(self) -> None: + self.playwright = sync_playwright().start() + print("创建浏览器实例，线程: ", threading.current_thread().name) + + playwright = None + + def __init__(self): + if not TlsPlaywright.playwright: + TlsPlaywright.playwright = TlsPlaywright.__TlsPlaywrightSingleton().playwright + + def __getattr__(self, name): + return getattr(self.playwright, name) diff --git a/src/workers/commandor_page.py b/src/workers/commandor_page.py index 4fa3552..ed37f19 100644 --- a/src/workers/commandor_page.py +++ b/src/workers/commandor_page.py @@ -17,6 +17,7 @@ from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType from src.pojo.contact_pojo import ContactPojo from src.proxy.proxy_type import ProxyType from src.workers.SolveCaptch import SolveCaptcha +from src.workers.TlsPlaywright import TlsPlaywright RDV_URL = "https://rendezvousparis.hermes.com/client/register" @@ -46,13 +47,8 @@ def get_random_wait_time() -> float: return wait_time -class Tls(threading.local): - def __init__(self) -> None: - self.playwright = sync_playwright().start() - - class CommandorPage: - tls = Tls() + tls = TlsPlaywright() def __init__(self, contact: ContactPojo, store_type=0, proxy_type=ProxyType.BRIGHT_DATA, mode: ModeEnum = ModeEnum.MANUAL, headless=False): From a7873dfe91ab0eb1ecb5b1b2c0d6ca40256d7fd6 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Tue, 26 Jul 2022 11:38:22 +0200 Subject: [PATCH 3/4] check accepted contact before sending request --- src/workers/TlsPlaywright.py | 37 +++-------------------------------- src/workers/commandor_page.py | 4 ++-- 2 files changed, 5 insertions(+), 36 deletions(-) diff --git a/src/workers/TlsPlaywright.py b/src/workers/TlsPlaywright.py index 798355f..c6e30d7 100644 --- a/src/workers/TlsPlaywright.py +++ b/src/workers/TlsPlaywright.py @@ -3,38 +3,7 @@ import threading from playwright.sync_api import sync_playwright -class OnlyOne: - class __OnlyOne: - def __init__(self, arg): - self.val = arg - - def __str__(self): - return repr(self) + self.val - - instance = None - - def __init__(self, arg): - if not OnlyOne.instance: - OnlyOne.instance = OnlyOne.__OnlyOne(arg) - else: - OnlyOne.instance.val = arg - - def __getattr__(self, name): - return getattr(self.instance, name) - - -class TlsPlaywright(): +class TlsPlaywright(threading.local): # private nested inner class - class __TlsPlaywrightSingleton(threading.local): - def __init__(self) -> None: - self.playwright = sync_playwright().start() - print("创建浏览器实例，线程: ", threading.current_thread().name) - - playwright = None - - def __init__(self): - if not TlsPlaywright.playwright: - TlsPlaywright.playwright = TlsPlaywright.__TlsPlaywrightSingleton().playwright - - def __getattr__(self, name): - return getattr(self.playwright, name) + def __init__(self) -> None: + self.playwright = sync_playwright().start() diff --git a/src/workers/commandor_page.py b/src/workers/commandor_page.py index ed37f19..cdc45f5 100644 --- a/src/workers/commandor_page.py +++ b/src/workers/commandor_page.py @@ -87,7 +87,7 @@ class CommandorPage: self.logger.info("will close timeout modem") self.termine() - def _run(self, e: threading.Event, proxy): + def _run(self, proxy): self.logger.info("will start browser") self.on_success_listener = on_success # reset otp_value to None @@ -153,7 +153,7 @@ class CommandorPage: def start_page(self, proxy): e = threading.Event() - self._run(e, proxy) + self._run(proxy) def _on_page_loaded(self): self.logger.info("页面加载完毕") From 42f2d0c128f82ed08c49513aa6aa3fc1af5a9e1a Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Wed, 27 Jul 2022 09:37:21 +0200 Subject: [PATCH 4/4] change the generated contact file name --- src/utils/excel_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/excel_reader.py b/src/utils/excel_reader.py index 440daa1..1fd38b8 100644 --- a/src/utils/excel_reader.py +++ b/src/utils/excel_reader.py @@ -113,7 +113,7 @@ def write_new_contacts_to_excel(valid_contacts: list): row = 0 col = 0 # Create a workbook and add a worksheet. - workbook = xlsxwriter.Workbook('new_profile_{}.xlsx'.format(len(valid_contacts))) + workbook = xlsxwriter.Workbook('real_contacts_{}.xlsx'.format(len(valid_contacts))) header_data = ['name', 'phone', 'passport', 'email'] worksheet = workbook.add_worksheet() header_format = workbook.add_format({'bold': True}) @@ -136,6 +136,6 @@ def write_new_contacts_to_excel(valid_contacts: list): if __name__ == '__main__': excel_reader = ExcelHelper() - contacts = excel_reader.read_names("C:/Users/landd/Desktop/rdv/08-06/500.xlsx") + contacts = excel_reader.read_names("/Users/lpan/Downloads/real_contacts.xlsx") print(contacts) write_new_contacts_to_excel(valid_contacts=contacts)