create src module for source code

src/db/DbManager.py

@@ -0,0 +1,110 @@
+import datetime
+from typing import Union
+
+import firebase_admin
+from firebase_admin import credentials, firestore
+
+from src import definitions, params
+from src.pojo.MailPojo import MailPojo
+from src.pojo.ReserveResultPojo import ReserveResultPojo
+from src.pojo.ResultEnum import ResultEnum
+from src.pojo.SimInfoPojo import SimInfoPojo
+from src.pojo.contact_pojo import ContactPojo
+
+ERROR_COLLECTION_NAME = "error_items"
+CONTACT_COLLECTION_NAME = "contact_list"
+MAIL_COLLECTION_NAME = "mail_list"
+SIM_INFOS = "sim_infos"
+TIMEOUT = "timeout_items"
+
+
+class DataManager:
+    batch_size = 20
+
+    def __init__(self):
+        cred = credentials.Certificate(definitions.FIREBASE_CONFIG_FILE)
+        self._app = firebase_admin.initialize_app(cred)
+        self._db = firestore.client()
+
+    def get_all_sim_infos(self):
+        params.oracle_log_sender.send_read_db_event("get_all_sim_infos")
+        sim_info_collection = self._db.collection(SIM_INFOS)
+        return sim_info_collection
+
+    def get_all_successful_items(self):
+        params.oracle_log_sender.send_read_db_event("get_all_successful_items")
+        return self.get_all_successful_items_for_day(str(datetime.date.today()), None)
+
+    def get_all_successful_items_for_day(self, day, source_from: Union[str, None]):
+        params.oracle_log_sender.send_read_db_event("get_all_successful_items_for_day for {}".format(day))
+        doc_ref = self._db.collection(day)
+        if source_from is not None:
+            doc_ref.where(u'source_from', u'==', source_from)
+        return doc_ref
+
+    def get_successful_item_for_day_by_status(self, day, status: ResultEnum):
+        params.oracle_log_sender.send_read_db_event("get_successful_item_for_day_by_status for {}".format(day))
+        doc_ref = self._db.collection(day)
+        if status is not None:
+            doc_ref.where(u'accepted', u'==', status.value)
+        return doc_ref
+
+    def save_sim_info(self, sim_info: SimInfoPojo):
+        params.oracle_log_sender.send_read_db_event("save_sim_info")
+        doc_ref = self._db.collection(SIM_INFOS).document(sim_info.phone)
+        doc_ref.set(sim_info.to_firestore_dict())
+
+    def save(self, result: ReserveResultPojo):
+        id = result.url.split("/")[-1]
+        result.id = id
+        collection_name = str(datetime.date.today())
+        doc_ref = self._db.collection(collection_name).document(result.id)
+        doc_ref.set(result.to_firestore_dict())
+        params.mongo_store_manager.insert_reserve_result(collection_name=collection_name, reserve=result)
+
+    def find_appointment_detail_via_phone(self, day, phone) -> ReserveResultPojo:
+        params.oracle_log_sender.send_read_db_event("find_appointment_detail_via_phone")
+        doc_ref = self._db.collection(day)
+        results = doc_ref.where(u'phone', u'==', phone).stream()
+        result_list = []
+        for result in results:
+            result_list.append(ReserveResultPojo.from_firestore_dict(result.to_dict()))
+        if len(result_list) > 0:
+            return result_list[0]
+
+    def save_timeout_contact(self, contact: ContactPojo):
+        doc_ref = self._db.collection(TIMEOUT).document(str(contact.phone))
+        doc_ref.set(contact.to_firestore_dict())
+
+    def _delete_collection(self, coll_ref, batch_size):
+        params.oracle_log_sender.send_read_db_event("_delete_collection")
+        docs = coll_ref.limit(batch_size).stream()
+        deleted = 0
+
+        for doc in docs:
+            print(f'Deleting doc {doc.id} => {doc.to_dict()}')
+            doc.reference.delete()
+            deleted = deleted + 1
+
+        if deleted >= batch_size:
+            return self._delete_collection(coll_ref, batch_size)
+
+    # 删除数据库中所有的sim卡信息
+    def clear_all_sim_info(self):
+        params.oracle_log_sender.send_read_db_event("clear_all_sim_info")
+        coll_ref = self._db.collection(SIM_INFOS)
+        self._delete_collection(coll_ref, self.batch_size)
+
+    def read_contacts_from_db(self) -> list:
+        params.oracle_log_sender.send_read_db_event("read_contacts_from_db")
+        contact_collection = self._db.collection(CONTACT_COLLECTION_NAME)
+        return contact_collection
+
+    def get_mail_list(self) -> list:
+        params.oracle_log_sender.send_read_db_event("get_mail_list")
+        mail_collection = self._db.collection(MAIL_COLLECTION_NAME)
+        mail_list = []
+        for mail in mail_collection.stream():
+            mail_pojo = MailPojo.from_firestore_dict(mail.to_dict())
+            mail_list.append(mail_pojo)
+        return mail_list

src/db/local_db_manager.py

@@ -0,0 +1,54 @@
+from builtins import list
+
+import sqlalchemy
+from sqlalchemy import MetaData, Column, String, Integer, DateTime, Table
+from sqlalchemy.orm import Session
+from sqlalchemy_utils import database_exists, create_database
+
+from src.pojo.ReserveResultPojo import ReserveResultPojo
+from src.pojo.captcha_error_contact_pojo import ContactInErrorPojo
+
+
+class LocalDbManager:
+
+    def __init__(self, path: str):
+        self.session = Session(self.init_db(path))
+
+    def init_db(self, path: str):
+        uri_for_db = "sqlite:///{}/{}.db?check_same_thread=false".format(path, "appointment")
+        print(uri_for_db)
+        # 2.-Turn on database engine
+        db_engine = sqlalchemy.create_engine(uri_for_db)  # ensure this is the correct path for the sqlite file.
+        if not database_exists(uri_for_db):
+            create_database(uri_for_db)
+        connextion = db_engine.connect()
+        if not db_engine.dialect.has_table(connextion,
+                                           ContactInErrorPojo.__tablename__):  # If table don't exist, Create.
+            metadata = MetaData(db_engine)
+            # Create a table with the appropriate Columns
+            Table(ContactInErrorPojo.__tablename__, metadata,
+                  Column('id', Integer, primary_key=True, autoincrement=True),
+                  Column('mail', String),
+                  Column('phone', String),
+                  Column('passport', String),
+                  Column('last_name', String),
+                  Column('first_name', String),
+                  Column('ccid', String),
+                  Column('position', Integer),
+                  Column('error_type', Integer),
+                  Column('update_at', DateTime),
+                  Column('create_at', DateTime))
+            # Implement the creation
+            metadata.create_all()
+
+        return db_engine
+
+    def insert_or_update(self, instance: ContactInErrorPojo):
+        self.session.merge(instance)
+        self.session.commit()
+
+    def handle_success(self, reservePojo: ReserveResultPojo):
+        #         delete the contact from table
+        self.session.query(ContactInErrorPojo).filter(ContactInErrorPojo.mail == reservePojo.email).delete()
+
+

src/db/mongo_manager.py

@@ -0,0 +1,63 @@
+import datetime
+import logging
+
+from pymongo import MongoClient
+
+from src.pojo.ReserveResultPojo import ReserveResultPojo
+from src.pojo.contact_pojo import ContactPojo
+
+MONGO_DB_URL = "mongo.lpaconsulting.fr"
+CAPTCHA_ERROR_COLLECTION_PREFIX = "CAPTCHA_ERROR_"
+
+
+class MongoDbManager:
+    def __init__(self):
+        client = MongoClient(MONGO_DB_URL, username='appointment', password='Rdv@2022', authSource='appointment')
+        self.db = client.appointment
+        self.logger = logging.getLogger("mongoDb")
+
+    def insert_one(self, collection_name: str, dict: dict):
+        collection_to_use = self.db[collection_name]
+        collection_to_use.insert_one(dict)
+
+    def insert_reserve_result(self, collection_name, reserve: ReserveResultPojo):
+        try:
+            collection_to_use = self.db[collection_name]
+            collection_to_use.replace_one(filter={'_id': reserve.id, }, replacement=reserve.to_firestore_dict(),
+                                          upsert=True)
+        except Exception as Error:
+            self.logger.info(Error)
+
+    def insert_captcha_error_contact(self, contact: ContactPojo):
+        day = str(datetime.date.today())
+        collection_name = CAPTCHA_ERROR_COLLECTION_PREFIX + day
+        try:
+            collection_to_use = self.db[collection_name]
+            collection_to_use.replace_one(filter={'_id': contact.mail, }, replacement=contact.to_firestore_dict(),
+                                          upsert=True)
+        except Exception as error:
+            self.logger.info(error)
+
+    def get_captcha_error_contacts_for_current_day(self) -> list:
+        day = str(datetime.date.today())
+        collection_name = CAPTCHA_ERROR_COLLECTION_PREFIX + day
+        cursor = self.db[collection_name]
+        contact_list = []
+        for document in cursor.find():
+            contact_list.append(ContactPojo.from_firestore_dict(document))
+        return contact_list
+
+    def delete_captcha_error_contact_for_current_day(self, contact: ContactPojo):
+        day = str(datetime.date.today())
+        collection_name = CAPTCHA_ERROR_COLLECTION_PREFIX + day
+        collection = self.db[collection_name]
+        to_delete = {'_id': contact.mail}
+        try:
+            collection.delete_one(to_delete)
+        except Exception as error:
+            self.logger.info(error)
+
+
+if __name__ == '__main__':
+    db_manager = MongoDbManager()
+    print(db_manager.get_captcha_error_contacts_for_current_day())

src/definitions.py

@@ -0,0 +1,17 @@
+import configparser
+import getpass
+import os
+from pathlib import Path
+home = str(Path.home())
+config = configparser.ConfigParser()
+print("home path: " + home)
+
+# check the config file exsistence
+config_file_path = home + "/config.ini"
+config.read(config_file_path)
+CONTACT_LIST_FILE = config['DEFAULT']['contact_list_file']
+FIREBASE_CONFIG_FILE = config['DEFAULT']['firebase_config_file']
+LOGS_DIR = config['DEFAULT']['LOGS_DIR']
+username = getpass.getuser()
+LOG_SOURCE = username
+ROOT_DIR = os.path.dirname(os.path.abspath(__file__))

src/logs/AppLogging.py

@@ -0,0 +1,12 @@
+import datetime
+import logging
+
+from src.definitions import LOGS_DIR
+
+
+def init_logger():
+    logging.basicConfig(filename=LOGS_DIR + "/appointment_{}.log".format(str(datetime.date.today())),
+                        filemode='a',
+                        format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
+                        datefmt='%Y-%m-%d %H:%M:%S',
+                        level=logging.INFO)

src/logs/LogSender.py

@@ -0,0 +1,139 @@
+import datetime
+import logging
+import sys
+
+import oci
+from oci.loggingingestion import LoggingClient
+from oci.loggingingestion.models import PutLogsDetails, LogEntryBatch, LogEntry
+
+from src import definitions
+from src.logs.AppLogging import init_logger
+from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType
+from src.pojo.contact_pojo import ContactPojo
+
+LOG_SUBJECT_EVENT = "EVENT"
+LOG_SUBJECT_SMS = "SMS"
+SUBJECT_SIM_INFO = "sim_card"
+# Log type
+TYPE_EVENT_CHECK_RESULTS = "EVENT_CHECK_RESULTS"
+TYPE_EVENT_READ_DB = "EVENT_READ_DB"
+TYPE_EVENT_RESET_ALL_SIM_CARDS = "EVENT_RESET_ALL_SIM_CARDS"
+TYPE_EVENT_CHANGE_SLOT = "EVENT_CHANGE_SLOT"
+TYPE_SMS_RECEIVED = "TYPE_SMS_RECEIVED"
+LOG_ERROR = "ERROR"
+LOG_TYPE_INFO = "INFO"
+LOG_ERROR_TYPE_DOUBLE_DATA = "DOUBLE_REQUEST_FOR_SAME_DATA"
+LOG_ERROR_TOO_MANY_REQUEST_TODAY = "TOO_MANY_REQUEST_TODAY"
+LOG_ERROR_CAPTCHA_ERROR_MESSAGE = "CAPTCHA_ERROR"
+LOG_SUBJECT_ERROR = "ERROR"
+LOG_APPOINTMENT_ERROR = "APPOINTMENT_ERROR"
+LOG_APPOINTMENT_TIMEOUT = "TIMEOUT"
+LOG_APPOINTMENT_CONTACT_NOT_FOUND = "CONTACT_NOT_FOUND"
+LOG_APPOINTMENT_SUCCESS = "SUCCESS"
+
+custom_retry_strategy = oci.retry.RetryStrategyBuilder(
+    # Make up to 10 service calls
+    max_attempts_check=True,
+    max_attempts=10,
+
+    # Don't exceed a total of 600 seconds for all service calls
+    total_elapsed_time_check=True,
+    total_elapsed_time_seconds=600,
+
+    # Wait 45 seconds between attempts
+    retry_max_wait_between_calls_seconds=45,
+
+    # Use 2 seconds as the base number for doing sleep time calculations
+    retry_base_sleep_time_seconds=2,
+
+    # Retry on certain service errors:
+    #
+    #   - 5xx code received for the request
+    #   - Any 429 (this is signified by the empty array in the retry config)
+    #   - 400s where the code is QuotaExceeded or LimitExceeded
+    service_error_check=True,
+    service_error_retry_on_any_5xx=True,
+    service_error_retry_config={
+        400: ['QuotaExceeded', 'LimitExceeded'],
+        429: []
+    },
+
+    # Use exponential backoff and retry with full jitter, but on throttles use
+    # exponential backoff and retry with equal jitter
+    backoff_type=oci.retry.BACKOFF_FULL_JITTER_EQUAL_ON_THROTTLE_VALUE
+).get_retry_strategy()
+
+
+class LogSender:
+    def __init__(self):
+        self._config = oci.config.from_file("~/.oci/logger_config_appointment")
+        self._identity = oci.identity.IdentityClient(self._config)
+        self._loggingingestion_client = LoggingClient(self._config, timeout=60.0, retry_strategy=custom_retry_strategy)
+
+    def send_double_data_error(self, contact: ContactPojo):
+        error_msg = contact.mail
+        self.send_log(msg=error_msg, type=LOG_ERROR_TYPE_DOUBLE_DATA, subject=LOG_SUBJECT_ERROR)
+
+    def send_too_many_error(self, contact: ContactPojo):
+        error_msg = contact.mail
+        self.send_log(msg=error_msg, type=LOG_ERROR_TOO_MANY_REQUEST_TODAY, subject=LOG_SUBJECT_ERROR)
+
+    def send_captcha_error(self, contact: ContactPojo):
+        error_msg = contact.mail
+        self.send_log(msg=error_msg, type=LOG_ERROR_CAPTCHA_ERROR_MESSAGE, subject=LOG_SUBJECT_ERROR)
+
+    def send_appoint_result(self, result: ReserveResultPojo):
+        if result.type == PublishType.SUCCESS:
+            # get id
+            self.send_log(result.id, type=LOG_APPOINTMENT_SUCCESS)
+        else:
+            msg = "{}, email: {}".format(result.message, result.email)
+            self.send_log(msg, type=LOG_APPOINTMENT_ERROR)
+
+    def send_error(self, msg: str):
+        self.send_log(msg=msg, type=LOG_ERROR)
+
+    def send_read_db_event(self, msg: str):
+        self.send_log(msg=msg, type=TYPE_EVENT_READ_DB, subject=LOG_SUBJECT_EVENT)
+
+    def send_log(self, msg: str, source=definitions.LOG_SOURCE, subject="appointment", type: str = "INFO"):
+        log_id = "ocid1.log.oc1.eu-frankfurt-1.amaaaaaas4ft22ya3ub6glkltqqbnmkxo3ui7xwq3dxtjd2scdhme4deyu2q"
+        response = self._loggingingestion_client.put_logs(
+            log_id=log_id,
+            put_logs_details=PutLogsDetails(
+                specversion="1.0",
+                log_entry_batches=[
+                    LogEntryBatch(
+                        entries=[LogEntry(
+                            data=msg,
+                            id=log_id,
+                            time=datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ"))],
+                        source=source,
+                        type=type,
+                        defaultlogentrytime=datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
+                        subject=subject)])
+        )
+        return response
+
+    def send_sms_reception_log(self, phone, sms_text, ccid):
+        msg = "from:{}, sms:{}, ccid:{}".format(phone, sms_text, ccid)
+        self.send_log(msg=msg, subject=LOG_SUBJECT_SMS, type=TYPE_SMS_RECEIVED)
+
+    def send_timeout_log(self, contact: ContactPojo):
+        msg = "phone:{}, mail:{}".format(contact.phone, contact.mail)
+        self.send_log(msg, type=LOG_APPOINTMENT_TIMEOUT)
+
+    def send_contact_not_found(self, msg: str):
+        self.send_log(msg, subject=SUBJECT_SIM_INFO, type=LOG_APPOINTMENT_CONTACT_NOT_FOUND)
+
+    def send_wait_sms_log(self):
+        self.send_log("等待短信", subject=LOG_SUBJECT_EVENT, type=TYPE_SMS_RECEIVED)
+
+
+if __name__ == '__main__':
+    init_logger()
+    logger = logging.getLogger()
+    logger.addHandler(logging.StreamHandler(stream=sys.stdout))
+    log_sender = LogSender()
+    response = log_sender.send_log("test")
+    print(response)

src/notification/AcceptedResultPojo.py

@@ -0,0 +1,35 @@
+from dataclasses import dataclass
+
+from dataclasses_json import dataclass_json
+
+from pojo import ReserveResultPojo
+
+
+@dataclass_json
+@dataclass
+class AcceptedResultPojo:
+
+    def __init__(self, msg: str, slot_position, sim_position, passport, email, name, phone, ccid: str, url):
+        self.sms_msg = msg
+        self.slot_position = slot_position
+        self.sim_position = sim_position
+        self.passport = "{}".format(int(passport))
+        self.email = email
+        self.name = name
+        self.phone = phone
+        self.ccid = ccid
+        self.url = url
+
+
+def get_accepted_result_from(sim_info: ReserveResultPojo) -> AcceptedResultPojo:
+    if sim_info is None:
+        #     send email even there are no reserve info
+        return AcceptedResultPojo("", slot_position=0, sim_position=0,
+                                  passport="", email="", phone="",
+                                  name="", ccid="",
+                                  url="")
+    else:
+        return AcceptedResultPojo("", slot_position=sim_info.slot_position, sim_position=sim_info.sim_position,
+                                  passport=sim_info.passport, email=sim_info.email, phone=sim_info.phone,
+                                  name="{} {}".format(sim_info.lastName, sim_info.firstName), ccid=sim_info.ccid,
+                                  url=sim_info.url)

src/notification/mailer.py

@@ -0,0 +1,53 @@
+import base64
+import logging
+from mako.template import Template
+
+import boto3
+
+import definitions
+from src.notification import AcceptedResultPojo
+
+TEMPLATE_NAME = 'scrapy_template_email'
+AWS_CLIENT_NAME = "sesv2"
+AWS_REGION_NAME = "eu-central-1"
+UTF8 = 'utf-8'
+
+
+class Mailer:
+
+    def __init__(self):
+        self.logger = logging.getLogger("Mailer")
+        encoded_bytes = "4JWcU9B/BzTRXIPlBDt4EobibUU7ji1Sgx+VTsEf".encode(UTF8)
+        encoded_base64_bytes = base64.b64encode(encoded_bytes)
+        decoded_bytes = base64.b64decode(encoded_base64_bytes)
+        secret = decoded_bytes.decode(UTF8)
+        self.ses_client = boto3.client(AWS_CLIENT_NAME, region_name=AWS_REGION_NAME,
+                                       aws_access_key_id="AKIAU67WLKZUXEHXDJB5",
+                                       aws_secret_access_key=secret)
+
+    def send_email(self, result: AcceptedResultPojo):
+        recipients = ['panleicim@gmail.com', 'kamenonly@gmail.com', 'tangliang0411@gmail.com']
+        # recipients = ['panleicim@gmail.com']
+
+        mytemplate = Template(filename=definitions.ROOT_DIR + "/templates/appointment_results.html")
+        self.logger.info("send email to " + str(recipients))
+        self.ses_client.send_email(
+            FromEmailAddress="noreply@lpaconsulting.fr",
+            Destination={
+                "ToAddresses": recipients
+            },
+            Content={
+                'Simple': {
+                    'Subject': {
+                        'Data': '约会成功',
+                        'Charset': 'utf8'
+                    },
+                    'Body': {
+                        'Html': {
+                            'Data': mytemplate.render(result=result),
+                            'Charset': 'utf8'
+                        }
+                    }
+                },
+            }
+        )

src/params.py

@@ -0,0 +1,69 @@
+import random
+import string
+
+from src import definitions
+from src.db.DbManager import DataManager
+from src.db.local_db_manager import LocalDbManager
+from src.db.mongo_manager import MongoDbManager
+from src.logs.LogSender import LogSender
+from src.proxy.proxy_type import ProxyType
+
+firebase_store_manager = DataManager()
+oracle_log_sender = LogSender()
+
+# proxy
+PROXY_SERVER = "http://gw.ntnt.io:5959"
+PROXY_PASSWORD = "94sY7zwBG13i"
+
+BRIGHT_DATA_PROXY_SERVER = "http://zproxy.lum-superproxy.io:22225"
+BRIGHT_DATA_PROXY_USERNAME = "lum-customer-c_daabba94-zone-residential-country-fr"
+BRIGHT_DATA_MOBILE_PROXY_USERNAME = "lum-customer-c_daabba94-zone-mobile-country-fr-city-paris-mobile"
+BRIGHT_DATA_PROXY_PASSWORD = "9dwmh54u3bbh"
+BRIGHT_DATA_MOBILE_PROXY_PASSWORD = "fk5f7c2z2c19"
+PROXY_NAME_PREFIX_RES = "panleicim-res-fr-"
+PROXY_NAME_PREFIX_CC = "panleicim-cc-fr-"
+
+mongo_store_manager = MongoDbManager()
+
+
+def get_proxy_name_prefix() -> str:
+    return PROXY_NAME_PREFIX_RES
+
+
+def get_proxy(proxy_type: ProxyType):
+    if proxy_type == ProxyType.NETNUT:
+        random_id_number = get_random_id_number_for_proxy()
+        proxy_username = get_proxy_name_prefix() + random_id_number
+        proxy = {
+            "server": PROXY_SERVER,
+            "username": proxy_username,
+            "password": PROXY_PASSWORD
+        }
+    else:
+        proxy = {
+            "server": BRIGHT_DATA_PROXY_SERVER,
+            "username": BRIGHT_DATA_PROXY_USERNAME,
+            "password": BRIGHT_DATA_PROXY_PASSWORD
+        }
+    return proxy
+
+
+def get_random_id_number_for_proxy() -> str:
+    S = 8  # number of characters in the string.
+    ran = ''.join(random.choices(string.digits, k=S))
+    id_number = str(ran)
+    print("The randomly generated string is : " + str(ran))  # print the random data
+    return id_number
+
+
+local_db_manager = LocalDbManager(definitions.home)
+
+DEVICES = ['iPad (gen 6)', 'iPad (gen 6) landscape', 'iPad (gen 7)', 'iPad (gen 7) landscape', 'iPad Mini',
+           'iPad Mini landscape', 'iPad Pro 11', 'iPad Pro 11 landscape', 'iPhone 6', 'iPhone 6 landscape',
+           'iPhone 6 Plus', 'iPhone 6 Plus landscape', 'iPhone 7', 'iPhone 7 landscape', 'iPhone 7 Plus',
+           'iPhone 7 Plus landscape', 'iPhone 8', 'iPhone 8 landscape', 'iPhone 8 Plus', 'iPhone 8 Plus landscape',
+           'iPhone SE', 'iPhone SE landscape', 'iPhone X', 'iPhone X landscape', 'iPhone XR', 'iPhone XR landscape',
+           'iPhone 11', 'iPhone 11 landscape', 'iPhone 11 Pro', 'iPhone 11 Pro landscape', 'iPhone 11 Pro Max',
+           'iPhone 11 Pro Max landscape', 'iPhone 12', 'iPhone 12 landscape', 'iPhone 12 Pro',
+           'iPhone 12 Pro landscape', 'iPhone 12 Pro Max', 'iPhone 12 Pro Max landscape', 'iPhone 12 Mini', 'iPhone 13',
+           'iPhone 13 landscape', 'iPhone 13 Pro', 'iPhone 13 Pro landscape', 'iPhone 13 Pro Max', 'iPhone 13 Mini']

src/pojo/MailPojo.py

@@ -0,0 +1,14 @@
+class MailPojo:
+    email: str
+
+    def __init__(self, email: str):
+        self.email = email
+
+    @staticmethod
+    def from_firestore_dict(source):
+        email = source['email']
+        result = MailPojo(email=email)
+        return result
+
+    def __repr__(self):
+        return "email = " + self.email

src/pojo/ModeEnum.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class ModeEnum(Enum):
+    AUTOMATIC = "AUTOMATIC"
+    MANUAL = "MANUAL"

src/pojo/ReserveResultPojo.py

@@ -0,0 +1,90 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import Union
+
+from dataclasses_json import dataclass_json
+
+from src import definitions
+
+
+class PublishType(Enum):
+    SUCCESS = "SUCCESS"
+    ERROR = "ERROR"
+    PENDING = "PENDING"
+
+
+@dataclass_json
+@dataclass
+class ReserveResultPojo:
+    type: PublishType
+    phone: str
+    message: str
+    url: str
+    firstName: Union[None, str]
+    lastName: Union[None, str]
+    email: Union[None, str]
+    id = None
+    accepted = None
+    passport: str = ""
+    slot_position = None
+    sim_position = None
+    ccid: str = ""
+    source_from: str = definitions.LOG_SOURCE
+    store_type = 0
+
+    @staticmethod
+    def from_firestore_dict(source):
+        publish_type = source['type']
+        phone = source['phone']
+        message = source['message']
+        url = source['url']
+        id = source['id']
+        email = source['email']
+        lastName = source['lastName']
+        firstName = source['firstName']
+        result = ReserveResultPojo(type=publish_type, phone=phone,
+                                   message=message, url=url, email=email,
+                                   firstName=firstName, lastName=lastName)
+        if 'accepted' in source:
+            accepted = source['accepted']
+            result.accepted = accepted
+        if 'source' in source:
+            source_from = source['source']
+            result.source_from = source_from
+        if 'sim_position' in source:
+            sim_position = source['sim_position']
+            result.sim_position = sim_position
+        if 'slot_position' in source:
+            slot_position = source['slot_position']
+            result.slot_position = slot_position
+        if 'passport' in source:
+            passport = source['passport']
+            result.passport = passport
+        if 'ccid' in source:
+            ccid = source['ccid']
+            result.ccid = ccid
+        if 'store_type' in source:
+            store_type = source['store_type']
+            result.store_type = store_type
+        result.id = id
+        return result
+
+    def to_firestore_dict(self):
+        dest = {
+            u'type': self.type.value,
+            u'id': self.id,
+            u'message': self.message,
+            u'phone': self.phone,
+            u'firstName': self.firstName,
+            u'lastName': self.lastName,
+            u'email': self.email,
+            u'passport': self.passport,
+            u'url': self.url,
+            u'sim_position': self.sim_position,
+            u'slot_position': self.slot_position,
+            u'ccid': self.ccid,
+            u'source_from': self.source_from,
+            u'store_type': self.store_type,
+        }
+
+        return dest

src/pojo/ResultEnum.py

@@ -0,0 +1,7 @@
+from enum import Enum
+
+
+class ResultEnum(Enum):
+    ACCEPTED = "ACCEPTED"
+    REFUSED = "REFUSED"
+    PENDING = "PENDING"

src/pojo/SimInfoPojo.py

@@ -0,0 +1,64 @@
+import time
+
+
+class SimInfoPojo:
+    phone: str
+    ccid: str
+    update_at: int
+    position: int
+    operator: str
+    slot_position: str
+    sim_position: str
+    name: str
+    passport: str
+    email: str
+
+    def __init__(self, phone: str, ccid: str, position, operator: str, slot_position, sim_position, name=None,
+                 passport=None,
+                 email=None,
+                 update_at: int = int(time.time())):
+        self.phone = phone
+        self.ccid = ccid
+        self.update_at = update_at
+        self.position = position
+        self.operator = operator
+        self.slot_position = slot_position
+        self.sim_position = sim_position
+        self.name = name
+        self.passport = passport
+        self.email = email
+
+    @staticmethod
+    def from_firestore_dict(source):
+        phone = source['phone']
+        ccid = source['ccid']
+        sim_position = source['sim_position']
+        slot_position = source['slot_position']
+        update_at = None
+        if 'update_at' in source:
+            update_at = source['update_at']
+        position = None
+        if 'position' in source:
+            position = source['position']
+        result = SimInfoPojo(phone=phone, ccid=ccid, update_at=update_at, sim_position=sim_position,
+                             slot_position=slot_position, position=position, operator="")
+        if 'operator' in source:
+            operator = source['operator']
+            result.operator = operator
+        result.id = id
+        return result
+
+    def to_firestore_dict(self):
+        dest = {
+            u'phone': self.phone,
+            u'ccid': self.ccid,
+            u'update_at': self.update_at,
+            u'position': self.position,
+            u'operator': self.operator,
+            u'slot_position': self.slot_position,
+            u'sim_position': self.sim_position,
+            u'email': self.email,
+            u'passport': self.passport,
+        }
+
+        return dest

src/pojo/captcha_error_contact_pojo.py

@@ -0,0 +1,23 @@
+from sqlalchemy import Column, String, Integer, DateTime, func
+from sqlalchemy.orm import declarative_base
+
+
+Base = declarative_base()
+
+ERROR_TYPE_CAPTCHA = 1
+TOO_MANY_REQUEST_ERROR = 2
+
+
+class ContactInErrorPojo(Base):
+    __tablename__ = "contacts_in_error"
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    mail: str = Column(String)
+    phone: str = Column(String)
+    passport: str = Column(String)
+    last_name: str = Column(String)
+    first_name: str = Column(String)
+    ccid: str = Column(String)
+    position: int = Column(Integer)
+    error_type = Column(Integer)
+    update_at = Column(DateTime, onupdate=func.now())
+    create_at = Column(DateTime, default=func.now())

src/pojo/contact_pojo.py

@@ -0,0 +1,57 @@
+from dataclasses import dataclass
+
+from src.pojo.captcha_error_contact_pojo import ContactInErrorPojo
+
+
+@dataclass
+class ContactPojo:
+    phone: str
+    passport: str
+    last_name: str
+    first_name: str
+    mail: str
+    ccid: str
+    position: int
+
+    def __init__(self, phone_number: str, passport_number: str, last_name: str, first_name: str, mail: str,
+                 ccid: str = "",
+                 position: int = 0):
+        self.phone = phone_number
+        self.passport = passport_number
+        self.last_name = last_name
+        self.first_name = first_name
+        self.ccid = ccid
+        self.mail = mail
+        self.position = position
+
+    def to_firestore_dict(self):
+        dest = {
+            u'phone': self.phone,
+            u'passport': self.passport,
+            u'last_name': self.last_name,
+            u'first_name': self.first_name,
+            u'mail': self.mail,
+            u'ccid': self.ccid,
+            u'position': self.position
+        }
+
+        return dest
+
+    @staticmethod
+    def get_contact_from_error_contact(errorContact: ContactInErrorPojo):
+        return ContactPojo(phone_number=errorContact.phone, mail=errorContact.mail, ccid=errorContact.ccid,
+                           last_name=errorContact.last_name, first_name=errorContact.first_name,
+                           position=errorContact.position, passport_number=errorContact.passport)
+
+    @staticmethod
+    def from_firestore_dict(source):
+        ccid = source['ccid']
+        phone = source['phone']
+        position = source['position']
+        passport = source['passport']
+        email = source['mail']
+        lastName = source['last_name']
+        firstName = source['last_name']
+        result = ContactPojo(ccid=ccid, phone_number=phone, passport_number=passport, position=position, mail=email,
+                             last_name=lastName, first_name=firstName)
+        return result

src/proxy/proxy_type.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class ProxyType(Enum):
+    NETNUT = "NETNUT"
+    BRIGHT_DATA = "BRIGHT_DATA"

src/utils/excel_reader.py

@@ -0,0 +1,141 @@
+import json
+import random
+import string
+
+import pandas as pandas
+import xlsxwriter
+
+from src.definitions import CONTACT_LIST_FILE
+from src.pojo.contact_pojo import ContactPojo
+from src.utils.generate_random_passport_id import get_random_passport_id_number
+
+phone_number_prefix = ['6']
+
+
+class ExcelHelper:
+
+    def __init__(self):
+        self._df = pandas.Series()
+
+    def write_to_exel(self, file_name, data_list: list):
+        new_df = pandas.Series(data_list)
+        self._df = pandas.concat([self._df, new_df])
+        self._df.to_excel(file_name)
+
+    def read_contacts(self, file_name=CONTACT_LIST_FILE) -> list:
+        contact_list_in_json = pandas.read_excel(file_name).to_json(orient='records')
+        contact_dict_list = json.loads(contact_list_in_json)
+        contact_list = []
+        for contact_dict in contact_dict_list:
+            if contact_dict['name']:
+                raw_name = contact_dict['name'].strip()
+                name = raw_name.split(' ')
+                last_name = name[0]
+                if len(name) == 2:
+                    first_name = name[-1]
+                else:
+                    first_name = ''.join(name[1:len(name)])
+
+                contact = ContactPojo(phone_number=contact_dict['phone'],
+                                      last_name=last_name,
+                                      first_name=first_name,
+                                      passport_number=contact_dict['passport'],
+                                      mail=contact_dict['email'])
+                contact_list.append(contact)
+        return contact_list
+
+    def read_names(self, file_name=CONTACT_LIST_FILE) -> list:
+        contact_list_in_json = pandas.read_excel(file_name).to_json(orient='records')
+        contact_dict_list = json.loads(contact_list_in_json)
+        contact_list = []
+        count = 2
+        for contact_dict in contact_dict_list:
+            if contact_dict['name']:
+                raw_name = contact_dict['name'].strip()
+                name = raw_name.split(' ')
+                if len(name) == 1:
+                    name = raw_name.split('\xa0')
+                if len(name) == 1:
+                    print("error in " + str(name))
+                last_name = name[0]
+                if len(name) == 2:
+                    first_name = name[-1]
+                else:
+                    first_name = ''.join(name[1:len(name)])
+
+                contact = ContactPojo(phone_number="",
+                                      last_name=last_name,
+                                      first_name=first_name,
+                                      passport_number="",
+                                      mail="")
+
+                if len(first_name) == 0:
+                    print("first_name is empty: position:" + str(count))
+                    print(name)
+                if len(last_name) == 0:
+                    print("last_name is empty: position:" + str(count))
+                count = count + 1
+                contact_list.append(contact)
+
+        return contact_list
+
+
+def get_random_phone_numbers():
+    length = 8  # number of characters in the string.
+    ran = ''.join(random.choices(string.digits, k=length))
+    id_number = random.choice(phone_number_prefix) + str(ran)
+    return id_number
+
+
+def generate_email_from_name(first_name: str, last_name: str) -> str:
+    length = 2  # number of characters in the string.
+    ran = ''.join(random.choices(string.digits, k=length))
+    separator = ['.', '_', '']
+    domains = ['gmail.com', 'hotmail.com', 'yahoo.com', 'aol.com', 'outlook.com', 'hotmail.fr', 'gmx.com',
+               'hotmail.com', 'yahoo.com', 'aol.com', 'hotmail.com']
+    email = "{}{}{}{}@{}".format(last_name.lower(), random.choice(separator),
+                                 first_name.replace("-", "").replace("'", "").lower(), ran,
+                                 random.choice(domains))
+    print(email)
+    return email
+
+
+def get_random_id_number() -> str:
+    # write_the_valid_profiles_to_excel()
+    S = 8  # number of characters in the string.
+    # call random.choices() string module to find the string in Uppercase + numeric data.
+    ran = ''.join(random.choices(string.digits, k=S))
+    print("The randomly generated string is : 94" + str(ran))  # print the random data
+    return ran
+
+
+def write_new_contacts_to_excel(valid_contacts: list):
+    row = 0
+    col = 0
+    # Create a workbook and add a worksheet.
+    workbook = xlsxwriter.Workbook('new_profile_{}.xlsx'.format(len(valid_contacts)))
+    header_data = ['name', 'phone', 'passport', 'email']
+    worksheet = workbook.add_worksheet()
+    header_format = workbook.add_format({'bold': True})
+
+    for col_num, data in enumerate(header_data):
+        worksheet.write(row, col_num, data, header_format)
+    row = row + 1
+    for info in valid_contacts:
+        info.phone = get_random_phone_numbers()
+        info.passport = get_random_passport_id_number()
+        info.mail = generate_email_from_name(info.first_name, info.last_name)
+        # Iterate over the data and write it out row by row.
+        worksheet.write(row, col, "{} {}".format(info.last_name, info.first_name))
+        worksheet.write(row, col + 1, info.phone)
+        worksheet.write(row, col + 2, info.passport)
+        worksheet.write(row, col + 3, info.mail)
+        row += 1
+    workbook.close()
+
+
+if __name__ == '__main__':
+    excel_reader = ExcelHelper()
+    contacts = excel_reader.read_names("C:/Users/landd/Desktop/rdv/08-06/500.xlsx")
+    print(contacts)
+    write_new_contacts_to_excel(valid_contacts=contacts)

src/utils/export_valid_profiles.py

@@ -0,0 +1,73 @@
+import random
+import string
+
+import xlsxwriter
+
+import params
+from pojo.ReserveResultPojo import ReserveResultPojo
+from pojo.contact_pojo import ContactPojo
+
+
+def get_random_id_number() -> str:
+    # write_the_valid_profiles_to_excel()
+    S = 8  # number of characters in the string.
+    # call random.choices() string module to find the string in Uppercase + numeric data.
+    ran = ''.join(random.choices(string.digits, k=S))
+    id_number = "57" + str(ran)
+    print("The randomly generated string is : 94" + str(ran))  # print the random data
+    return id_number
+
+
+def write_the_valid_profiles_to_excel():
+    day_list = ['2022-06-02']
+
+    collection = []
+    for day in day_list:
+        collection.extend(params.firebase_store_manager.get_all_successful_items_for_day(day, source_from=None).stream())
+
+    valid_contacts = []
+    # exist_contacts = ExcelHelper().read_contacts()
+    for valid_appointment in collection:
+        reserve_pojo = ReserveResultPojo.from_firestore_dict(valid_appointment.to_dict())
+        # check whether the contact exists already
+        # exist = [contact for contact in valid_contacts if contact.mail == reserve_pojo.email]
+        # if len(exist) == 0:
+        contact = ContactPojo(reserve_pojo.phone, passport_number=get_random_id_number(),
+                              last_name=reserve_pojo.lastName, first_name=reserve_pojo.firstName, ccid="",
+                              mail=reserve_pojo.email, position=0)
+        # seed = 8  # number of characters in the string.
+        # call random.choices() string module to find the string in Uppercase + numeric data.
+        # contact.passport = get_random_id_number()
+        # if contact.passport == None or len(contact.passport) == 0:
+        #     old_contact = [item for item in exist_contacts if item.mail == contact.mail]
+        #     if len(old_contact) > 0:
+        #         contact.passport = old_contact[0].passport
+
+        print(contact)
+        valid_contacts.append(contact)
+
+    row = 0
+    col = 0
+    # Create a workbook and add a worksheet.
+    workbook = xlsxwriter.Workbook('valid_profile.xlsx')
+    header_data = ['name', 'phone', 'passport', 'email', 'ccid', 'position']
+    worksheet = workbook.add_worksheet()
+    header_format = workbook.add_format({'bold': True})
+
+    for col_num, data in enumerate(header_data):
+        worksheet.write(row, col_num, data, header_format)
+    row = row + 1
+    for info in valid_contacts:
+        # Iterate over the data and write it out row by row.
+        worksheet.write(row, col, "{} {}".format(info.last_name, info.first_name))
+        worksheet.write(row, col + 1, info.phone)
+        worksheet.write(row, col + 2, info.passport)
+        worksheet.write(row, col + 3, info.mail)
+        worksheet.write(row, col + 4, info.ccid)
+        row += 1
+    workbook.close()
+
+
+if __name__ == '__main__':
+    # get_random_id_number()
+    write_the_valid_profiles_to_excel()

src/utils/generate_random_passport_id.py

@@ -0,0 +1,43 @@
+import random
+import string
+
+
+# letters = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'P', 'Q', 'R', 'S', 'T', '1', '2', '3', '4', '5', '6', '7',
+#            '8', '9']
+from src.pojo.captcha_error_contact_pojo import ContactInErrorPojo
+from src.pojo.contact_pojo import ContactPojo
+
+letters = ['E', 'G', 'M']
+
+
+def get_random_id_number() -> str:
+    S = 8  # number of characters in the string.
+    ran = ''.join(random.choices(string.digits, k=S))
+    id_number = str(ran)
+    return id_number
+
+# 产生中国护照号
+def get_random_passport_id_number() -> str:
+    S = 8  # number of characters in the string.
+    ran = ''.join(random.choices(string.digits, k=S))
+    id_number = random.choice(letters) + str(ran)
+    return id_number
+
+
+def get_captcha_error_contact_from_contact(contact: ContactPojo, error_type: int) -> ContactInErrorPojo:
+    captcha_error = ContactInErrorPojo()
+    captcha_error.mail = contact.mail
+    captcha_error.ccid = contact.ccid
+    captcha_error.phone = contact.phone
+    captcha_error.passport = contact.passport
+    captcha_error.first_name = contact.first_name
+    captcha_error.last_name = contact.last_name
+    captcha_error.position = contact.position
+    captcha_error.error_type = error_type
+    return captcha_error
+
+if __name__ == '__main__':
+    # for i in range(1,200):
+    #     print(get_random_id_number())
+    for i in range(1, 804):
+        print(get_random_id_number())

src/utils/message_receiver.py

@@ -0,0 +1,22 @@
+import threading
+
+from params import rabittmq_connection
+
+APPOINTMENT_QUEUE = "APPOINTMENT_QUEUE"
+
+
+class MessageReceiver:
+
+    def start_listener(self, callback):
+        t = threading.Thread(target=self._run, args=(callback,))
+        t.start()
+
+    def _run(self, callback):
+
+        channel = rabittmq_connection.channel()
+        channel.queue_declare(queue=APPOINTMENT_QUEUE)
+        channel.basic_consume(queue=APPOINTMENT_QUEUE,
+                              auto_ack=True,
+                              on_message_callback=callback)
+        print(' [*] Waiting for messages. To exit press CTRL+C')
+        channel.start_consuming()

src/utils/name_generator.py

@@ -0,0 +1,19 @@
+import json
+
+import requests
+
+
+def generate_names() -> list:
+    res = requests.get(
+        "https://www.namegeneratorfun.com/api/namegenerator?generatorType=list&firstName=&lastName=&minLength=0&maxLength=255&sexId=2&generatorId=43")
+    response_json = json.loads(res.text)
+    names = response_json['names']
+    # print(names)
+    return names
+
+
+if __name__ == '__main__':
+    for i in range(0, 500):
+        names = generate_names()
+        for name in names:
+            print(name)

src/workers/SolveCaptch.py

@@ -0,0 +1,54 @@
+# for recaptcha
+import logging
+import random
+import re
+import time
+import requests
+
+CAPCHA_NOT_READY = "CAPCHA_NOT_READY"
+REGEX_DATA_SITE_KEY = "data-sitekey=[\"a-z0-9A-Z]+"
+API_KEY = "d66aaf490d8aa424a5175e1fbd1aadea"
+
+
+class SolveCaptcha:
+    def __init__(self, page):
+        self.page = page
+        self.logger = logging.getLogger("SolveCaptcha")
+        self.main_frame = None
+        self.recaptcha = None
+
+    def delay(self):
+        self.page.wait_for_timeout(random.randint(1, 3) * 1000)
+
+    def start(self, handle_solution_received):
+        self.logger.info("start to resolve captcha")
+        content = self.page.content()
+        data_sitekey = re.findall(REGEX_DATA_SITE_KEY, content)
+        self.logger.info(data_sitekey)
+        if len(data_sitekey) == 1:
+            key_with_comma = data_sitekey[0].split("=")[-1]
+            key = key_with_comma.replace("\"", '')
+            self.logger.info("key is : " + key)
+            self.solve_captcha(key, handle_solution_received)
+
+    def solve_captcha(self, google_key: str, handle_solution_received):
+        self.logger.info("solve_captcha()")
+        url_get = "http://2captcha.com/in.php?key={}&method=userrecaptcha&googlekey={}&pageurl={}".format(API_KEY,
+                                                                                                          google_key,
+                                                                                                          self.page.url)
+        res = requests.get(url_get)
+        self.logger.info(res.text)
+        results = res.text.split("|")
+        self.captcha_id = results[-1]
+        #       wait for 15 seconds
+        time.sleep(15)
+        #         get result of the captcha
+        url_response = "http://2captcha.com/res.php?key={}&action=get&id={}".format(API_KEY,
+                                                                                    self.captcha_id)
+        solution = CAPCHA_NOT_READY
+        while solution == CAPCHA_NOT_READY:
+            solution_res = requests.get(url_response)
+            time.sleep(5)
+            solution = solution_res.text
+        print(solution)
+        handle_solution_received(solution.split("|")[-1])

src/workers/commandor_page.py

@@ -0,0 +1,398 @@
+import logging
+import random
+import re
+import sys
+import threading
+import time
+import traceback
+from typing import Union
+
+from playwright.sync_api import sync_playwright
+
+from src import params
+from src.pojo.ModeEnum import ModeEnum
+from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType
+from src.pojo.captcha_error_contact_pojo import TOO_MANY_REQUEST_ERROR, ERROR_TYPE_CAPTCHA
+from src.pojo.contact_pojo import ContactPojo
+from src.proxy.proxy_type import ProxyType
+from src.utils.generate_random_passport_id import get_captcha_error_contact_from_contact
+from src.workers.SolveCaptch import SolveCaptcha
+
+RDV_URL = "https://rendezvousparis.hermes.com/client/register"
+
+# RDV_URL = "file:///Users/lpan/Downloads/test_appointment.html"
+# RDV_URL = "https://api.ipify.org"
+# RDV_URL ="https://bot.sannysoft.com/"
+REGEX_RDV_URL = "https:\/\/rendezvousparis\.hermes\.com\/client\/register\/[A-Z0-9]+"
+otp_value = None
+OTP_FIELD_ID = "#sms_code"
+MESSAGE_FIELD_CLASS = ".message"
+BLANK_URL = "about:blank"
+CONFIRMED_MESSAGE = "Your request for a Leather Goods appointment has been registered"
+CONFIRMED_MESSAGE_FR = "Votre demande de rendez-vous Maroquinerie a bien été enregistrée et nous vous en remercions."
+DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today."
+DOUBLE_REQUEST_ERROR_MESSAGE_FR = "Une demande avec les données saisies a déjà été validée aujourd’hui."
+TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests"
+TOO_MANY_REQUEST_ERROR_MESSAGE_FR = "Suite à un trop grand nombre de demandes"
+CAPTCHA_ERROR_MESSAGE = "Error verifying captcha, please try again"
+CAPTCHA_ERROR_MESSAGE_FR = "La vérification du captcha a échoué"
+TIME_OUT = 10*60*1000 #10 mins
+OTP_TIMEOUT = 240
+PAGE_TIMEOUT = 40000
+
+
+def get_random_wait_time() -> float:
+    wait_time = random.randint(0, 10) / 10.0 * 1
+    return wait_time
+
+
+class Tls(threading.local):
+    def __init__(self) -> None:
+        self.playwright = sync_playwright().start()
+
+
+class CommandorPage:
+    tls = Tls()
+
+    def __init__(self, contact: ContactPojo, store_type=0, proxy_type=ProxyType.BRIGHT_DATA,
+                 mode: ModeEnum = ModeEnum.MANUAL):
+        self.otp_value = None
+        self.logger = logging.getLogger("约会页面:" + str(contact.phone))
+        self.is_finished = False
+        self.contact = contact
+        self.proxy_type = proxy_type
+        self.is_event_sent = False
+        self.is_captcha_in_error = False
+        self.is_filling_fields = False
+        self.appointment_mode = mode
+        # 0: random
+        # 1: faubourg
+        # 2: George
+        # 3: Sèvres
+        self.store_map = {
+            1: "faubourg",
+            2: "georgev",
+            3: "sevres"
+        }
+        self.store_type = store_type
+
+    def on_success(self, result: ReserveResultPojo):
+        self.logger.info("on_success called.")
+        self.is_finished = True
+        if not self.is_event_sent:
+            self.logger.info("will send successful event")
+            self.logger.info(result)
+            params.oracle_log_sender.send_appoint_result(result)
+            self.is_event_sent = True
+
+    def timeout_occurred(self):
+        params.oracle_log_sender.send_timeout_log(self.contact)
+        self.logger.info("will close timeout modem")
+        self.termine()
+
+    def _run(self, e: threading.Event, proxy):
+        self.logger.info("will start browser")
+        self.on_success_listener = on_success
+        # reset otp_value to None
+        self.otp_value = None
+        devices = random.choice(params.DEVICES)
+        first_page = None
+        while first_page is None:
+            first_page = self.start_browser(proxy, self.tls.playwright, devices)
+            proxy = params.get_proxy(self.proxy_type)
+        # self.thread_event = e
+        otp_input = self.page.locator(OTP_FIELD_ID)
+        otp_input.wait_for(state='visible', timeout=TIME_OUT)
+        # event_is_set = e.wait()
+        # logging.info('event set: %s', event_is_set)
+        # if self.otp_value:
+        #     self.fill_otp(self.otp_value)
+        #     time.sleep(get_random_wait_time())
+        #     self.clickOnValidBtn()
+        #     otp_sent = self.page.locator(MESSAGE_FIELD_CLASS)
+        #     otp_sent.wait_for(state='visible', timeout=TIME_OUT)
+        # time.sleep(get_random_wait_time())
+        # message = self.page.content()
+        # if CONFIRMED_MESSAGE in message or CONFIRMED_MESSAGE_FR in message:
+        #     # publish the successful message
+        #     self.logger.info("url is " + self.page.url)
+        #     self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url)
+        # else:
+        self.logger.info("timeout")
+        self.termine()
+
+    def fill_fields(self):
+        if not self.is_filling_fields:
+            self.is_filling_fields = True
+            self.logger.info("填充信息: " + str(self.contact.phone))
+            self._set_name(self.contact.last_name, self.contact.first_name)
+            self._setPhoneCountryAndStore()
+            self._set_phone_number("0" + str(self.contact.phone))
+            self._set_email(self.contact.mail)
+            self._set_id_number(self.contact.passport)
+            self._checkCgu()
+            if self.appointment_mode == ModeEnum.AUTOMATIC:
+                self.resolve_captcha()
+            self.is_filling_fields = False
+
+    def start_browser(self, proxy, pwright, device) -> Union[str, None]:
+        try:
+            self.browser = pwright.webkit.launch(headless=False, timeout=PAGE_TIMEOUT, proxy=proxy)
+            self.logger.info("模拟设备: " + device)
+            simulated_mobile = pwright.devices[device]
+            context = self.browser.new_context(**simulated_mobile, locale='fr-FR')
+            self.page = context.new_page()
+            # hide webdriver information
+            self.page.add_init_script("""() => {
+    Object.defineProperty(navigator,'webdriver',{get: () => undefined});
+    Object.defineProperty(navigator, 'platform', {
+        get: () => {
+            return "iPhone";
+        }});
+}
+            """)
+            self.page.on("load", self._on_page_loaded)
+            self.page.on("response", self.handle_response)
+            self.page.goto(RDV_URL, timeout=PAGE_TIMEOUT)
+            return self.page.content()
+        except Exception as error:
+            params.oracle_log_sender.send_error(str(error))
+            traceback.print_exc(*sys.exc_info())
+            self.logger.exception(error)
+            self.logger.info("will close browser")
+            self.browser.close()
+            return None
+
+    def handle_response(self, response):
+        pattern = re.compile(REGEX_RDV_URL)
+        if pattern.match(response.url):
+            self.logger.info("result url found: " + response.url)
+            # self.publish_message_to_queue(self.contact, PublishType.PENDING, response.url)
+
+    def start_page(self, proxy):
+        e = threading.Event()
+        self._run(e, proxy)
+
+    def _on_page_loaded(self):
+        self.logger.info("页面加载完毕")
+        self.logger.info("url is " + self.page.url)
+        if self.page.url == RDV_URL:
+            self.fill_fields()
+        try:
+            message = self.page.content()
+            if CONFIRMED_MESSAGE in message or CONFIRMED_MESSAGE_FR in message:
+                # publish the successful message
+                self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url)
+            self.get_errors()
+        except Exception as error:
+            self.logger.error(error)
+
+    def on_document_loaded(self):
+        self.logger.info("on_document_loaded called")
+
+    def _setPhoneCountryAndStore(self):
+        try:
+            if self.store_type == 0:
+                self.page.evaluate("""()=>{
+                //document.getElementById("phone_country").focus();
+                document.getElementById("phone_country").value = \"FR\" }""")
+            else:
+                store_to_choose = self.store_map[self.store_type]
+                self.page.evaluate("""(store_to_choose)=>{
+                document.getElementById("prefer").value = store_to_choose;
+                //document.getElementById("phone_country").focus();
+                document.getElementById("phone_country").value = \"FR\" }""", store_to_choose)
+        except Exception as error:
+            self.logger.error(error)
+
+    def _set_phone_number(self, phoneNumber):
+        time.sleep(get_random_wait_time())
+        try:
+            self.page.evaluate("""(phoneNumber)=>document.getElementById("phone_number").value =phoneNumber""",
+                               phoneNumber)
+        except Exception as error:
+            self.logger.error(error)
+
+    def _set_name(self, lastName, firstName):
+        time.sleep(get_random_wait_time())
+        try:
+            self.page.evaluate("""(name)=>   {
+            let surname = document.getElementById("surname");
+            if(surname.value.length == 0){
+                        // surname.focus();
+            surname.value = name.lastName;
+                            document.getElementById("name").focus();
+            document.getElementById("name").value = name.firstName
+            }}
+        """, {'lastName': lastName, 'firstName': firstName})
+        except Exception as error:
+            self.logger.error(error)
+
+    def get_errors(self):
+        # send error result
+        if self.page.url != BLANK_URL:
+            # no need to push blank url to db
+            if self.page.url != RDV_URL:
+                # no need to push RDV url to db
+                self.publish_message_to_queue(self.contact, PublishType.ERROR, self.page.url)
+        try:
+            items = self.page.query_selector("div.alert")
+            if items:
+                erro_content = items.inner_html()
+                self.logger.info("错误:" + erro_content)
+                self._handle_errors(erro_content)
+        except Exception as ext:
+            self.logger.error(ext)
+
+    def _handle_errors(self, erro_content: str):
+        if DOUBLE_REQUEST_ERROR_MESSAGE in erro_content or DOUBLE_REQUEST_ERROR_MESSAGE_FR in erro_content:
+            # this email has been already used
+            if not self.is_finished:
+                params.oracle_log_sender.send_double_data_error(self.contact)
+            self.is_finished = True
+            self.termine()
+        elif TOO_MANY_REQUEST_ERROR_MESSAGE in erro_content or TOO_MANY_REQUEST_ERROR_MESSAGE_FR in erro_content:
+            # this email has been already used
+            if not self.is_finished:
+                params.local_db_manager.insert_or_update(
+                    get_captcha_error_contact_from_contact(self.contact, TOO_MANY_REQUEST_ERROR))
+                params.oracle_log_sender.send_too_many_error(self.contact)
+            self.is_finished = True
+            self.termine()
+        elif CAPTCHA_ERROR_MESSAGE in erro_content or CAPTCHA_ERROR_MESSAGE_FR in erro_content:
+            # this email has been already used
+            self.is_captcha_in_error = True
+            if not self.is_finished:
+                # save the error to database with contact info
+                self.handle_captcha_error()
+            self.is_finished = True
+            # no need to retry captcha, if retry ,will generate DOUBLE_REQUEST_ERROR_MESSAGE
+            self.termine()
+            # self.resolve_captcha()
+
+    def _set_email(self, email):
+        time.sleep(get_random_wait_time())
+        try:
+            self.page.evaluate("""(email)=>{
+            let emailElement = document.getElementById("email")
+            if(emailElement.value.length == 0){
+                                emailElement.focus();
+                document.getElementById("email").value = email;}}""", email)
+        except Exception as error:
+            self.logger.error(error)
+
+    def _set_id_number(self, id):
+        time.sleep(get_random_wait_time())
+        try:
+            self.page.evaluate(""" (id) =>{
+            document.getElementById("passport_id").focus();
+             document.getElementById("passport_id").value = id}""", id)
+        except Exception as error:
+            self.logger.error(error)
+
+    def _checkCgu(self):
+        try:
+            self.page.evaluate("""
+                    document.getElementById("cgu").focus();
+            document.getElementById("cgu").checked = true;
+                            document.getElementById("processing").focus();
+        document.getElementById("processing").checked = true""")
+        except Exception as error:
+            self.logger.error(error)
+
+    def clickOnValidBtn(self):
+        time.sleep(get_random_wait_time())
+        try:
+            self.page.evaluate("""document.getElementsByClassName("btn")[0].focus();""")
+            time.sleep(get_random_wait_time())
+            self.page.evaluate("""
+            document.getElementsByClassName("btn")[0].click();""")
+        except Exception as error:
+            self.logger.error(error)
+
+    def fill_otp(self, otp: str):
+        self.page.focus(OTP_FIELD_ID)
+        time.sleep(get_random_wait_time())
+        self.page.fill(OTP_FIELD_ID, otp)
+
+    def termine(self):
+        self.logger.info("will close browser")
+        time.sleep(1)
+        self.browser.close()
+
+    def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str):
+        # create the message
+        id = url.split("/")[-1]
+        result = ReserveResultPojo(type=status, phone=contact.phone, message=status.value, url=url,
+                                   firstName=contact.first_name, lastName=contact.last_name, email=contact.mail,
+                                   passport=contact.passport, ccid=contact.ccid)
+        result.id = id
+        result.store_type = self.store_type
+        params.firebase_store_manager.save(result)
+        params.local_db_manager.handle_success(result)
+        params.mongo_store_manager.delete_captcha_error_contact_for_current_day(self.contact)
+        if status is PublishType.SUCCESS:
+            self.on_success(result)
+            time.sleep(2)
+            self.browser.close()
+
+    def resolve_captcha(self):
+        self.captcha_solver = SolveCaptcha(self.page)
+        self.captcha_solver.start(self.fill_captcha_solution)
+
+    def fill_captcha_solution(self, solution):
+        self.logger.info("will input solution: " + solution)
+        try:
+            self.page.evaluate("""(solution)=>{
+               document.getElementById("g-recaptcha-response").innerHTML=solution;}""", solution)
+            self.logger.info("will click on valid btn")
+            self.clickOnValidBtn()
+            #     wait for 20s
+            time.sleep(20)
+            if not self.is_finished:
+                if not self.is_captcha_in_error:
+                    self.clickOnValidBtn()
+                else:
+                    self.is_captcha_in_error = False
+
+        except Exception as error:
+            self.logger.error(error)
+            self.page.reload(timeout=PAGE_TIMEOUT)
+
+    def handle_captcha_error(self):
+        params.local_db_manager.insert_or_update(
+            get_captcha_error_contact_from_contact(self.contact, ERROR_TYPE_CAPTCHA))
+        params.mongo_store_manager.insert_captcha_error_contact(self.contact)
+        params.oracle_log_sender.send_captcha_error(self.contact)
+
+
+def on_success(result: ReserveResultPojo):
+    pass
+
+
+def launch_page():
+    contact = ContactPojo(phone_number="+33758912245", passport_number="82546975", last_name="XU",
+                          first_name="xingzhen",
+                          mail="ColbyPatel653@gmail.com", ccid="", position=0)
+    page = CommandorPage(contact, store_type=1)
+    return page.start_page(params.get_proxy(ProxyType.BRIGHT_DATA))
+
+
+def wait_for_otp(event: threading.Event, commandor: CommandorPage):
+    sec = input("Press Enter otp to continue...\n")
+    print("input otp is: " + sec)
+    commandor.otp_value = sec
+    event.set()
+
+
+if __name__ == '__main__':
+    launch_page()
+    # time = get_random_wait_time()
+    # print(time)
+    # import urllib.request
+    #
+    # proxy = urllib.request.ProxyHandler({'https': 'http://panleicim-res-fr-121:94sY7zwBG13i@gw.ntnt.io:5959'})
+    # opener = urllib.request.build_opener(proxy)
+    # urllib.request.install_opener(opener)
+    # content = urllib.request.urlopen('https://api.ipify.org').read()
+    # print(content)