Merge branch 'master' of bitbucket.org:panleicim/appointment_tool

.gitignore

@@ -12,3 +12,4 @@ out
 lib
 appointment_*.log
 ./build
+./dist

appointment.spec

@@ -49,7 +49,7 @@ coll = COLLECT(
     name='appointment',
 )
 import shutil
-shutil.copyfile('C:\\Users\\landd\\IdeaProjects\\appointment_tool\\appointment.json', '{0}/appointment.json'.format(DISTPATH))
+shutil.copyfile('./appointment.json', '{0}/appointment.json'.format(DISTPATH))
-shutil.copyfile('C:\\Users\\landd\\IdeaProjects\\appointment_tool\\contact_all.xlsx', '{0}/contact_all.xlsx'.format(DISTPATH))
+shutil.copyfile('./contact_all.xlsx', '{0}/contact_all.xlsx'.format(DISTPATH))
-shutil.copyfile('.\config.ini', '{0}/config.ini'.format(DISTPATH))
+shutil.copyfile('./config.ini', '{0}/config.ini'.format(DISTPATH))
 shutil.copytree('./venv\Lib/site-packages/grpc/_cython/_credentials', '{0}/appointment/grpc/_cython/_credentials'.format(DISTPATH))

appointment_mac.spec

@@ -0,0 +1,55 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+
+block_cipher = None
+
+
+a = Analysis(
+    ['appointment.py'],
+    pathex=[],
+    binaries=[],
+    datas=[],
+    hiddenimports=[],
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[],
+    win_no_prefer_redirects=False,
+    win_private_assemblies=False,
+    cipher=block_cipher,
+    noarchive=False,
+)
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    [],
+    exclude_binaries=True,
+    name='appointment',
+    debug=True,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=False,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+)
+coll = COLLECT(
+    exe,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    name='appointment',
+)
+import shutil
+shutil.copyfile('./appointment.json', '{0}/appointment.json'.format(DISTPATH))
+shutil.copyfile('./contact_all.xlsx', '{0}/contact_all.xlsx'.format(DISTPATH))
+shutil.copyfile('./config.ini', '{0}/config.ini'.format(DISTPATH))
+shutil.copytree('./venv/lib/python3.8/site-packages/grpc/_cython/_credentials', '{0}/appointment/grpc/_cython/_credentials'.format(DISTPATH))

check_results.py

@@ -8,12 +8,13 @@ from typing import Union
 from playwright.sync_api import sync_playwright
 
 import params
-from workers.commandor_page import get_random_id_number_for_proxy
 from logs.LogSender import TYPE_EVENT_CHECK_RESULTS, LOG_SUBJECT_EVENT
-from pojo.ReserveResultPojo import ReserveResultPojo
+from pojo.ReserveResultPojo import ReserveResultPojo, PublishType
 
 SORRY_SENTENCE = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci"
+NOT_AVAILABLE_CONTENT = "For more than 130 years, our House has offered its full expertise to satisfy"
 PENDING_SENTENCE = "Ce soir, entre 20:00 et 20:30, vous obtiendrez une réponse par e-mail."
+PENDING_SENTENCE_EN = "This evening between 20:00 and 20:30 you will receive a response by email."
 
 
 class ResultEnum(Enum):
@@ -22,14 +23,14 @@ class ResultEnum(Enum):
     PENDING = "PENDING"
 
 
-class Tls(threading.local):
+class TlsPlaywright(threading.local):
     def __init__(self) -> None:
         self.playwright = sync_playwright().start()
         print("Create playwright instance in Thread", threading.current_thread().name)
 
 
-class Worker:
+class ResultChecker:
-    tls = Tls()
+    tls = TlsPlaywright()
 
     def __init__(self):
         self.logger = logging.getLogger("Worker")
@@ -38,7 +39,7 @@ class Worker:
         try:
             self.browser = playwright.webkit.launch(headless=False, timeout=90000, proxy=proxy)
             device = random.choice(params.DEVICES)
-            self.logger.info("device is " + device)
+            self.logger.info("模拟设备: " + device)
             pixel_2 = self.tls.playwright.devices[device]
             context = self.browser.new_context(**pixel_2, locale='en-GB')
             self.page = context.new_page()
@@ -74,8 +75,8 @@ class Worker:
             "password": params.PROXY_PASSWORD
         }
         while content is None:
-            content = self.load_page(self.tls.playwright, url, proxy)
+            content = self.load_page(self.tls.playwright, proxy, url)
-            random_id_number = get_random_id_number_for_proxy()
+            random_id_number = params.get_random_id_number_for_proxy()
             proxy_username = "panleicim-res-fr-" + random_id_number
             print("proxy_username is " + proxy_username)
             proxy = {
@@ -93,6 +94,12 @@ class Worker:
         elif PENDING_SENTENCE in content:
             print("status is PENDING")
             status = ResultEnum.PENDING
+        elif PENDING_SENTENCE_EN in content:
+            print("status is PENDING")
+            status = ResultEnum.PENDING
+        elif NOT_AVAILABLE_CONTENT in content:
+            print("status is REFUSED")
+            status = ResultEnum.REFUSED
         else:
             print("status is ACCEPTED")
             status = ResultEnum.ACCEPTED
@@ -104,15 +111,28 @@ if __name__ == '__main__':
     # get the list
     params.oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS)
     db_manager = params.firebase_store_manager
+    # collection = db_manager.get_all_successful_items_for_day("2022-05-14", "landd")
     collection = db_manager.get_all_successful_items()
     count = 0
-    task_list = list()
+    # result_pojo = ReserveResultPojo(type=PublishType.SUCCESS, phone="0649614591", email="panleicim@gmail.com",
-    with ThreadPoolExecutor(max_workers=2) as executor:
+    #                                 message="SUCCESS", firstName="Lei", lastName="PAN", url='https://api.ipify.org')
+    result_list = []
     for appointment in collection.stream():
-            count = count + 1
         reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict())
-            if reserve_pojo.accepted:
+        result_list.append(reserve_pojo)
-                print("status is " + reserve_pojo.accepted)
+    # result_list.append(result_pojo)
-            if reserve_pojo.accepted is None or ResultEnum.PENDING.value == reserve_pojo.accepted:
+    # for result in result_list:
-                executor.submit(Worker().run, threading.Event(), reserve_pojo, collection)
+    #     if result.accepted is None or ResultEnum.PENDING.value == result.accepted:
+    #         ResultChecker().run(result, collection)
+    #     else:
+    #         print("status is " + result.accepted)
+
+    with ThreadPoolExecutor(max_workers=5) as executor:
+        for reserve in result_list:
+            count = count + 1
+            if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted:
+                executor.submit(ResultChecker().run, reserve, collection)
+            else:
+                print("status is " + reserve.accepted)
+
         print(count)

config.ini

@@ -1,4 +1,4 @@
 [DEFAULT]
-contact_list_file = .\contact_all.xlsx
+contact_list_file = ./contact_all.xlsx
-firebase_config_file = .\appointment.json
+firebase_config_file = ./appointment.json
-LOGS_DIR = .\
+LOGS_DIR = ./

db/DbManager.py

@@ -1,4 +1,5 @@
 import datetime
+from typing import Union
 
 import firebase_admin
 from firebase_admin import credentials, firestore
@@ -15,6 +16,7 @@ MAIL_COLLECTION_NAME = "mail_list"
 SIM_INFOS = "sim_infos"
 TIMEOUT = "timeout_items"
 
+
 class DataManager:
     batch_size = 20
 
@@ -28,10 +30,12 @@ class DataManager:
         return sim_info_collection
 
     def get_all_successful_items(self):
-        return self.get_all_successful_items_for_day(str(datetime.date.today()))
+        return self.get_all_successful_items_for_day(str(datetime.date.today()), None)
 
-    def get_all_successful_items_for_day(self, day):
+    def get_all_successful_items_for_day(self, day, source_from: Union[str, None]):
         doc_ref = self._db.collection(day)
+        if source_from is not None:
+            doc_ref.where(u'source_from', u'==', source_from)
         return doc_ref
 
     def save_sim_info(self, sim_info: SimInfoPojo):

logs/LogSender.py

@@ -23,6 +23,9 @@ TYPE_EVENT_CHANGE_SLOT = "EVENT_CHANGE_SLOT"
 TYPE_SMS_RECEIVED = "TYPE_SMS_RECEIVED"
 LOG_ERROR = "ERROR"
 LOG_TYPE_INFO = "INFO"
+LOG_ERROR_TYPE_DOUBLE_DATA = "DOUBLE_REQUEST_FOR_SAME_DATA"
+LOG_ERROR_TOO_MANY_REQUEST_TODAY = "TOO_MANY_REQUEST_TODAY"
+LOG_SUBJECT_ERROR = "ERROR"
 LOG_APPOINTMENT_ERROR = "APPOINTMENT_ERROR"
 LOG_APPOINTMENT_TIMEOUT = "TIMEOUT"
 LOG_APPOINTMENT_CONTACT_NOT_FOUND = "CONTACT_NOT_FOUND"
@@ -67,6 +70,14 @@ class LogSender:
         self._identity = oci.identity.IdentityClient(self._config)
         self._loggingingestion_client = LoggingClient(self._config, timeout=60.0, retry_strategy=custom_retry_strategy)
 
+    def send_double_data_error(self, contact: ContactPojo):
+        error_msg = contact.mail
+        self.send_log(msg=error_msg, type=LOG_ERROR_TYPE_DOUBLE_DATA, subject=LOG_SUBJECT_ERROR)
+
+    def send_too_many_error(self, contact: ContactPojo):
+        error_msg = contact.mail
+        self.send_log(msg=error_msg, type=LOG_ERROR_TOO_MANY_REQUEST_TODAY, subject=LOG_SUBJECT_ERROR)
+
     def send_appoint_result(self, result: ReserveResultPojo):
         if result.type == PublishType.SUCCESS:
             # get id

main.py

@@ -42,4 +42,4 @@ def get_proxy(phone_number):
 
 if __name__ == '__main__':
     # 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机，传入1的时候是总店
-    start_book(1, 1, store_choose_state=0)
+    start_book(16, 16, store_choose_state=0)

params.py

@@ -1,4 +1,8 @@
 import configparser
+import random
+import string
+
+from playwright.sync_api import sync_playwright
 
 from db.DbManager import DataManager
 from logs.LogSender import LogSender
@@ -10,6 +14,13 @@ oracle_log_sender = LogSender()
 PROXY_SERVER = "http://gw.ntnt.io:5959"
 PROXY_PASSWORD = "94sY7zwBG13i"
 
+def get_random_id_number_for_proxy() -> str:
+    S = 8  # number of characters in the string.
+    ran = ''.join(random.choices(string.digits, k=S))
+    id_number = str(ran)
+    print("The randomly generated string is : " + str(ran))  # print the random data
+    return id_number
+
 DEVICES = ['iPad (gen 6)', 'iPad (gen 6) landscape', 'iPad (gen 7)', 'iPad (gen 7) landscape', 'iPad Mini',
            'iPad Mini landscape', 'iPad Pro 11', 'iPad Pro 11 landscape', 'iPhone 6', 'iPhone 6 landscape',
            'iPhone 6 Plus', 'iPhone 6 Plus landscape', 'iPhone 7', 'iPhone 7 landscape', 'iPhone 7 Plus',

utils/excel_reader.py

@@ -20,6 +20,7 @@ class ExcelHelper:
         contact_dict_list = json.loads(contact_list_in_json)
         contact_list = []
         for contact_dict in contact_dict_list:
+            if contact_dict['name']:
                 raw_name = contact_dict['name'].strip()
                 name = raw_name.split(' ')
                 last_name = name[0]

workers/commandor_page.py

@@ -22,6 +22,8 @@ OTP_FIELD_ID = "#sms_code"
 MESSAGE_FIELD_CLASS = ".message"
 
 CONFIRMED_MESSAGE = "Your request for a Leather Goods appointment has been registered"
+DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today."
+TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests"
 TIME_OUT = 400000
 OTP_TIMEOUT = 240
 
@@ -76,7 +78,7 @@ class CommandorPage:
         first_page = None
         while first_page is None:
             first_page = self.start_browser(proxy, self.tls.playwright, devices)
-            proxy_username = "panleicim-res-fr-" + get_random_id_number_for_proxy()
+            proxy_username = "panleicim-res-fr-" + params.get_random_id_number_for_proxy()
             self.logger.info("proxy_username is " + proxy_username)
             proxy = {
                 "server": params.PROXY_SERVER,
@@ -152,7 +154,7 @@ class CommandorPage:
         if CONFIRMED_MESSAGE in message:
             # publish the successful message
             self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url)
-        self.getErrors()
+        self.get_errors()
 
     def on_document_loaded(self):
         print("on_document_loaded called")
@@ -191,16 +193,32 @@ class CommandorPage:
         except Exception as error:
             self.logger.error(error)
 
-    def getErrors(self):
+    def get_errors(self):
         # send error result
         self.publish_message_to_queue(self.contact, PublishType.ERROR, self.page.url)
         try:
             items = self.page.query_selector("div.alert")
             if items:
-                print("error is " + items.inner_html())
+                erro_content = items.inner_html()
+                print("错误:" + erro_content)
+                self._handle_errors(erro_content)
         except Exception as ext:
             self.logger.error(ext)
 
+    def _handle_errors(self, erro_content: str):
+        if DOUBLE_REQUEST_ERROR_MESSAGE in erro_content:
+            # this email has been already used
+            params.oracle_log_sender.send_double_data_error(self.contact)
+            #         close browser
+            time.sleep(2)
+            self.browser.close()
+        elif TOO_MANY_REQUEST_ERROR_MESSAGE in erro_content:
+            # this email has been already used
+            params.oracle_log_sender.send_too_many_error(self.contact)
+            #         close browser
+            time.sleep(2)
+            self.browser.close()
+
     def _set_email(self, email):
         time.sleep(get_random_wait_time())
         try:
@@ -266,14 +284,6 @@ class CommandorPage:
             self.browser.close()
 
 
-def get_random_id_number_for_proxy() -> str:
-    S = 8  # number of characters in the string.
-    ran = ''.join(random.choices(string.digits, k=S))
-    id_number = str(ran)
-    print("The randomly generated string is : " + str(ran))  # print the random data
-    return id_number
-
-
 def get_random_id_number() -> str:
     S = 8  # number of characters in the string.
     ran = ''.join(random.choices(string.digits, k=S))
@@ -287,7 +297,7 @@ def on_success(result: ReserveResultPojo):
 
 
 def launch_page():
-    PROXY_USERNAME = "panleicim-res-fr-" + get_random_id_number_for_proxy()
+    PROXY_USERNAME = "panleicim-res-fr-" + params.get_random_id_number_for_proxy()
     print("proxy_username is " + PROXY_USERNAME)
     proxy = {
         "server": PROXY_SERVER,