Merge branch 'master' into feature/check_results

# Conflicts:
#	src/db/mongo_manager.py

.gitignore

@@ -13,5 +13,7 @@ appointment_*.log
 venv
 docs
 ./dist
+dist
 ./contact_all.xlsx
 ./utils/new_profile_500.xlsx
+.DS_Store

appointment.py

@@ -1,9 +1,8 @@
 import PySimpleGUI as sg
 
-# First the window layout in 2 columns
 from main import start_book, recheck_the_captcha_error_contacts
-from pojo.ModeEnum import ModeEnum
+from src.pojo.ModeEnum import ModeEnum
-from proxy.proxy_type import ProxyType
+from src.proxy.proxy_type import ProxyType
 
 KEY_CHOOSE_STORE = "CHOOSE_STORE"
 KEY_START_NUMBER = "KEY_START_NUMBER"
@@ -71,7 +70,9 @@ while True:
     print(event)
     print(values)
     # try:
-    max_workers = int(values[KEY_MAX_WORKERS])
+    max_workers = 10
+    if values[KEY_MAX_WORKERS]:
+        max_workers = int(values[KEY_MAX_WORKERS])
     store_type = 0
     mode = ModeEnum.MANUAL
     if values[KEY_FAUBOURG]:

appointment_mac.spec

@@ -48,7 +48,5 @@ coll = COLLECT(
     upx_exclude=[],
     name='appointment',
 )
-import shutil
+#import shutil
-shutil.copyfile('./contact_all.xlsx', '{0}/contact_all.xlsx'.format(DISTPATH))
+#shutil.copytree('./venv/lib/python3.8/site-packages/grpc/_cython/_credentials', '{0}/appointment/grpc/_cython/_credentials'.format(DISTPATH))
-shutil.copyfile('./config.ini', '{0}/config.ini'.format(DISTPATH))
-shutil.copytree('./venv/lib/python3.8/site-packages/grpc/_cython/_credentials', '{0}/appointment/grpc/_cython/_credentials'.format(DISTPATH))

build.sh

@@ -0,0 +1,16 @@
+dest_dir=dist/appointment
+if [[ -d $dest_dir ]]
+then
+    echo "$dest_dir exists on your filesystem."
+else
+  mkdir -p $dest_dir
+fi
+if [[ "$OSTYPE" == "msys" ]]
+then
+./venv/Scripts/pyarmor.exe obfuscate --recursive --output $dest_dir/src src/__init__.py
+else
+  pyarmor obfuscate --recursive --output $dest_dir/src src/__init__.py
+fi
+cp main.py $dest_dir
+cp appointment.py $dest_dir
+cp requirements.txt $dest_dir

main.py

@@ -2,13 +2,13 @@ import logging
 import sys
 from concurrent.futures import ThreadPoolExecutor
 
-import params
+from src import params
-from logs.AppLogging import init_logger
+from src.logs.AppLogging import init_logger
-from pojo.ModeEnum import ModeEnum
+from src.pojo.ModeEnum import ModeEnum
-from pojo.contact_pojo import ContactPojo
+from src.pojo.contact_pojo import ContactPojo
-from proxy.proxy_type import ProxyType
+from src.proxy.proxy_type import ProxyType
-from utils.excel_reader import ExcelHelper
+from src.utils.excel_reader import ExcelHelper
-from workers.commandor_page import CommandorPage
+from src.workers.commandor_page import CommandorPage
 
 # used to save the current slot position
 init_logger()

requirements.txt

@@ -1,15 +1,16 @@
 dataclasses_json==0.5.7
 firebase_admin==5.2.0
 pandas~=1.3.5
-playwright==1.21.0
+playwright==1.23.0
 dataclasses~=0.6
+pymongo==4.1.1
 oci~=2.54.1
 XlsxWriter~=3.0.3
 boto3~=1.21.13
 openpyxl==3.0.9
 google-cloud-firestore==2.4.0
-PySimpleGUI==4.60.0
+PySimpleGUI==4.60.1
-SpeechRecognition==3.8.1
 SQLAlchemy~=1.4.37
 requests~=2.27.1
+sqlalchemy_utils~=0.38.2
 Mako~=1.2.0

src/db/DbManager.py

@@ -4,13 +4,12 @@ from typing import Union
 import firebase_admin
 from firebase_admin import credentials, firestore
 
-import definitions
+from src import definitions, params
-import params
+from src.pojo.MailPojo import MailPojo
-from pojo import ResultEnum
+from src.pojo.ReserveResultPojo import ReserveResultPojo
-from pojo.MailPojo import MailPojo
+from src.pojo.ResultEnum import ResultEnum
-from pojo.ReserveResultPojo import ReserveResultPojo
+from src.pojo.SimInfoPojo import SimInfoPojo
-from pojo.SimInfoPojo import SimInfoPojo
+from src.pojo.contact_pojo import ContactPojo
-from pojo.contact_pojo import ContactPojo
 
 ERROR_COLLECTION_NAME = "error_items"
 CONTACT_COLLECTION_NAME = "contact_list"

src/db/local_db_manager.py

@@ -5,9 +5,8 @@ from sqlalchemy import MetaData, Column, String, Integer, DateTime, Table
 from sqlalchemy.orm import Session
 from sqlalchemy_utils import database_exists, create_database
 
-import params
+from src.pojo.ReserveResultPojo import ReserveResultPojo
-from pojo.ReserveResultPojo import ReserveResultPojo
+from src.pojo.captcha_error_contact_pojo import ContactInErrorPojo
-from pojo.captcha_error_contact_pojo import ContactInErrorPojo, ERROR_TYPE_CAPTCHA
 
 
 class LocalDbManager:

src/db/mongo_manager.py

@@ -3,13 +3,13 @@ import logging
 
 from pymongo import MongoClient
 
-import params
+from src.pojo.ReserveResultPojo import ReserveResultPojo
-from pojo.ReserveResultPojo import ReserveResultPojo
+from src.pojo.black_contact import BlackContactPojo
-from pojo.ResultEnum import ResultEnum
+from src.pojo.contact_pojo import ContactPojo
-from pojo.contact_pojo import ContactPojo
 
 MONGO_DB_URL = "mongo.lpaconsulting.fr"
 CAPTCHA_ERROR_COLLECTION_PREFIX = "CAPTCHA_ERROR_"
+BLACK_LIST = "BLACK_LIST"
 
 
 class MongoDbManager:
@@ -40,6 +40,18 @@ class MongoDbManager:
         except Exception as error:
             self.logger.info(error)
 
+    def insert_blacklist_contact(self, contact: ContactPojo):
+        collection_name = BLACK_LIST
+        black_contact = BlackContactPojo(contact.phone, contact.passport, contact.last_name, contact.first_name,
+                                         contact.mail)
+        try:
+            collection_to_use = self.db[collection_name]
+            collection_to_use.replace_one(filter={'_id': black_contact.mail, },
+                                          replacement=black_contact.to_firestore_dict(),
+                                          upsert=True)
+        except Exception as error:
+            self.logger.info(error)
+
     def get_all_successful_items_for_day(self) -> list:
         params.oracle_log_sender.send_read_db_event("get_all_successful_items_for_day")
         collection_name = str(datetime.date.today())
@@ -85,4 +97,6 @@ class MongoDbManager:
 
 if __name__ == '__main__':
     db_manager = MongoDbManager()
-    db_manager.update_reserve_result("welcome", ResultEnum.ACCEPTED)
+    contact = ContactPojo(phone_number='0755667750', passport_number='123456789', last_name='PAN', first_name='Lei',
+                          mail='panleicim@gmail.com')
+    print(db_manager.insert_blacklist_contact(contact))

src/logs/AppLogging.py

@@ -1,7 +1,7 @@
 import datetime
 import logging
 
-from definitions import LOGS_DIR
+from src.definitions import LOGS_DIR
 
 
 def init_logger():

src/logs/LogSender.py

@@ -6,12 +6,10 @@ import oci
 from oci.loggingingestion import LoggingClient
 from oci.loggingingestion.models import PutLogsDetails, LogEntryBatch, LogEntry
 
-import definitions
+from src import definitions
-from logs.AppLogging import init_logger
+from src.logs.AppLogging import init_logger
-from pojo import ReserveResultPojo
+from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType
-from pojo.ReserveResultPojo import PublishType
+from src.pojo.contact_pojo import ContactPojo
-# Log subjects
-from pojo.contact_pojo import ContactPojo
 
 LOG_SUBJECT_EVENT = "EVENT"
 LOG_SUBJECT_SMS = "SMS"

src/notification/mailer.py

@@ -5,7 +5,7 @@ from mako.template import Template
 import boto3
 
 import definitions
-from notification.AcceptedResultPojo import AcceptedResultPojo
+from src.notification import AcceptedResultPojo
 
 TEMPLATE_NAME = 'scrapy_template_email'
 AWS_CLIENT_NAME = "sesv2"
@@ -26,8 +26,8 @@ class Mailer:
                                        aws_secret_access_key=secret)
 
     def send_email(self, result: AcceptedResultPojo):
-        # recipients = ['panleicim@gmail.com', 'kamenonly@gmail.com', 'tangliang0411@gmail.com']
+        recipients = ['panleicim@gmail.com', 'kamenonly@gmail.com', 'tangliang0411@gmail.com']
-        recipients = ['panleicim@gmail.com']
+        # recipients = ['panleicim@gmail.com']
 
         mytemplate = Template(filename=definitions.ROOT_DIR + "/templates/appointment_results.html")
         self.logger.info("send email to " + str(recipients))

src/params.py

@@ -1,12 +1,12 @@
 import random
 import string
 
-import definitions
+from src import definitions
-from db.DbManager import DataManager
+from src.db.DbManager import DataManager
-from db.local_db_manager import LocalDbManager
+from src.db.local_db_manager import LocalDbManager
-from db.mongo_manager import MongoDbManager
+from src.db.mongo_manager import MongoDbManager
-from logs.LogSender import LogSender
+from src.logs.LogSender import LogSender
-from proxy.proxy_type import ProxyType
+from src.proxy.proxy_type import ProxyType
 
 firebase_store_manager = DataManager()
 oracle_log_sender = LogSender()

src/pojo/ReserveResultPojo.py

@@ -4,7 +4,7 @@ from typing import Union
 
 from dataclasses_json import dataclass_json
 
-import definitions
+from src import definitions
 
 
 class PublishType(Enum):

src/pojo/black_contact.py

@@ -0,0 +1,38 @@
+import time
+from typing import Union
+
+from src.pojo.contact_pojo import ContactPojo
+
+
+class BlackContactPojo(ContactPojo):
+    update_at_in_s: float
+
+    def __init__(self, phone_number: str, passport_number: str, last_name: str, first_name: str, mail: str,
+                 update_at_in_s: Union[float, None] = None,
+                 ccid: str = "",
+                 position: int = 0, ):
+        super().__init__(phone_number, passport_number, last_name, first_name, mail, ccid, position)
+        if update_at_in_s:
+            self.update_at_in_s = update_at_in_s
+        else:
+            self.update_at_in_s = time.time()
+
+    def to_firestore_dict(self):
+        dest = super().to_firestore_dict()
+        dest.setdefault(u'update_at_in_s', self.update_at_in_s)
+        return dest
+
+    @staticmethod
+    def from_firestore_dict(source):
+        ccid = source['ccid']
+        phone = source['phone']
+        position = source['position']
+        passport = source['passport']
+        email = source['mail']
+        last_name = source['last_name']
+        first_name = source['first_name']
+        update_at_in_s = source['update_at_in_s']
+        result = BlackContactPojo(ccid=ccid, phone_number=phone, passport_number=passport, position=position,
+                                  mail=email, update_at_in_s=update_at_in_s,
+                                  last_name=last_name, first_name=first_name)
+        return result

src/pojo/contact_pojo.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 
-from pojo.captcha_error_contact_pojo import ContactInErrorPojo
+from src.pojo.captcha_error_contact_pojo import ContactInErrorPojo
 
 
 @dataclass
@@ -50,8 +50,8 @@ class ContactPojo:
         position = source['position']
         passport = source['passport']
         email = source['mail']
-        lastName = source['last_name']
+        last_name = source['last_name']
-        firstName = source['last_name']
+        first_name = source['first_name']
         result = ContactPojo(ccid=ccid, phone_number=phone, passport_number=passport, position=position, mail=email,
-                             last_name=lastName, first_name=firstName)
+                             last_name=last_name, first_name=first_name)
         return result

src/utils/excel_reader.py

@@ -5,9 +5,9 @@ import string
 import pandas as pandas
 import xlsxwriter
 
-from definitions import CONTACT_LIST_FILE
+from src.definitions import CONTACT_LIST_FILE
-from pojo.contact_pojo import ContactPojo
+from src.pojo.contact_pojo import ContactPojo
-from utils.generate_random_passport_id import get_random_passport_id_number
+from src.utils.generate_random_passport_id import get_random_passport_id_number
 
 phone_number_prefix = ['6']
 

src/utils/generate_random_passport_id.py

@@ -4,8 +4,8 @@ import string
 
 # letters = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'P', 'Q', 'R', 'S', 'T', '1', '2', '3', '4', '5', '6', '7',
 #            '8', '9']
-from pojo.captcha_error_contact_pojo import ContactInErrorPojo
+from src.pojo.captcha_error_contact_pojo import ContactInErrorPojo
-from pojo.contact_pojo import ContactPojo
+from src.pojo.contact_pojo import ContactPojo
 
 letters = ['E', 'G', 'M']
 

src/workers/commandor_page.py

@@ -1,21 +1,22 @@
 import logging
 import random
 import re
-import string
+import sys
 import threading
 import time
+import traceback
 from typing import Union
 
 from playwright.sync_api import sync_playwright
-import params
+
-from pojo.ModeEnum import ModeEnum
+from src import params
-from pojo.ReserveResultPojo import ReserveResultPojo, PublishType
+from src.pojo.ModeEnum import ModeEnum
-from pojo.captcha_error_contact_pojo import ERROR_TYPE_CAPTCHA, \
+from src.pojo.ReserveResultPojo import ReserveResultPojo, PublishType
-    TOO_MANY_REQUEST_ERROR
+from src.pojo.captcha_error_contact_pojo import TOO_MANY_REQUEST_ERROR, ERROR_TYPE_CAPTCHA
-from pojo.contact_pojo import ContactPojo
+from src.pojo.contact_pojo import ContactPojo
-from proxy.proxy_type import ProxyType
+from src.proxy.proxy_type import ProxyType
-from utils.generate_random_passport_id import get_captcha_error_contact_from_contact
+from src.utils.generate_random_passport_id import get_captcha_error_contact_from_contact
-from workers.SolveCaptch import SolveCaptcha
+from src.workers.SolveCaptch import SolveCaptcha
 
 RDV_URL = "https://rendezvousparis.hermes.com/client/register"
 
@@ -35,7 +36,7 @@ TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests"
 TOO_MANY_REQUEST_ERROR_MESSAGE_FR = "Suite à un trop grand nombre de demandes"
 CAPTCHA_ERROR_MESSAGE = "Error verifying captcha, please try again"
 CAPTCHA_ERROR_MESSAGE_FR = "La vérification du captcha a échoué"
-TIME_OUT = 10*60*1000 #10 mins
+TIME_OUT = 10 * 60 * 1000  # 10 mins
 OTP_TIMEOUT = 240
 PAGE_TIMEOUT = 40000
 
@@ -87,7 +88,6 @@ class CommandorPage:
     def timeout_occurred(self):
         params.oracle_log_sender.send_timeout_log(self.contact)
         self.logger.info("will close timeout modem")
-        self.thread_event.set()
         self.termine()
 
     def _run(self, e: threading.Event, proxy):
@@ -100,26 +100,26 @@ class CommandorPage:
         while first_page is None:
             first_page = self.start_browser(proxy, self.tls.playwright, devices)
             proxy = params.get_proxy(self.proxy_type)
-        self.thread_event = e
+        # self.thread_event = e
         otp_input = self.page.locator(OTP_FIELD_ID)
         otp_input.wait_for(state='visible', timeout=TIME_OUT)
-        event_is_set = e.wait()
+        # event_is_set = e.wait()
-        logging.info('event set: %s', event_is_set)
+        # logging.info('event set: %s', event_is_set)
-        if self.otp_value:
+        # if self.otp_value:
-            self.fill_otp(self.otp_value)
+        #     self.fill_otp(self.otp_value)
-            time.sleep(get_random_wait_time())
+        #     time.sleep(get_random_wait_time())
-            self.clickOnValidBtn()
+        #     self.clickOnValidBtn()
-            otp_sent = self.page.locator(MESSAGE_FIELD_CLASS)
+        #     otp_sent = self.page.locator(MESSAGE_FIELD_CLASS)
-            otp_sent.wait_for(state='visible', timeout=TIME_OUT)
+        #     otp_sent.wait_for(state='visible', timeout=TIME_OUT)
-        time.sleep(get_random_wait_time())
+        # time.sleep(get_random_wait_time())
-        message = self.page.content()
+        # message = self.page.content()
-        if CONFIRMED_MESSAGE in message or CONFIRMED_MESSAGE_FR in message:
+        # if CONFIRMED_MESSAGE in message or CONFIRMED_MESSAGE_FR in message:
-            # publish the successful message
+        #     # publish the successful message
-            self.logger.info("url is " + self.page.url)
+        #     self.logger.info("url is " + self.page.url)
-            self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url)
+        #     self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url)
-        else:
+        # else:
-            self.logger.info("timeout")
+        self.logger.info("timeout")
-            self.termine()
+        self.termine()
 
     def fill_fields(self):
         if not self.is_filling_fields:
@@ -139,8 +139,8 @@ class CommandorPage:
         try:
             self.browser = pwright.webkit.launch(headless=False, timeout=PAGE_TIMEOUT, proxy=proxy)
             self.logger.info("模拟设备: " + device)
-            pixel_2 = pwright.devices[device]
+            simulated_mobile = pwright.devices[device]
-            context = self.browser.new_context(**pixel_2, locale='fr-FR')
+            context = self.browser.new_context(**simulated_mobile, locale='fr-FR')
             self.page = context.new_page()
             # hide webdriver information
             self.page.add_init_script("""() => {
@@ -157,6 +157,7 @@ class CommandorPage:
             return self.page.content()
         except Exception as error:
             params.oracle_log_sender.send_error(str(error))
+            traceback.print_exc(*sys.exc_info())
             self.logger.exception(error)
             self.logger.info("will close browser")
             self.browser.close()
@@ -174,7 +175,6 @@ class CommandorPage:
 
     def _on_page_loaded(self):
         self.logger.info("页面加载完毕")
-        # self.logger.info("content is " + self.page.content())
         self.logger.info("url is " + self.page.url)
         if self.page.url == RDV_URL:
             self.fill_fields()
@@ -252,11 +252,12 @@ class CommandorPage:
             self.is_finished = True
             self.termine()
         elif TOO_MANY_REQUEST_ERROR_MESSAGE in erro_content or TOO_MANY_REQUEST_ERROR_MESSAGE_FR in erro_content:
-            # this email has been already used
+            # this email is in black list
             if not self.is_finished:
                 params.local_db_manager.insert_or_update(
                     get_captcha_error_contact_from_contact(self.contact, TOO_MANY_REQUEST_ERROR))
                 params.oracle_log_sender.send_too_many_error(self.contact)
+                params.mongo_store_manager.insert_blacklist_contact(self.contact)
             self.is_finished = True
             self.termine()
         elif CAPTCHA_ERROR_MESSAGE in erro_content or CAPTCHA_ERROR_MESSAGE_FR in erro_content:

start.bat

@@ -1 +0,0 @@
-.\venv\Scripts\python.exe appointment.py

utils/operator.py

@@ -1,15 +0,0 @@
-from enum import Enum
-
-
-class Operator(Enum):
-    SFR = "SFR"
-    LYCAMOBILE = "LYCAMOBILE"
-    CHINA_TELECOM = "CHINA_TELECOM"
-
-
-def check_operator(ccid: str) -> Operator:
-    if "893313" in ccid:
-        return Operator.LYCAMOBILE
-    elif "893310" in ccid:
-        return Operator.SFR
-    return Operator.CHINA_TELECOM