diff --git a/check_results.py b/check_results.py index 884a2a7..274ea31 100644 --- a/check_results.py +++ b/check_results.py @@ -1,14 +1,20 @@ +import random from enum import Enum +from typing import Union from playwright.sync_api import sync_playwright import params +from commandor_page import get_random_id_number_for_proxy from logs.LogSender import TYPE_EVENT_CHECK_RESULTS, LOG_SUBJECT_EVENT from pojo.ReserveResultPojo import ReserveResultPojo +from utils.excel_reader import ExcelHelper SORRY_SENTENCE = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci" PENDING_SENTENCE = "Ce soir, entre 20:00 et 20:30, vous obtiendrez une réponse par e-mail." +user_agent_list = ExcelHelper().read_user_agent_list() + class ResultEnum(Enum): ACCEPTED = "ACCEPTED" @@ -23,27 +29,47 @@ def check_result_page(url) -> ResultEnum: def run(playwright, url) -> ResultEnum: - browser = playwright.webkit.launch(headless=False) - page = browser.new_page( - user_agent="Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.87 Mobile Safari/537.36") + browser = playwright.firefox.launch(headless=False) url_to_check = url.replace("register/", "") url_to_check = url_to_check + "?lang=fr" - print(url_to_check) - page.goto(url_to_check) - content = page.content() + content = None + while content is None: + content = load_page(browser, url_to_check) print(content) browser.close() if SORRY_SENTENCE in content: - print("result is REFUSED") + print("status is REFUSED") return ResultEnum.REFUSED elif PENDING_SENTENCE in content: - print("result is PENDING") + print("status is PENDING") return ResultEnum.PENDING else: - print("result is ACCEPTED") + print("status is ACCEPTED") return ResultEnum.ACCEPTED +def load_page(browser, url) -> Union[str, None]: + try: + PROXY_USERNAME = "panleicim-cc-fr-sid-" + get_random_id_number_for_proxy() + proxy = { + "server": params.PROXY_SERVER, + "username": PROXY_USERNAME, + "password": params.PROXY_PASSWORD + } + firefox_user_agents = filter(lambda user_agent: "firefox" in user_agent.lower(), user_agent_list) + firefox_user_agents_list = list(firefox_user_agents) + user_agent = random.choice(firefox_user_agents_list) + page = browser.new_page( + user_agent=user_agent, + proxy=proxy) + page.add_init_script("""() => Object.defineProperty(navigator,'webdriver',{get: () => undefined}""") + page.goto(url, timeout=90000) + return page.content() + except Exception as error: + print(error) + return None + + # need to start at 21h00 if __name__ == '__main__': # get the list @@ -56,7 +82,8 @@ if __name__ == '__main__': reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) if reserve_pojo.accepted: print("status is " + reserve_pojo.accepted) - if reserve_pojo.accepted is None or ResultEnum.PENDING.value == reserve_pojo.accepted: + if reserve_pojo.accepted is None: + # or ResultEnum.PENDING.value == reserve_pojo.accepted: result = check_result_page(reserve_pojo.url) collection.document(reserve_pojo.id).update({u'accepted': result.name}) print(count) diff --git a/commandor_page.py b/commandor_page.py new file mode 100644 index 0000000..08138ca --- /dev/null +++ b/commandor_page.py @@ -0,0 +1,200 @@ +import logging +import random +import string +import threading +import time + +from playwright.sync_api import sync_playwright + +from params import PROXY_SERVER, PROXY_PASSWORD +from pojo.ReserveResultPojo import ReserveResultPojo, PublishType +from pojo.contact_pojo import ContactPojo +from utils.excel_reader import ExcelHelper + +RDV_URL = "https://rendezvousparis.hermes.com/client/register" + +# RDV_URL = "file:///Users/lpan/Downloads/test_appointment.html" + +otp_value = None +user_agent_list = ExcelHelper().read_user_agent_list() +OTP_FIELD_ID = "#sms_code" +MESSAGE_FIELD_CLASS = ".message" + +CONFIRMED_MESSAGE = "Your request for a Leather Goods appointment has been registered" + + +class CommandorPage: + def __init__(self): + self.otp_value = None + self.logger = logging.getLogger("CommandorPage") + + def _run(self, e: threading.Event, proxy, contact: ContactPojo, on_ready_for_otp, on_success): + self.contact = contact + self.on_success_listener = on_success + with sync_playwright() as pwright: + firefox_user_agents = filter(lambda user_agent: "firefox" in user_agent.lower(), user_agent_list) + firefox_user_agents_list = list(firefox_user_agents) + user_agent = random.choice(firefox_user_agents_list) + # if "chrome" in user_agent.lower(): + # self.browser = pwright.chromium.launch(headless=False, timeout=90000, proxy=proxy) + # elif "firefox" in user_agent.lower(): + # self.browser = pwright.firefox.launch(headless=False, timeout=90000, proxy=proxy) + # else: + self.start_brower(proxy, pwright, user_agent) + self._setPhoneCountryAndStore() + self._setPhoneNumber(contact.phone) + self._setName(contact.last_name, contact.first_name) + self._set_email(contact.mail) + self.setIdNumber(contact.passport) + self._checkCgu() + # wait for sms_code field + self.clickOnValidBtn() + # self.page.goto("file:///Users/lpan/Downloads/input_otp.html") + otp_input = self.page.locator(OTP_FIELD_ID) + otp_input.wait_for(state='visible',timeout=90000) + on_ready_for_otp(e, self) + event_is_set = e.wait() + logging.info('event set: %s', event_is_set) + if self.otp_value: + self.fill_otp(self.otp_value) + self.clickOnValidBtn() + otp_sent = self.page.locator(MESSAGE_FIELD_CLASS) + otp_sent.wait_for(state='visible') + message = self.page.content() + print("message is:" + message) + time.sleep(2) + if CONFIRMED_MESSAGE in message: + # publish the successful message + print("url is " + self.page.url) + self.publish_message_to_queue(contact, PublishType.SUCCESS.value, self.page.url) + else: + print("timeout") + self.reset_air_plan_mode() + # check and send successful event + + def start_brower(self, proxy, pwright, user_agent): + try: + self.browser = pwright.firefox.launch(headless=False, timeout=90000, proxy=proxy) + self.logger.info("user_agent is " + user_agent) + self.page = self.browser.new_page( + user_agent=user_agent) + # hide webdriver information + self.page.add_init_script("""() => Object.defineProperty(navigator,'webdriver',{get: () => undefined}""") + self.page.on("load", self._on_page_loaded) + self.page.goto(RDV_URL, timeout=90000) + except Exception as error: + self.logger.exception(error) + + def start_page(self, proxy, contact: ContactPojo, on_ready_for_otp, on_sucess) -> threading.Event: + e = threading.Event() + t = threading.Thread(target=self._run, args=(e, proxy, contact, on_ready_for_otp, on_sucess)) + t.start() + return e + + def _on_page_loaded(self): + print("page loaded") + print("content is " + self.page.content()) + print("url is " + self.page.url) + self.getErrors() + + def on_document_loaded(self): + print("on_document_loaded called") + + def _setPhoneCountryAndStore(self): + # document.getElementById("prefer").value = \"faubourg\"; + self.page.evaluate("""()=>{ + document.getElementById("phone_country").value = \"FR\" }""") + + def _setPhoneNumber(self, phoneNumber): + self.page.evaluate("""(phoneNumber)=>document.getElementById("phone_number").value =phoneNumber""", + phoneNumber) + + def _setName(self, lastName, firstName): + self.page.evaluate("""(name)=> { + document.getElementById("surname").value = name.lastName; + document.getElementById("name").value = name.firstName}""", {'lastName': lastName, 'firstName': firstName}) + + def getErrors(self): + items = self.page.query_selector("div.alert") + if items: + print(items.inner_html()) + + def _set_email(self, email): + self.page.evaluate("""(email)=>document.getElementById("email").value = email""", email) + + def setIdNumber(self, id): + self.page.evaluate(""" (id) => document.getElementById("passport_id").value = id""", id) + + def _checkCgu(self): + self.page.evaluate("""document.getElementById("cgu").checked = true; + document.getElementById("processing").checked = true""") + + def clickOnValidBtn(self): + self.page.evaluate("""document.getElementsByClassName("btn")[0].click();""") + + def clear_app_data(self): + pass + + def fill_otp(self, otp: str): + self.page.fill(OTP_FIELD_ID, otp) + + def reset_air_plan_mode(self): + print("will close browser") + self.browser.close() + + def publish_message_to_queue(self, contact: ContactPojo, message: str, url: str): + # create the message + id = url.split("/")[-1] + result = ReserveResultPojo(type=PublishType.SUCCESS, phone=contact.phone, message=message, url=url, + firstName=contact.first_name, lastName=contact.last_name, email=contact.mail, + passport=contact.passport, ccid=contact.ccid) + result.id = id + self.on_success_listener(result) + + +def get_random_id_number_for_proxy() -> str: + S = 8 # number of characters in the string. + ran = ''.join(random.choices(string.digits, k=S)) + id_number = str(ran) + print("The randomly generated string is : " + str(ran)) # print the random data + return id_number + + +def get_random_id_number() -> str: + S = 8 # number of characters in the string. + ran = ''.join(random.choices(string.digits, k=S)) + id_number = "57" + str(ran) + print("The randomly generated string is : 94" + str(ran)) # print the random data + return id_number + + +def on_success(result: ReserveResultPojo): + pass + + +def launch_page(ready_for_otp) -> threading.Event: + PROXY_USERNAME = "panleicim-cc-fr-sid-" + get_random_id_number_for_proxy() + print("proxy_username is " + PROXY_USERNAME) + proxy = { + "server": PROXY_SERVER, + "username": PROXY_USERNAME, + "password": PROXY_PASSWORD + } + passport_number = get_random_id_number() + print("passport_number is " + passport_number) + contact = ContactPojo(phone_number="+33758912245", passport_number=passport_number, last_name="XU", + first_name="xingzhen", + mail="ColbyPatel653@gmail.com", ccid="", position=0) + page = CommandorPage() + return page.start_page(proxy, contact, ready_for_otp, on_sucess=on_success) + + +def wait_for_otp(event: threading.Event, commandor: CommandorPage): + sec = input("Press Enter otp to continue...\n") + print("input otp is: " + sec) + commandor.otp_value = sec + event.set() + + +if __name__ == '__main__': + event = launch_page(wait_for_otp) diff --git a/contact.xlsx b/contact.xlsx index 9a365f8..5be6c6f 100644 Binary files a/contact.xlsx and b/contact.xlsx differ diff --git a/db/.~sim_infos.xlsx b/db/.~sim_infos.xlsx new file mode 100644 index 0000000..4c491c6 Binary files /dev/null and b/db/.~sim_infos.xlsx differ diff --git a/db/DbManager.py b/db/DbManager.py index bf60153..fc90aab 100644 --- a/db/DbManager.py +++ b/db/DbManager.py @@ -10,6 +10,7 @@ from pojo.ReserveResultPojo import ReserveResultPojo, PublishType from pojo.SimInfoPojo import SimInfoPojo from pojo.contact_pojo import ContactPojo from utils.excel_reader import ExcelHelper +from utils.operator import Operator ERROR_COLLECTION_NAME = "error_items" CONTACT_COLLECTION_NAME = "contact_list" @@ -107,9 +108,13 @@ class DataManager: worksheet = workbook.add_worksheet() for info in sim_info_list: # Iterate over the data and write it out row by row. - worksheet.write(row, col, info.phone[2:len(info.phone)]) + if info.operator == Operator.LYCAMOBILE.value: + worksheet.write(row, col, info.phone[2:len(info.phone)]) + else: + worksheet.write(row, col, info.phone) worksheet.write(row, col + 1, info.ccid) - worksheet.write(row, col + 2, info.position) + worksheet.write(row, col + 2, info.operator) + worksheet.write(row, col + 3, info.position) row += 1 workbook.close() diff --git a/docs/mobile_user_agent_list.xlsx b/docs/mobile_user_agent_list.xlsx new file mode 100644 index 0000000..8f2e604 Binary files /dev/null and b/docs/mobile_user_agent_list.xlsx differ diff --git a/export_valid_profiles.py b/export_valid_profiles.py index 36fe297..1213e4b 100644 --- a/export_valid_profiles.py +++ b/export_valid_profiles.py @@ -6,21 +6,20 @@ import xlsxwriter import params from pojo.ReserveResultPojo import ReserveResultPojo from pojo.contact_pojo import ContactPojo -from utils.excel_reader import ExcelHelper -def get_random_id_number ()-> str: + +def get_random_id_number() -> str: # write_the_valid_profiles_to_excel() S = 8 # number of characters in the string. # call random.choices() string module to find the string in Uppercase + numeric data. - ran = ''.join(random.choices(string.digits, k = S)) - id_number = "57"+str(ran) - print("The randomly generated string is : 94" + str(ran)) # print the random data + ran = ''.join(random.choices(string.digits, k=S)) + id_number = "57" + str(ran) + print("The randomly generated string is : 94" + str(ran)) # print the random data return id_number def write_the_valid_profiles_to_excel(): - day_list = ['2022-03-04', '2022-03-07', '2022-03-08', '2022-03-09', '2022-03-10', '2022-03-11', '2022-03-14', - '2022-03-15', '2022-03-16'] + day_list = ['2022-03-23','2022-03-24','2022-03-25'] collection = [] for day in day_list: @@ -36,6 +35,8 @@ def write_the_valid_profiles_to_excel(): contact = ContactPojo(reserve_pojo.phone, passport_number=get_random_id_number(), last_name=reserve_pojo.lastName, first_name=reserve_pojo.firstName, ccid="", mail=reserve_pojo.email, position=0) + # seed = 8 # number of characters in the string. + # call random.choices() string module to find the string in Uppercase + numeric data. # contact.passport = get_random_id_number() # if contact.passport == None or len(contact.passport) == 0: # old_contact = [item for item in exist_contacts if item.mail == contact.mail] @@ -67,7 +68,6 @@ def write_the_valid_profiles_to_excel(): workbook.close() - if __name__ == '__main__': # get_random_id_number() - write_the_valid_profiles_to_excel() \ No newline at end of file + write_the_valid_profiles_to_excel() diff --git a/logs/LogSender.py b/logs/LogSender.py index 660343a..76ef3da 100644 --- a/logs/LogSender.py +++ b/logs/LogSender.py @@ -11,6 +11,8 @@ from pojo import ReserveResultPojo from pojo.ReserveResultPojo import PublishType # Log subjects +from pojo.serial_modem import SerialModem + LOG_SUBJECT_EVENT = "EVENT" LOG_SUBJECT_SMS = "SMS" SUBJECT_SIM_INFO = "sim_card" @@ -23,6 +25,7 @@ LOG_ERROR = "ERROR" LOG_TYPE_INFO = "INFO" LOG_APPOINTMENT_ERROR = "APPOINTMENT_ERROR" LOG_APPOINTMENT_TIMEOUT = "TIMEOUT" +LOG_APPOINTMENT_CONTACT_NOT_FOUND = "CONTACT_NOT_FOUND" LOG_APPOINTMENT_SUCCESS = "SUCCESS" @@ -63,6 +66,14 @@ class LogSender: msg = "phone:{}, sms:{}".format(phone, sms_text) self.send_log(msg=msg, subject=LOG_SUBJECT_SMS, type=TYPE_SMS_RECEIVED) + def send_timeout_log(self, serial_modem: SerialModem): + msg = "phone:{}, ccid:{}".format(serial_modem.phone_number, serial_modem.ccid) + self.send_log(msg, type=LOG_APPOINTMENT_TIMEOUT) + + def send_contact_not_found(self, msg: str): + self.send_log(msg, subject=SUBJECT_SIM_INFO, type=LOG_APPOINTMENT_CONTACT_NOT_FOUND) + + if __name__ == '__main__': init_logger() logger = logging.getLogger() diff --git a/main.py b/main.py index 68a4758..5496924 100644 --- a/main.py +++ b/main.py @@ -3,14 +3,14 @@ import logging import re import sys import time +from threading import Event from typing import Union from gsmmodem import GsmModem import params -from commandor import Commandor +from commandor_page import CommandorPage, get_random_id_number_for_proxy from logs.AppLogging import init_logger -from logs.LogSender import LOG_APPOINTMENT_TIMEOUT from modems.ModemPool import ModemPool from modems.card_pool import CardPool from params import MODEM_POOL_PORTS, CARD_POOL_PORT, firebase_store_manager, oracle_log_sender @@ -18,11 +18,12 @@ from pojo.ReserveResultPojo import ReserveResultPojo from pojo.serial_modem import SerialModem from utils.excel_reader import ExcelHelper from utils.message_receiver import MessageReceiver +from utils.operator import check_operator, Operator -OTP_TIMEOUT = 600 +OTP_TIMEOUT = 180 is_finished = False -commandor = Commandor() -contacts = [] +commandor = CommandorPage() +thread_event = None current_gsm_modem = None card_pool = CardPool(CARD_POOL_PORT) # used to save the current slot position @@ -30,10 +31,6 @@ current_card_pool_slot = 1 current_sim_position = 1 -def get_devices_ports() -> list: - return MODEM_POOL_PORTS - - def send_command(cmd: str, ser, wait_time_in_s: int = 0) -> bytes: ser.write(cmd.encode()) msg = ser.read(100) @@ -59,8 +56,10 @@ def create_modem_for_port(port: str) -> Union[SerialModem, None]: def timeout_occurred(serial_modem: SerialModem): firebase_store_manager.save_timeout_contact(serial_modem.contact) - oracle_log_sender.send_log(str(serial_modem.phone_number), type=LOG_APPOINTMENT_TIMEOUT) + oracle_log_sender.send_timeout_log(serial_modem) logger.info("will close timeout modem") + global thread_event + thread_event.set() serial_modem.modem.close() commandor.reset_air_plan_mode() @@ -68,7 +67,9 @@ def timeout_occurred(serial_modem: SerialModem): def start_to_handle_sms(serial_modem: SerialModem): global current_gsm_modem current_gsm_modem = serial_modem.modem - current_gsm_modem.deleteMultipleStoredSms(memory="SM") + if check_operator(serial_modem.ccid) == Operator.LYCAMOBILE: + # lycamobile + current_gsm_modem.deleteMultipleStoredSms(memory="SM") serial_modem.modem.smsReceivedCallback = handle_sms global is_finished is_finished = False @@ -96,14 +97,19 @@ def handle_sms(sms): params.oracle_log_sender.send_sms_reception_log(sms.number, sms.text) if date == str(datetime.date.today()): logger.info("this sms is for today") - if "rendez-vous" in sms.text: + if "rendez-vous" in sms.text or "appointment" in sms.text: logger.info("try to extract the otp") pattern = r'\d{6,8}' # if re.match(pattern, sms.text): match = re.search(pattern, sms.text) otp = match.group(0) logger.info("otp is " + otp) - commandor.send_otp(otp) + global thread_event + global commandor + commandor.otp_value = otp + logger.info("will set thread event") + thread_event.set() + # commandor.send_otp(otp) # wait for the sms for 20 seconds global is_finished while not is_finished: @@ -121,7 +127,7 @@ def handle_sms(sms): def init_modems() -> list: modems = [] - for port in get_devices_ports(): + for port in params.MODEM_POOL_PORTS: serial_modem = create_modem_for_port(port) if serial_modem: modems.append(serial_modem) @@ -143,58 +149,90 @@ def on_message_received(ch, method, properties, body): is_finished = True +def on_success(result: ReserveResultPojo): + logger.info("on_success called.") + result.sim_position = current_sim_position + result.slot_position = current_card_pool_slot + logger.info(result) + firebase_store_manager.save(result) + oracle_log_sender.send_appoint_result(result) + # set the flag to True + global is_finished + is_finished = True + + def start_listen(): logger.info("start to listen to message queue") receiver = MessageReceiver() receiver.start_listener(on_message_received) +def on_ready_for_otp(e: Event, commandor: CommandorPage): + logger.info("on_ready_for_otp() called.") + global thread_event + thread_event = e + + def start_book(): - slot_number = 1 - slot_sum = 22 - for i in range(slot_number, slot_sum + 1): + start_slot_number = 1 + end_slot_number = 32 + for i in range(start_slot_number, end_slot_number + 1): card_pool.reset() logger.info("will switch to " + str(i)) + global current_card_pool_slot + current_card_pool_slot = i card_pool.switch_to_slot(i) - modem_pool = ModemPool(get_devices_ports()) + modem_pool = ModemPool(MODEM_POOL_PORTS) modem_pool.reset_all_modems() modem_list = init_modems() - # create listeners for chaque modem # read the contact, and contact the 2 objects together excel_reader = ExcelHelper() - global contacts contacts = excel_reader.read_contacts() global current_sim_position - current_sim_position = 1 + current_sim_position = 0 for modem in modem_list: current_sim_position = current_sim_position + 1 try: - # get contact for current modem modem.get_ccid() # find the contact with ccid - contact = [contact for contact in contacts if contact.ccid == modem.ccid] + contact = [contact for contact in contacts if + contact.ccid.replace("F", "") == modem.ccid.replace("F", "")] if len(contact) > 0: modem.phone_number = contact[0].phone modem.contact = contact[0] else: - logger.info("contact not found for this ccid") + logger.info("contact not found for this ccid:{}".format(modem.ccid)) + error_msg = "slot({}):sim({}):ccid({})".format(i, current_sim_position, modem.ccid) + oracle_log_sender.send_contact_not_found(error_msg) modem.modem.close() continue if modem.contact: logger.info("contact found for this ccid") signal = modem.modem.signalStrength logger.info("信号强度: " + str(signal)) - commandor.start_page(modem.contact) + proxy = get_proxy(modem.phone_number) + commandor.start_page(proxy=proxy, contact=modem.contact, + on_ready_for_otp=on_ready_for_otp, on_sucess=on_success) start_to_handle_sms(modem) except Exception as error: print(error) continue +def get_proxy(phone_number): + random_id_number = str(phone_number)[1:len(str(phone_number))] + proxy_username = "panleicim-cc-fr-sid-" + random_id_number + logger.info("proxy_username is " + proxy_username) + proxy = { + "server": params.PROXY_SERVER, + "username": proxy_username, + "password": params.PROXY_PASSWORD + } + return proxy + + if __name__ == '__main__': init_logger() logger = logging.getLogger() logger.addHandler(logging.StreamHandler(stream=sys.stdout)) - start_listen() start_book() - # card_pool.switch_to_slot(12) diff --git a/modems/ModemPool.py b/modems/ModemPool.py index 3edeab3..adc3f36 100644 --- a/modems/ModemPool.py +++ b/modems/ModemPool.py @@ -2,7 +2,6 @@ import re import time import serial -from serial import Serial from definitions import BAUDRATE from error.SIMError import SIMError @@ -10,6 +9,7 @@ from logs.LogSender import LOG_APPOINTMENT_SUCCESS, SUBJECT_SIM_INFO from params import firebase_store_manager, oracle_log_sender from pojo.SimInfoPojo import SimInfoPojo from utils.excel_reader import ExcelHelper +from utils.operator import check_operator, Operator class ModemPool: @@ -43,42 +43,59 @@ class ModemPool: return msg def get_raw_phone_number(self, slot_position): + for index, ser in enumerate(self._serial_list): - print("will get phone number for slot({}) SIM({}), port:{}".format(slot_position, index + 1, ser.port)) - if not self._select_sim_storage(ser): - print(self._generate_error_msg(slot_position, index, SIMError.STORAGE_ERROR)) - continue - msg = self._execut_USSD_cmd("AT+CUSD=1, *132#\r", ser) - if "Unfortunately" in str(msg): - print(self._generate_error_msg(slot_position, index, SIMError.SIM_DISABLED)) - continue - elif "CME ERROR" in str(msg): - print(self._generate_error_msg(slot_position, index, SIMError.CME_ERROR)) - continue - elif len(msg) == 0: - print(self._generate_error_msg(slot_position, index, SIMError.TIMEOUT)) - continue - # find phone number - match = re.search(r'33\d{9}', str(msg)) - phone_number = match.group(0) - print("phone is " + phone_number) + sim_position = index + 1 + position = (slot_position - 1) * len(self._port_list) + sim_position + # unlock sim + unlock_cmd = 'AT+CPIN="{0}\r"'.format("0000") + self._send_command(unlock_cmd, ser, 10) cmd = "AT+CCID\r" response = str(self._send_command(cmd, ser)) ccid_group = re.search("[0-9F]+", response) ccid = ccid_group.group(0) - sim_position = index + 1 - position = (slot_position - 1) * 15 + sim_position - if phone_number: - self._db_manager.save_sim_info(SimInfoPojo(phone=phone_number, ccid=ccid, position=position)) - self._log_sender.send_log(phone_number, source=self.TAG, subject=SUBJECT_SIM_INFO, - type=LOG_APPOINTMENT_SUCCESS) - # write the number to sim card's phonebook - cmd = f'AT+CPBW={self.phone_number_position},\"{phone_number}\"\r' - self._send_command(cmd, ser, wait_time_in_s=2) - self.get_own_number(ser) + operator = check_operator(ccid) + if operator == Operator.SFR or operator == Operator.CHINA_TELECOM: + contacts = self._excel_helper.read_contacts() + contact = [contact for contact in contacts if + contact.ccid.replace("F", "") == ccid.replace("F", "")] + if len(contact) > 0: + phone_number = contact[0].phone + self._db_manager.save_sim_info( + SimInfoPojo(phone=str(phone_number), ccid=ccid, position=position, sim_position=sim_position, + slot_position=slot_position, operator=operator.value)) + else: + error_msg = "slot({}),sim({})".format(slot_position, sim_position) + oracle_log_sender.send_contact_not_found(error_msg) + else: + print("will get phone number for slot({}) SIM({}), port:{}".format(slot_position, index + 1, ser.port)) + if not self._select_sim_storage(ser): + print(self._generate_error_msg(slot_position, index, SIMError.STORAGE_ERROR)) + continue + msg = self._execut_USSD_cmd("AT+CUSD=1, *132#\r", ser) + if "Unfortunately" in str(msg): + print(self._generate_error_msg(slot_position, index, SIMError.SIM_DISABLED)) + continue + elif "CME ERROR" in str(msg): + print(self._generate_error_msg(slot_position, index, SIMError.CME_ERROR)) + continue + elif len(msg) == 0: + print(self._generate_error_msg(slot_position, index, SIMError.TIMEOUT)) + continue + # find phone number + match = re.search(r'33\d{9}', str(msg)) + phone_number = match.group(0) + print("phone is " + phone_number) - def get_own_number(self, ser: Serial): - print("saved phone number: " + str(self._send_command(f'AT+CPBR={self.phone_number_position}\r', ser))) + if phone_number: + self._db_manager.save_sim_info( + SimInfoPojo(phone=phone_number, ccid=ccid, position=position, slot_position=slot_position, + sim_position=sim_position, operator=operator.value)) + self._log_sender.send_log(phone_number, source=self.TAG, subject=SUBJECT_SIM_INFO, + type=LOG_APPOINTMENT_SUCCESS) + # write the number to sim card's phonebook + # cmd = f'AT+CPBW={self.phone_number_position},\"{phone_number}\"\r' + # self._send_command(cmd, ser, wait_time_in_s=2) def _select_sim_storage(self, ser) -> bool: # use SIM Card storage diff --git a/params.py b/params.py index 88a71f1..37c8c6b 100644 --- a/params.py +++ b/params.py @@ -1,3 +1,4 @@ +import pika from db.DbManager import DataManager from logs.LogSender import LogSender @@ -25,3 +26,15 @@ CARD_POOL_PORT = "/dev/tty.usbmodem1432101" firebase_store_manager = DataManager() oracle_log_sender = LogSender() + +# proxy +PROXY_SERVER = "http://gw.ntnt.io:5959" +PROXY_USERNAME = "panleicim-cc-fr-sid-192012" +PROXY_PASSWORD = "M3PZAXgW5V27" + +# rabittmq +rabittMQ_host = "rabbitmq.lpaconsulting.fr" +rabittMQ_port = 6672 +_credentials = pika.PlainCredentials('scrapy_rabbitmq', '4x!hReCbA5v3heKWfPJV-Y') +rabittmq_connection = pika.BlockingConnection( + pika.ConnectionParameters(host='rabbitmq.lpaconsulting.fr', port=6672, credentials=_credentials)) \ No newline at end of file diff --git a/pojo/ReserveResultPojo.py b/pojo/ReserveResultPojo.py index ecee8c6..3f6fee5 100644 --- a/pojo/ReserveResultPojo.py +++ b/pojo/ReserveResultPojo.py @@ -23,6 +23,7 @@ class ReserveResultPojo: passport: str = "" slot_position = None sim_position = None + ccid: str = "" @staticmethod def from_firestore_dict(source): @@ -49,6 +50,9 @@ class ReserveResultPojo: if 'passport' in source: passport = source['passport'] result.passport = passport + if 'ccid' in source: + ccid = source['ccid'] + result.ccid = ccid result.id = id return result @@ -64,7 +68,8 @@ class ReserveResultPojo: u'passport': self.passport, u'url': self.url, u'sim_position': self.sim_position, - u'slot_position': self.slot_position + u'slot_position': self.slot_position, + u'ccid': self.ccid } return dest diff --git a/pojo/SimInfoPojo.py b/pojo/SimInfoPojo.py index 075ac0d..fa2b573 100644 --- a/pojo/SimInfoPojo.py +++ b/pojo/SimInfoPojo.py @@ -1,4 +1,3 @@ -import datetime import time @@ -7,24 +6,37 @@ class SimInfoPojo: ccid: str update_at: int position: int + operator: str + slot_position: str + sim_position: str - def __init__(self, phone: str, ccid: str, position, update_at: int = int(time.time())): + def __init__(self, phone: str, ccid: str, position, operator: str, slot_position, sim_position, + update_at: int = int(time.time())): self.phone = phone self.ccid = ccid self.update_at = update_at self.position = position + self.operator = operator + self.slot_position = slot_position + self.sim_position = sim_position @staticmethod def from_firestore_dict(source): phone = source['phone'] ccid = source['ccid'] + sim_position = source['sim_position'] + slot_position = source['slot_position'] update_at = None if 'update_at' in source: update_at = source['update_at'] position = None if 'position' in source: position = source['position'] - result = SimInfoPojo(phone=phone, ccid=ccid, update_at=update_at, position=position) + result = SimInfoPojo(phone=phone, ccid=ccid, update_at=update_at, sim_position=sim_position, + slot_position=slot_position, position=position, operator="") + if 'operator' in source: + operator = source['operator'] + result.operator = operator result.id = id return result @@ -34,6 +46,9 @@ class SimInfoPojo: u'ccid': self.ccid, u'update_at': self.update_at, u'position': self.position, + u'operator': self.operator, + u'slot_position': self.slot_position, + u'sim_position': self.sim_position, } return dest diff --git a/pojo/serial_modem.py b/pojo/serial_modem.py index 81448df..f639638 100644 --- a/pojo/serial_modem.py +++ b/pojo/serial_modem.py @@ -1,3 +1,4 @@ +import logging from dataclasses import dataclass from gsmmodem import GsmModem @@ -13,14 +14,15 @@ class SerialModem(): def __init__(self, modem: GsmModem, ccid: str = None): self.modem = modem self.ccid = ccid + self.logger = logging.getLogger("SerialModem") def get_ccid(self): cmd = "AT+CCID" self.modem.connect("0000") - print("try to get ccid") + self.logger.info("try to get ccid") response = self.modem.write(cmd, True) self.ccid = response[0].split(" ")[1].replace("\"", "") - print("The SIM card ccid is:" + self.ccid) + self.logger.info("The SIM card ccid is:" + self.ccid) # print("try to get phone number") # cmd_phone_number = "AT+CPBS=ON" # openBook = self.modem.write(cmd_phone_number, True) diff --git a/reset_all_sim_card.py b/reset_all_sim_card.py index b12ead4..bc896a7 100644 --- a/reset_all_sim_card.py +++ b/reset_all_sim_card.py @@ -1,23 +1,23 @@ import logging +import sys import params from modems.ModemPool import ModemPool from logs.AppLogging import init_logger from logs.LogSender import LOG_SUBJECT_EVENT, TYPE_EVENT_RESET_ALL_SIM_CARDS -from main import card_pool, get_devices_ports +from main import card_pool def read_all_the_phone_number(): params.oracle_log_sender.send_log(msg="SIM卡自检开始", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_RESET_ALL_SIM_CARDS) - slot_number = 1 - slot_sum = 23 - # card_pool.switch_to_slot(29) + start_slot_number = 1 + end_slot_number = 32 params.firebase_store_manager.clear_all_sim_info() - for i in range(slot_number, slot_sum + 1): + for i in range(start_slot_number, end_slot_number + 1): card_pool.reset() logger.info("will switch to " + str(i)) card_pool.switch_to_slot(i) - modem_pool = ModemPool(get_devices_ports()) + modem_pool = ModemPool(params.MODEM_POOL_PORTS) modem_pool.reset_all_modems() modem_pool.get_raw_phone_number(i) @@ -25,4 +25,5 @@ def read_all_the_phone_number(): if __name__ == '__main__': init_logger() logger = logging.getLogger() + logger.addHandler(logging.StreamHandler(stream=sys.stdout)) read_all_the_phone_number() diff --git a/utils/excel_reader.py b/utils/excel_reader.py index 9175977..847b05b 100644 --- a/utils/excel_reader.py +++ b/utils/excel_reader.py @@ -2,6 +2,7 @@ import json import pandas as pandas +import definitions from pojo.contact_pojo import ContactPojo @@ -24,7 +25,17 @@ class ExcelHelper: self.write_to_exel("ccid_list.xlsx", ccids.split(",")) print(lines) - # read the contact list from the exel file + def read_user_agent_list(self): + # read the contact list from the exel file + contact_list_in_json = pandas.read_excel(definitions.ROOT_DIR + "/docs/mobile_user_agent_list.xlsx").to_json( + orient='records') + contact_dict_list = json.loads(contact_list_in_json) + user_agents = [] + for contact_dict in contact_dict_list: + user_agent = contact_dict['user_agent'] + user_agents.append(user_agent) + return user_agents + def read_contacts(self) -> list: contact_list_in_json = pandas.read_excel(r'./contact.xlsx').to_json(orient='records') contact_dict_list = json.loads(contact_list_in_json) @@ -46,4 +57,5 @@ class ExcelHelper: if __name__ == '__main__': helper = ExcelHelper() - helper.generate_exel_from_txt() + # helper.generate_exel_from_txt() + print(helper.read_user_agent_list()) diff --git a/utils/message_receiver.py b/utils/message_receiver.py index 3c4368c..f026a27 100644 --- a/utils/message_receiver.py +++ b/utils/message_receiver.py @@ -1,24 +1,19 @@ -import logging import threading -from datetime import datetime -import pika +from params import rabittmq_connection APPOINTMENT_QUEUE = "APPOINTMENT_QUEUE" class MessageReceiver: - def __init__(self): - self._credentials = pika.PlainCredentials('scrapy_rabbitmq', '4x!hReCbA5v3heKWfPJV-Y') def start_listener(self, callback): t = threading.Thread(target=self._run, args=(callback,)) t.start() def _run(self, callback): - connection = pika.BlockingConnection( - pika.ConnectionParameters(host='rabbitmq.lpaconsulting.fr', port=6672, credentials=self._credentials)) - channel = connection.channel() + + channel = rabittmq_connection.channel() channel.queue_declare(queue=APPOINTMENT_QUEUE) channel.basic_consume(queue=APPOINTMENT_QUEUE, auto_ack=True, diff --git a/utils/operator.py b/utils/operator.py new file mode 100644 index 0000000..9f9e0fe --- /dev/null +++ b/utils/operator.py @@ -0,0 +1,15 @@ +from enum import Enum + + +class Operator(Enum): + SFR = "SFR" + LYCAMOBILE = "LYCAMOBILE" + CHINA_TELECOM = "CHINA_TELECOM" + + +def check_operator(ccid: str) -> Operator: + if "893313" in ccid: + return Operator.LYCAMOBILE + elif "893310" in ccid: + return Operator.SFR + return Operator.CHINA_TELECOM diff --git a/utils/utils.py b/utils/utils.py new file mode 100644 index 0000000..e69de29 diff --git a/valid_profile.xlsx b/valid_profile.xlsx index 781212e..4b9fac9 100644 Binary files a/valid_profile.xlsx and b/valid_profile.xlsx differ diff --git a/wait_for_sms.py b/wait_for_sms.py new file mode 100644 index 0000000..fa2196a --- /dev/null +++ b/wait_for_sms.py @@ -0,0 +1,118 @@ +import logging +import sys +import time +from typing import Union + +from gsmmodem import GsmModem + +import params +from commandor_page import CommandorPage +from logs.AppLogging import init_logger +from modems.ModemPool import ModemPool +from modems.card_pool import CardPool +from params import MODEM_POOL_PORTS, CARD_POOL_PORT +from pojo.serial_modem import SerialModem +from utils.excel_reader import ExcelHelper + +OTP_TIMEOUT = 40 +commandor = CommandorPage() +thread_event = None +current_gsm_modem = None +card_pool = CardPool(CARD_POOL_PORT) + + +def get_devices_ports() -> list: + return MODEM_POOL_PORTS + + +def create_modem_for_port(port: str) -> Union[SerialModem, None]: + logger.info('Initializing modem... for ' + port) + serial_modem = None + try: + modem = GsmModem(port) + return SerialModem(modem=modem) + except Exception as ext: + logger.error(ext) + return serial_modem + + +def timeout_occurred(serial_modem: SerialModem): + logger.info("will close timeout modem") + serial_modem.modem.close() + commandor.reset_air_plan_mode() + + +def start_to_handle_sms(serial_modem: SerialModem): + global current_gsm_modem + current_gsm_modem = serial_modem.modem + try: + current_gsm_modem.deleteMultipleStoredSms(memory="SM") + except Exception as error: + print(error) + + serial_modem.modem.smsReceivedCallback = handle_sms + serial_modem.modem.smsTextMode = False + logger.info('Waiting for SMS message, for phone number ' + str(serial_modem.phone_number)) + listen_at = time.time() + while True: + time.sleep(2) + # check whether timeout + now = time.time() + if (listen_at + OTP_TIMEOUT) < now: + logger.info("time out for {}, switch to next contact".format(serial_modem.phone_number)) + # save the contact in timeout + timeout_occurred(serial_modem) + current_gsm_modem.close() + return + + +def handle_sms(sms): + logger.info( + u'== SMS message received ==\nFrom: {0}\nTime: {1}\nMessage:\n{2}\n'.format(sms.number, sms.time, sms.text)) + # extract the otp number + params.oracle_log_sender.send_sms_reception_log(sms.number, sms.text) + + +def init_modems() -> list: + modems = [] + for port in get_devices_ports(): + serial_modem = create_modem_for_port(port) + if serial_modem: + modems.append(serial_modem) + return modems + + +def start_waiting_sms(): + # logger = logging.getLogger() + slot_number = 12 + slot_sum = 32 + for i in range(slot_number, slot_sum + 1): + card_pool.reset() + logger.info("will switch to " + str(i)) + card_pool.switch_to_slot(i) + modem_pool = ModemPool(get_devices_ports()) + modem_pool.reset_all_modems() + modem_list = init_modems() + # read the contact, and merge the 2 objects together + excel_reader = ExcelHelper() + contacts = excel_reader.read_contacts() + for modem in modem_list: + try: + # get contact for current modem + modem.get_ccid() + contact = [contact for contact in contacts if + contact.ccid.replace("F", "") == modem.ccid.replace("F", "")] + if len(contact) > 0: + modem.phone_number = contact[0].phone + modem.contact = contact[0] + start_to_handle_sms(modem) + except Exception as error: + print(error) + continue + + +if __name__ == '__main__': + init_logger() + logger = logging.getLogger() + logger.addHandler(logging.StreamHandler(stream=sys.stdout)) + start_waiting_sms()