diff --git a/.gitignore b/.gitignore index 606c743..496accb 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ appointment_*.log ./build ./dist ./contact_all.xlsx +./utils/new_profile_500.xlsx diff --git a/check_results.py b/check_results.py index f423d79..2e2ec2c 100644 --- a/check_results.py +++ b/check_results.py @@ -18,6 +18,7 @@ SORRY_SENTENCE_EN = "we are extremely sorry that we were not able to fulfill" NOT_AVAILABLE_CONTENT = "For more than 130 years, our House has offered its full expertise to satisfy" PENDING_SENTENCE = "Ce soir, entre 20:00 et 20:30, vous obtiendrez une réponse par e-mail." PENDING_SENTENCE_EN = "This evening between 20:00 and 20:30 you will receive a response by email." +BLANK_URL = "about:blank" mailer = Mailer() @@ -69,19 +70,17 @@ class ResultChecker: proxy_username = "panleicim-res-fr-" + random_id_number print("proxy_username is " + proxy_username) proxy = { - "server": params.PROXY_SERVER, - "username": proxy_username, - "password": params.PROXY_PASSWORD + "server": params.BRIGHT_DATA_PROXY_SERVER, + "username": params.BRIGHT_DATA_PROXY_USERNAME, + "password": params.BRIGHT_DATA_PROXY_PASSWORD } while content is None: content = self.load_page(self.tls.playwright, proxy, url) random_id_number = params.get_random_id_number_for_proxy() - proxy_username = "panleicim-res-fr-" + random_id_number - print("proxy_username is " + proxy_username) proxy = { - "server": params.PROXY_SERVER, - "username": proxy_username, - "password": params.PROXY_PASSWORD + "server": params.BRIGHT_DATA_PROXY_SERVER, + "username": params.BRIGHT_DATA_PROXY_USERNAME, + "password": params.BRIGHT_DATA_PROXY_PASSWORD } print(content) self.browser.close() @@ -129,7 +128,8 @@ if __name__ == '__main__': for reserve in result_list: count = count + 1 if reserve.accepted is None or ResultEnum.ACCEPTED.value == reserve.accepted: - executor.submit(ResultChecker().run, reserve, collection) + if reserve.url != BLANK_URL: + executor.submit(ResultChecker().run, reserve, collection) else: print("status is " + reserve.accepted) diff --git a/params.py b/params.py index c3f32f7..b1a9776 100644 --- a/params.py +++ b/params.py @@ -10,11 +10,16 @@ oracle_log_sender = LogSender() # proxy PROXY_SERVER = "http://gw.ntnt.io:5959" PROXY_PASSWORD = "94sY7zwBG13i" + +BRIGHT_DATA_PROXY_SERVER = "http://zproxy.lum-superproxy.io:22225" +BRIGHT_DATA_PROXY_USERNAME = " lum-customer-c_daabba94-zone-residential-country-fr" +BRIGHT_DATA_PROXY_PASSWORD = "9dwmh54u3bbh" PROXY_NAME_PREFIX_RES = "panleicim-res-fr-" PROXY_NAME_PREFIX_CC = "panleicim-cc-fr-" -def get_proxy_name_prefix(proxy_type = 0) -> str: - if proxy_type ==0: + +def get_proxy_name_prefix(proxy_type=0) -> str: + if proxy_type == 0: return PROXY_NAME_PREFIX_RES else: return PROXY_NAME_PREFIX_CC diff --git a/utils/excel_reader.py b/utils/excel_reader.py index df90efc..04a8281 100644 --- a/utils/excel_reader.py +++ b/utils/excel_reader.py @@ -1,9 +1,15 @@ import json +import random +import string import pandas as pandas +import xlsxwriter + from definitions import CONTACT_LIST_FILE from pojo.contact_pojo import ContactPojo +phone_number_prefix = ['6', '7'] + class ExcelHelper: @@ -36,3 +42,93 @@ class ExcelHelper: mail=contact_dict['email']) contact_list.append(contact) return contact_list + + def read_names(self, file_name=CONTACT_LIST_FILE) -> list: + contact_list_in_json = pandas.read_excel(file_name).to_json(orient='records') + contact_dict_list = json.loads(contact_list_in_json) + contact_list = [] + count = 2 + for contact_dict in contact_dict_list: + if contact_dict['name']: + raw_name = contact_dict['name'].strip() + name = raw_name.split(' ') + if len(name) == 1: + name = raw_name.split('\xa0') + last_name = name[0] + if len(name) == 2: + first_name = name[-1] + else: + first_name = ''.join(name[1:len(name)]) + + contact = ContactPojo(phone_number="", + last_name=last_name, + first_name=first_name, + passport_number="", + mail="") + + if len(first_name) == 0: + print("first_name is empty: position:" + str(count)) + print(name) + if len(last_name) == 0: + print("last_name is empty: position:" + str(count)) + count = count + 1 + contact_list.append(contact) + + return contact_list + + +def get_random_phone_numbers(): + length = 8 # number of characters in the string. + ran = ''.join(random.choices(string.digits, k=length)) + id_number = random.choice(phone_number_prefix) + str(ran) + return id_number + + +def generate_email_from_name(first_name: str, last_name: str) -> str: + length = 4 # number of characters in the string. + ran = ''.join(random.choices(string.digits, k=length)) + separator = ['.', '_', ''] + email = "{}{}{}{}@163.com".format(last_name.lower(), random.choice(separator), first_name.lower(), ran) + print(email) + return email + + +def get_random_id_number() -> str: + # write_the_valid_profiles_to_excel() + S = 8 # number of characters in the string. + # call random.choices() string module to find the string in Uppercase + numeric data. + ran = ''.join(random.choices(string.digits, k=S)) + print("The randomly generated string is : 94" + str(ran)) # print the random data + return ran + + +def write_new_contacts_to_excel(valid_contacts: list): + row = 0 + col = 0 + # Create a workbook and add a worksheet. + workbook = xlsxwriter.Workbook('new_profile_{}.xlsx'.format(len(valid_contacts))) + header_data = ['name', 'phone', 'passport', 'email'] + worksheet = workbook.add_worksheet() + header_format = workbook.add_format({'bold': True}) + + for col_num, data in enumerate(header_data): + worksheet.write(row, col_num, data, header_format) + row = row + 1 + for info in valid_contacts: + info.phone = get_random_phone_numbers() + info.passport = get_random_id_number() + info.mail = generate_email_from_name(info.first_name, info.last_name) + # Iterate over the data and write it out row by row. + worksheet.write(row, col, "{} {}".format(info.last_name, info.first_name)) + worksheet.write(row, col + 1, info.phone) + worksheet.write(row, col + 2, info.passport) + worksheet.write(row, col + 3, info.mail) + row += 1 + workbook.close() + + +if __name__ == '__main__': + excel_reader = ExcelHelper() + contacts = excel_reader.read_names("C:/Users/landd/Desktop/1000.xlsx") + print(contacts) + write_new_contacts_to_excel(valid_contacts=contacts) diff --git a/utils/export_valid_profiles.py b/utils/export_valid_profiles.py index 1213e4b..6ef769b 100644 --- a/utils/export_valid_profiles.py +++ b/utils/export_valid_profiles.py @@ -19,32 +19,32 @@ def get_random_id_number() -> str: def write_the_valid_profiles_to_excel(): - day_list = ['2022-03-23','2022-03-24','2022-03-25'] + day_list = ['2022-05-30'] collection = [] for day in day_list: - collection.extend(params.firebase_store_manager.get_all_successful_items_for_day(day).stream()) + collection.extend(params.firebase_store_manager.get_all_successful_items_for_day(day, source_from=None).stream()) valid_contacts = [] # exist_contacts = ExcelHelper().read_contacts() for valid_appointment in collection: reserve_pojo = ReserveResultPojo.from_firestore_dict(valid_appointment.to_dict()) # check whether the contact exists already - exist = [contact for contact in valid_contacts if contact.mail == reserve_pojo.email] - if len(exist) == 0: - contact = ContactPojo(reserve_pojo.phone, passport_number=get_random_id_number(), - last_name=reserve_pojo.lastName, first_name=reserve_pojo.firstName, ccid="", - mail=reserve_pojo.email, position=0) - # seed = 8 # number of characters in the string. - # call random.choices() string module to find the string in Uppercase + numeric data. - # contact.passport = get_random_id_number() - # if contact.passport == None or len(contact.passport) == 0: - # old_contact = [item for item in exist_contacts if item.mail == contact.mail] - # if len(old_contact) > 0: - # contact.passport = old_contact[0].passport + # exist = [contact for contact in valid_contacts if contact.mail == reserve_pojo.email] + # if len(exist) == 0: + contact = ContactPojo(reserve_pojo.phone, passport_number=get_random_id_number(), + last_name=reserve_pojo.lastName, first_name=reserve_pojo.firstName, ccid="", + mail=reserve_pojo.email, position=0) + # seed = 8 # number of characters in the string. + # call random.choices() string module to find the string in Uppercase + numeric data. + # contact.passport = get_random_id_number() + # if contact.passport == None or len(contact.passport) == 0: + # old_contact = [item for item in exist_contacts if item.mail == contact.mail] + # if len(old_contact) > 0: + # contact.passport = old_contact[0].passport - print(contact) - valid_contacts.append(contact) + print(contact) + valid_contacts.append(contact) row = 0 col = 0 diff --git a/utils/generate_random_passport_id.py b/utils/generate_random_passport_id.py index e01ccca..a892600 100644 --- a/utils/generate_random_passport_id.py +++ b/utils/generate_random_passport_id.py @@ -8,8 +8,7 @@ letters = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'P', 'Q', 'R', 'S', def get_random_id_number() -> str: S = 8 # number of characters in the string. ran = ''.join(random.choices(string.digits, k=S)) - id_number = str(ran) - # print("The randomly generated string is : 94" + str(ran)) # print the random data + id_number = str(ran) return id_number diff --git a/workers/commandor_page.py b/workers/commandor_page.py index 1cb8c80..4422d08 100644 --- a/workers/commandor_page.py +++ b/workers/commandor_page.py @@ -15,10 +15,10 @@ from pojo.ReserveResultPojo import ReserveResultPojo, PublishType from pojo.contact_pojo import ContactPojo from workers.SolveCaptch import SolveCaptcha -RDV_URL = "https://rendezvousparis.hermes.com/client/register" +# RDV_URL = "https://rendezvousparis.hermes.com/client/register" # RDV_URL = "file:///Users/lpan/Downloads/test_appointment.html" -# RDV_URL = "https://api.ipify.org" +RDV_URL = "https://api.ipify.org" # RDV_URL ="https://bot.sannysoft.com/" REGEX_RDV_URL = "https:\/\/rendezvousparis\.hermes\.com\/client\/register\/[A-Z0-9]+" otp_value = None @@ -370,9 +370,9 @@ def launch_page(): PROXY_USERNAME = "panleicim-res-fr-" + params.get_random_id_number_for_proxy() print("proxy_username is " + PROXY_USERNAME) proxy = { - "server": PROXY_SERVER, - "username": PROXY_USERNAME, - "password": PROXY_PASSWORD + "server": params.BRIGHT_DATA_PROXY_SERVER, + "username": params.BRIGHT_DATA_PROXY_USERNAME, + "password": params.BRIGHT_DATA_PROXY_PASSWORD } passport_number = get_random_id_number() print("passport_number is " + passport_number) @@ -380,7 +380,7 @@ def launch_page(): first_name="xingzhen", mail="ColbyPatel653@gmail.com", ccid="", position=0) page = CommandorPage(contact, store_type=1) - return page.start_page(None) + return page.start_page(proxy) def wait_for_otp(event: threading.Event, commandor: CommandorPage):