Merge branch 'feature/brightdata_proxy' into feature/auto_with_checkresults

.gitignore

@@ -14,3 +14,4 @@ appointment_*.log
 ./build
 ./dist
 ./contact_all.xlsx
+./utils/new_profile_500.xlsx

check_results.py

@@ -18,6 +18,7 @@ SORRY_SENTENCE_EN = "we are extremely sorry that we were not able to fulfill"
 NOT_AVAILABLE_CONTENT = "For more than 130 years, our House has offered its full expertise to satisfy"
 PENDING_SENTENCE = "Ce soir, entre 20:00 et 20:30, vous obtiendrez une réponse par e-mail."
 PENDING_SENTENCE_EN = "This evening between 20:00 and 20:30 you will receive a response by email."
+BLANK_URL = "about:blank"
 
 mailer = Mailer()
 
@@ -69,19 +70,17 @@ class ResultChecker:
         proxy_username = "panleicim-res-fr-" + random_id_number
         print("proxy_username is " + proxy_username)
         proxy = {
-            "server": params.PROXY_SERVER,
+            "server": params.BRIGHT_DATA_PROXY_SERVER,
-            "username": proxy_username,
+            "username": params.BRIGHT_DATA_PROXY_USERNAME,
-            "password": params.PROXY_PASSWORD
+            "password": params.BRIGHT_DATA_PROXY_PASSWORD
         }
         while content is None:
             content = self.load_page(self.tls.playwright, proxy, url)
             random_id_number = params.get_random_id_number_for_proxy()
-            proxy_username = "panleicim-res-fr-" + random_id_number
-            print("proxy_username is " + proxy_username)
             proxy = {
-                "server": params.PROXY_SERVER,
+                "server": params.BRIGHT_DATA_PROXY_SERVER,
-                "username": proxy_username,
+                "username": params.BRIGHT_DATA_PROXY_USERNAME,
-                "password": params.PROXY_PASSWORD
+                "password": params.BRIGHT_DATA_PROXY_PASSWORD
             }
         print(content)
         self.browser.close()
@@ -129,7 +128,8 @@ if __name__ == '__main__':
         for reserve in result_list:
             count = count + 1
             if reserve.accepted is None or ResultEnum.ACCEPTED.value == reserve.accepted:
-                executor.submit(ResultChecker().run, reserve, collection)
+                if reserve.url != BLANK_URL:
+                    executor.submit(ResultChecker().run, reserve, collection)
             else:
                 print("status is " + reserve.accepted)
 

params.py

@@ -10,11 +10,16 @@ oracle_log_sender = LogSender()
 # proxy
 PROXY_SERVER = "http://gw.ntnt.io:5959"
 PROXY_PASSWORD = "94sY7zwBG13i"
+
+BRIGHT_DATA_PROXY_SERVER = "http://zproxy.lum-superproxy.io:22225"
+BRIGHT_DATA_PROXY_USERNAME = " lum-customer-c_daabba94-zone-residential-country-fr"
+BRIGHT_DATA_PROXY_PASSWORD = "9dwmh54u3bbh"
 PROXY_NAME_PREFIX_RES = "panleicim-res-fr-"
 PROXY_NAME_PREFIX_CC = "panleicim-cc-fr-"
 
-def get_proxy_name_prefix(proxy_type = 0) -> str:
+
-    if proxy_type ==0:
+def get_proxy_name_prefix(proxy_type=0) -> str:
+    if proxy_type == 0:
         return PROXY_NAME_PREFIX_RES
     else:
         return PROXY_NAME_PREFIX_CC

utils/excel_reader.py

@@ -1,9 +1,15 @@
 import json
+import random
+import string
 
 import pandas as pandas
+import xlsxwriter
+
 from definitions import CONTACT_LIST_FILE
 from pojo.contact_pojo import ContactPojo
 
+phone_number_prefix = ['6', '7']
+
 
 class ExcelHelper:
 
@@ -36,3 +42,93 @@ class ExcelHelper:
                                       mail=contact_dict['email'])
                 contact_list.append(contact)
         return contact_list
+
+    def read_names(self, file_name=CONTACT_LIST_FILE) -> list:
+        contact_list_in_json = pandas.read_excel(file_name).to_json(orient='records')
+        contact_dict_list = json.loads(contact_list_in_json)
+        contact_list = []
+        count = 2
+        for contact_dict in contact_dict_list:
+            if contact_dict['name']:
+                raw_name = contact_dict['name'].strip()
+                name = raw_name.split(' ')
+                if len(name) == 1:
+                    name = raw_name.split('\xa0')
+                last_name = name[0]
+                if len(name) == 2:
+                    first_name = name[-1]
+                else:
+                    first_name = ''.join(name[1:len(name)])
+
+                contact = ContactPojo(phone_number="",
+                                      last_name=last_name,
+                                      first_name=first_name,
+                                      passport_number="",
+                                      mail="")
+
+                if len(first_name) == 0:
+                    print("first_name is empty: position:" + str(count))
+                    print(name)
+                if len(last_name) == 0:
+                    print("last_name is empty: position:" + str(count))
+                count = count + 1
+                contact_list.append(contact)
+
+        return contact_list
+
+
+def get_random_phone_numbers():
+    length = 8  # number of characters in the string.
+    ran = ''.join(random.choices(string.digits, k=length))
+    id_number = random.choice(phone_number_prefix) + str(ran)
+    return id_number
+
+
+def generate_email_from_name(first_name: str, last_name: str) -> str:
+    length = 4  # number of characters in the string.
+    ran = ''.join(random.choices(string.digits, k=length))
+    separator = ['.', '_', '']
+    email = "{}{}{}{}@163.com".format(last_name.lower(), random.choice(separator), first_name.lower(), ran)
+    print(email)
+    return email
+
+
+def get_random_id_number() -> str:
+    # write_the_valid_profiles_to_excel()
+    S = 8  # number of characters in the string.
+    # call random.choices() string module to find the string in Uppercase + numeric data.
+    ran = ''.join(random.choices(string.digits, k=S))
+    print("The randomly generated string is : 94" + str(ran))  # print the random data
+    return ran
+
+
+def write_new_contacts_to_excel(valid_contacts: list):
+    row = 0
+    col = 0
+    # Create a workbook and add a worksheet.
+    workbook = xlsxwriter.Workbook('new_profile_{}.xlsx'.format(len(valid_contacts)))
+    header_data = ['name', 'phone', 'passport', 'email']
+    worksheet = workbook.add_worksheet()
+    header_format = workbook.add_format({'bold': True})
+
+    for col_num, data in enumerate(header_data):
+        worksheet.write(row, col_num, data, header_format)
+    row = row + 1
+    for info in valid_contacts:
+        info.phone = get_random_phone_numbers()
+        info.passport = get_random_id_number()
+        info.mail = generate_email_from_name(info.first_name, info.last_name)
+        # Iterate over the data and write it out row by row.
+        worksheet.write(row, col, "{} {}".format(info.last_name, info.first_name))
+        worksheet.write(row, col + 1, info.phone)
+        worksheet.write(row, col + 2, info.passport)
+        worksheet.write(row, col + 3, info.mail)
+        row += 1
+    workbook.close()
+
+
+if __name__ == '__main__':
+    excel_reader = ExcelHelper()
+    contacts = excel_reader.read_names("C:/Users/landd/Desktop/1000.xlsx")
+    print(contacts)
+    write_new_contacts_to_excel(valid_contacts=contacts)

utils/export_valid_profiles.py

@@ -19,32 +19,32 @@ def get_random_id_number() -> str:
 
 
 def write_the_valid_profiles_to_excel():
-    day_list = ['2022-03-23','2022-03-24','2022-03-25']
+    day_list = ['2022-05-30']
 
     collection = []
     for day in day_list:
-        collection.extend(params.firebase_store_manager.get_all_successful_items_for_day(day).stream())
+        collection.extend(params.firebase_store_manager.get_all_successful_items_for_day(day, source_from=None).stream())
 
     valid_contacts = []
     # exist_contacts = ExcelHelper().read_contacts()
     for valid_appointment in collection:
         reserve_pojo = ReserveResultPojo.from_firestore_dict(valid_appointment.to_dict())
         # check whether the contact exists already
-        exist = [contact for contact in valid_contacts if contact.mail == reserve_pojo.email]
+        # exist = [contact for contact in valid_contacts if contact.mail == reserve_pojo.email]
-        if len(exist) == 0:
+        # if len(exist) == 0:
-            contact = ContactPojo(reserve_pojo.phone, passport_number=get_random_id_number(),
+        contact = ContactPojo(reserve_pojo.phone, passport_number=get_random_id_number(),
-                                  last_name=reserve_pojo.lastName, first_name=reserve_pojo.firstName, ccid="",
+                              last_name=reserve_pojo.lastName, first_name=reserve_pojo.firstName, ccid="",
-                                  mail=reserve_pojo.email, position=0)
+                              mail=reserve_pojo.email, position=0)
-            # seed = 8  # number of characters in the string.
+        # seed = 8  # number of characters in the string.
-            # call random.choices() string module to find the string in Uppercase + numeric data.
+        # call random.choices() string module to find the string in Uppercase + numeric data.
-            # contact.passport = get_random_id_number()
+        # contact.passport = get_random_id_number()
-            # if contact.passport == None or len(contact.passport) == 0:
+        # if contact.passport == None or len(contact.passport) == 0:
-            #     old_contact = [item for item in exist_contacts if item.mail == contact.mail]
+        #     old_contact = [item for item in exist_contacts if item.mail == contact.mail]
-            #     if len(old_contact) > 0:
+        #     if len(old_contact) > 0:
-            #         contact.passport = old_contact[0].passport
+        #         contact.passport = old_contact[0].passport
 
-            print(contact)
+        print(contact)
-            valid_contacts.append(contact)
+        valid_contacts.append(contact)
 
     row = 0
     col = 0

utils/generate_random_passport_id.py

@@ -8,8 +8,7 @@ letters = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'P', 'Q', 'R', 'S',
 def get_random_id_number() -> str:
     S = 8  # number of characters in the string.
     ran = ''.join(random.choices(string.digits, k=S))
-    id_number =  str(ran)
+    id_number = str(ran)
-    # print("The randomly generated string is : 94" + str(ran))  # print the random data
     return id_number
 
 

workers/commandor_page.py

@@ -15,10 +15,10 @@ from pojo.ReserveResultPojo import ReserveResultPojo, PublishType
 from pojo.contact_pojo import ContactPojo
 from workers.SolveCaptch import SolveCaptcha
 
-RDV_URL = "https://rendezvousparis.hermes.com/client/register"
+# RDV_URL = "https://rendezvousparis.hermes.com/client/register"
 
 # RDV_URL = "file:///Users/lpan/Downloads/test_appointment.html"
-# RDV_URL = "https://api.ipify.org"
+RDV_URL = "https://api.ipify.org"
 # RDV_URL ="https://bot.sannysoft.com/"
 REGEX_RDV_URL = "https:\/\/rendezvousparis\.hermes\.com\/client\/register\/[A-Z0-9]+"
 otp_value = None
@@ -370,9 +370,9 @@ def launch_page():
     PROXY_USERNAME = "panleicim-res-fr-" + params.get_random_id_number_for_proxy()
     print("proxy_username is " + PROXY_USERNAME)
     proxy = {
-        "server": PROXY_SERVER,
+        "server": params.BRIGHT_DATA_PROXY_SERVER,
-        "username": PROXY_USERNAME,
+        "username": params.BRIGHT_DATA_PROXY_USERNAME,
-        "password": PROXY_PASSWORD
+        "password": params.BRIGHT_DATA_PROXY_PASSWORD
     }
     passport_number = get_random_id_number()
     print("passport_number is " + passport_number)
@@ -380,7 +380,7 @@ def launch_page():
                           first_name="xingzhen",
                           mail="ColbyPatel653@gmail.com", ccid="", position=0)
     page = CommandorPage(contact, store_type=1)
-    return page.start_page(None)
+    return page.start_page(proxy)
 
 
 def wait_for_otp(event: threading.Event, commandor: CommandorPage):