use brightdata proxy

check_results.py

@@ -76,7 +76,6 @@ class ResultChecker:
         }
         while content is None:
             content = self.load_page(self.tls.playwright, proxy, url)
-            random_id_number = params.get_random_id_number_for_proxy()
             proxy = {
                 "server": params.BRIGHT_DATA_PROXY_SERVER,
                 "username": params.BRIGHT_DATA_PROXY_USERNAME,
@@ -104,10 +103,10 @@ class ResultChecker:
             print("status is ACCEPTED")
             status = ResultEnum.ACCEPTED
             #     send email
-            try:
-                mailer.send_email(get_accepted_result_from(reserve_pojo))
-            except Exception as err:
-                print(err)
+            # try:
+            #     mailer.send_email(get_accepted_result_from(reserve_pojo))
+            # except Exception as err:
+            #     print(err)
         collection.document(reserve_pojo.id).update({u'accepted': status.name})
 
 
@@ -124,10 +123,10 @@ if __name__ == '__main__':
         reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict())
         result_list.append(reserve_pojo)
 
-    with ThreadPoolExecutor(max_workers=10) as executor:
+    with ThreadPoolExecutor(max_workers=20) as executor:
         for reserve in result_list:
             count = count + 1
-            if reserve.accepted is None or ResultEnum.ACCEPTED.value == reserve.accepted:
+            if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted:
                 if reserve.url != BLANK_URL:
                     executor.submit(ResultChecker().run, reserve, collection)
             else:

definitions.py

@@ -3,6 +3,21 @@ import os
 import getpass
 from pathlib import Path
 
+import sqlalchemy as sqlalchemy
+from sqlalchemy_utils import create_database
+from sqlalchemy_utils.functions import database_exists
+
+
+def init_db(path: str):
+    uri_for_db = "sqlite:///{}/{}.db".format(path, "appointment")
+    print(uri_for_db)
+    # 2.-Turn on database engine
+    dbEngine = sqlalchemy.create_engine(uri_for_db)  # ensure this is the correct path for the sqlite file.
+    if not database_exists(uri_for_db):
+        create_database(uri_for_db)
+    dbEngine.connect()
+
+
 home = str(Path.home())
 config = configparser.ConfigParser()
 print("home path: " + home)
@@ -15,4 +30,5 @@ LOGS_DIR = config['DEFAULT']['LOGS_DIR']
 username = getpass.getuser()
 LOG_SOURCE = username
 
+init_db(home)
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))

main.py

@@ -38,9 +38,9 @@ def get_proxy(phone_number, proxy_type=0):
     proxy_username = params.get_proxy_name_prefix(proxy_type) + random_id_number
     logger.info("proxy_username is " + proxy_username)
     proxy = {
-        "server": params.PROXY_SERVER,
-        "username": proxy_username,
-        "password": params.PROXY_PASSWORD
+        "server": params.BRIGHT_DATA_PROXY_SERVER,
+        "username": params.BRIGHT_DATA_PROXY_USERNAME,
+        "password": params.BRIGHT_DATA_PROXY_PASSWORD
     }
     return proxy
 

params.py

@@ -12,7 +12,7 @@ PROXY_SERVER = "http://gw.ntnt.io:5959"
 PROXY_PASSWORD = "94sY7zwBG13i"
 
 BRIGHT_DATA_PROXY_SERVER = "http://zproxy.lum-superproxy.io:22225"
-BRIGHT_DATA_PROXY_USERNAME = " lum-customer-c_daabba94-zone-residential-country-fr"
+BRIGHT_DATA_PROXY_USERNAME = "lum-customer-c_daabba94-zone-residential-country-fr"
 BRIGHT_DATA_PROXY_PASSWORD = "9dwmh54u3bbh"
 PROXY_NAME_PREFIX_RES = "panleicim-res-fr-"
 PROXY_NAME_PREFIX_CC = "panleicim-cc-fr-"

utils/excel_reader.py

@@ -54,6 +54,8 @@ class ExcelHelper:
                 name = raw_name.split(' ')
                 if len(name) == 1:
                     name = raw_name.split('\xa0')
+                if len(name) == 1:
+                    print("error in " + str(name))
                 last_name = name[0]
                 if len(name) == 2:
                     first_name = name[-1]
@@ -85,10 +87,14 @@ def get_random_phone_numbers():
 
 
 def generate_email_from_name(first_name: str, last_name: str) -> str:
-    length = 4  # number of characters in the string.
+    length = 2  # number of characters in the string.
     ran = ''.join(random.choices(string.digits, k=length))
     separator = ['.', '_', '']
-    email = "{}{}{}{}@163.com".format(last_name.lower(), random.choice(separator), first_name.lower(), ran)
+    domains = ['gmail.com', 'hotmail.com', 'yahoo.com', 'aol.com', 'outlook.com', 'hotmail.fr', 'gmx.com',
+               'hotmail.com', 'yahoo.com', 'aol.com', 'hotmail.com']
+    email = "{}{}{}{}@{}".format(last_name.lower(), random.choice(separator),
+                                 first_name.replace("-", "").replace("'", "").lower(), ran,
+                                 random.choice(domains))
     print(email)
     return email
 
@@ -129,6 +135,6 @@ def write_new_contacts_to_excel(valid_contacts: list):
 
 if __name__ == '__main__':
     excel_reader = ExcelHelper()
-    contacts = excel_reader.read_names("C:/Users/landd/Desktop/1000.xlsx")
+    contacts = excel_reader.read_names("C:/Users/landd/Downloads/names.xlsx")
     print(contacts)
     write_new_contacts_to_excel(valid_contacts=contacts)

utils/name_generator.py

@@ -0,0 +1,19 @@
+import json
+
+import requests
+
+
+def generate_names() -> list:
+    res = requests.get(
+        "https://www.namegeneratorfun.com/api/namegenerator?generatorType=list&firstName=&lastName=&minLength=0&maxLength=255&sexId=2&generatorId=43")
+    response_json = json.loads(res.text)
+    names = response_json['names']
+    # print(names)
+    return names
+
+
+if __name__ == '__main__':
+    for i in range(0, 5):
+        names = generate_names()
+        for name in names:
+            print(name)

workers/commandor_page.py

@@ -92,12 +92,10 @@ class CommandorPage:
         first_page = None
         while first_page is None:
             first_page = self.start_browser(proxy, self.tls.playwright, devices)
-            proxy_username = params.get_proxy_name_prefix(self.proxy_type) + params.get_random_id_number_for_proxy()
-            self.logger.info("proxy_username is " + proxy_username)
             proxy = {
-                "server": params.PROXY_SERVER,
-                "username": proxy_username,
-                "password": params.PROXY_PASSWORD
+                "server": params.BRIGHT_DATA_PROXY_SERVER,
+                "username": params.BRIGHT_DATA_PROXY_USERNAME,
+                "password": params.BRIGHT_DATA_PROXY_PASSWORD
             }
         # wait for sms_code field
         # self.clickOnValidBtn()