Merge branch 'feature/multi-threads'

check_results.py

@@ -1,7 +1,11 @@
+import asyncio
 import random
+import threading
+from concurrent.futures import ThreadPoolExecutor
 from enum import Enum
 from typing import Union
 
+from playwright.async_api import async_playwright
 from playwright.sync_api import sync_playwright
 
 import params
@@ -22,28 +26,39 @@ class ResultEnum(Enum):
     PENDING = "PENDING"
 
 
-# check result with playright
+class Tls(threading.local):
-def check_result_page(url, phone_number: str) -> ResultEnum:
+    def __init__(self) -> None:
-    with sync_playwright() as p:
+        self.playwright = sync_playwright().start()
-        return run(p, url, phone_number)
+        print("Create playwright instance in Thread", threading.current_thread().name)
 
 
-def run(playwright, url, phone_number: str) -> ResultEnum:
+class Worker:
-    browser = playwright.firefox.launch(headless=False)
+    tls = Tls()
-    url_to_check = url.replace("register/", "")
+
-    url_to_check = url_to_check + "?lang=fr"
+    def load_page(self, browser, url, proxy) -> Union[str, None]:
-    content = None
+        try:
-    random_id_number = str(phone_number)[1:len(str(phone_number))]
+            firefox_user_agents = filter(lambda user_agent: "firefox" in user_agent.lower(), user_agent_list)
-    proxy_username = "panleicim-stc-fr-" + random_id_number
+            firefox_user_agents_list = list(firefox_user_agents)
-    print("proxy_username is " + proxy_username)
+            user_agent = random.choice(firefox_user_agents_list)
-    proxy = {
+            page = browser.new_page(
-        "server": params.PROXY_SERVER,
+                user_agent=user_agent,
-        "username": proxy_username,
+                proxy=proxy)
-        "password": params.PROXY_PASSWORD
+            page.add_init_script("""() => Object.defineProperty(navigator,'webdriver',{get: () => undefined}""")
-    }
+            page.goto(url, timeout=90000)
-    while content is None:
+            return page.content()
-        content = load_page(browser, url_to_check, proxy)
+        except Exception as error:
-        random_id_number = get_random_id_number_for_proxy()
+            print(error)
+            return None
+
+    def run(self, reserve_pojo: ReserveResultPojo, collection):
+        print("Launched worker in ", threading.current_thread().name)
+        browser = self.tls.playwright.firefox.launch(headless=False)
+        url = reserve_pojo.url
+        phone_number = reserve_pojo.phone
+        url_to_check = url.replace("register/", "")
+        url_to_check = url_to_check + "?lang=fr"
+        content = None
+        random_id_number = str(phone_number)[1:len(str(phone_number))]
         proxy_username = "panleicim-stc-fr-" + random_id_number
         print("proxy_username is " + proxy_username)
         proxy = {
@@ -51,33 +66,30 @@ def run(playwright, url, phone_number: str) -> ResultEnum:
             "username": proxy_username,
             "password": params.PROXY_PASSWORD
         }
-    print(content)
+        while content is None:
-    browser.close()
+            content = self.load_page(browser, url_to_check, proxy)
-    if SORRY_SENTENCE in content:
+            random_id_number = get_random_id_number_for_proxy()
-        print("status is REFUSED")
+            proxy_username = "panleicim-stc-fr-" + random_id_number
-        return ResultEnum.REFUSED
+            print("proxy_username is " + proxy_username)
-    elif PENDING_SENTENCE in content:
+            proxy = {
-        print("status is PENDING")
+                "server": params.PROXY_SERVER,
-        return ResultEnum.PENDING
+                "username": proxy_username,
-    else:
+                "password": params.PROXY_PASSWORD
-        print("status is ACCEPTED")
+            }
-        return ResultEnum.ACCEPTED
+        print(content)
-
+        browser.close()
-
+        print("Stopped worker in ", threading.current_thread().name)
-def load_page(browser, url, proxy) -> Union[str, None]:
+        status = None
-    try:
+        if SORRY_SENTENCE in content:
-        firefox_user_agents = filter(lambda user_agent: "firefox" in user_agent.lower(), user_agent_list)
+            print("status is REFUSED")
-        firefox_user_agents_list = list(firefox_user_agents)
+            status = ResultEnum.REFUSED
-        user_agent = random.choice(firefox_user_agents_list)
+        elif PENDING_SENTENCE in content:
-        page = browser.new_page(
+            print("status is PENDING")
-            user_agent=user_agent,
+            status = ResultEnum.PENDING
-            proxy=proxy)
+        else:
-        page.add_init_script("""() => Object.defineProperty(navigator,'webdriver',{get: () => undefined}""")
+            print("status is ACCEPTED")
-        page.goto(url, timeout=90000)
+            status = ResultEnum.ACCEPTED
-        return page.content()
+        collection.document(reserve_pojo.id).update({u'accepted': status.name})
-    except Exception as error:
-        print(error)
-        return None
 
 
 # need to start at 21h00
@@ -85,15 +97,18 @@ if __name__ == '__main__':
     # get the list
     params.oracle_log_sender.send_log(msg="开始检查约会结果", subject=LOG_SUBJECT_EVENT, type=TYPE_EVENT_CHECK_RESULTS)
     db_manager = params.firebase_store_manager
-    collection = db_manager.get_all_successful_items()
+    collection = db_manager.get_all_successful_items_for_day("2022-03-16")
     count = 0
-    for appointment in collection.stream():
+    task_list = list()
-        count = count + 1
+    with ThreadPoolExecutor(max_workers=6) as executor:
-        reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict())
+        for appointment in collection.stream():
-        if reserve_pojo.accepted:
+            worker = Worker()
-            print("status is " + reserve_pojo.accepted)
+            count = count + 1
-        if reserve_pojo.accepted is None:
+            reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict())
-                #or ResultEnum.PENDING.value == reserve_pojo.accepted:
+            # if reserve_pojo.accepted:
-            result = check_result_page(reserve_pojo.url, reserve_pojo.phone)
+            #     print("status is " + reserve_pojo.accepted)
-            collection.document(reserve_pojo.id).update({u'accepted': result.name})
+            # if reserve_pojo.accepted is None:
-    print(count)
+            # or ResultEnum.PENDING.value == reserve_pojo.accepted:
+            # tasks = check_result_page(reserve_pojo.url, reserve_pojo.phone)
+            executor.submit(worker.run, reserve_pojo, collection)
+        print(count)