From 08ed7c8954d861237badd8e0ed3c01a2ee0cd98c Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 16 May 2022 08:48:55 +0200 Subject: [PATCH] can run playwright in multi-threads --- check_results.py | 44 +++++++++++++++++++-------------------- params.py | 10 ++++++++- workers/commandor_page.py | 16 +++++--------- 3 files changed, 36 insertions(+), 34 deletions(-) diff --git a/check_results.py b/check_results.py index ca698e1..8d7915c 100644 --- a/check_results.py +++ b/check_results.py @@ -8,7 +8,6 @@ from typing import Union from playwright.sync_api import sync_playwright import params -from workers.commandor_page import get_random_id_number_for_proxy from logs.LogSender import TYPE_EVENT_CHECK_RESULTS, LOG_SUBJECT_EVENT from pojo.ReserveResultPojo import ReserveResultPojo, PublishType @@ -25,7 +24,7 @@ class ResultEnum(Enum): class TlsPlaywright(threading.local): def __init__(self) -> None: - self.playwright = params.SINGLE_PLAYWRIGHT_INSTANCE + self.playwright = sync_playwright().start() print("Create playwright instance in Thread", threading.current_thread().name) @@ -76,7 +75,7 @@ class ResultChecker: } while content is None: content = self.load_page(self.tls.playwright, proxy, url) - random_id_number = get_random_id_number_for_proxy() + random_id_number = params.get_random_id_number_for_proxy() proxy_username = "panleicim-res-fr-" + random_id_number print("proxy_username is " + proxy_username) proxy = { @@ -110,24 +109,25 @@ if __name__ == '__main__': db_manager = params.firebase_store_manager collection = db_manager.get_all_successful_items_for_day("2022-05-14", "landd") count = 0 - # result_pojo = ReserveResultPojo(type=PublishType.SUCCESS, phone="0649614591", email="panleicim@gmail.com", - # message="SUCCESS", firstName="Lei", lastName="PAN", url='https://api.ipify.org') + result_pojo = ReserveResultPojo(type=PublishType.SUCCESS, phone="0649614591", email="panleicim@gmail.com", + message="SUCCESS", firstName="Lei", lastName="PAN", url='https://api.ipify.org') result_list = [] - for appointment in collection.stream(): - reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) - result_list.append(reserve_pojo) - # result_list.append(result_pojo) - for result in result_list: - if result.accepted is None or ResultEnum.ACCEPTED.value == result.accepted: - ResultChecker().run(result, collection) - else: - print("status is " + result.accepted) + # for appointment in collection.stream(): + # reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict()) + # result_list.append(reserve_pojo) + result_list.append(result_pojo) + # for result in result_list: + # if result.accepted is None or ResultEnum.PENDING.value == result.accepted: + # ResultChecker().run(result, collection) + # else: + # print("status is " + result.accepted) - # with ThreadPoolExecutor(max_workers=1) as executor: - # for reserve in result_list: - # count = count + 1 - # if reserve.accepted: - # print("status is " + reserve.accepted) - # if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted: - # executor.submit(ResultChecker().run, reserve, collection) - # print(count) + with ThreadPoolExecutor(max_workers=1) as executor: + for reserve in result_list: + count = count + 1 + if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted: + executor.submit(ResultChecker().run, reserve, collection) + else: + print("status is " + reserve.accepted) + + print(count) diff --git a/params.py b/params.py index d154bc9..0f413bf 100644 --- a/params.py +++ b/params.py @@ -1,4 +1,6 @@ import configparser +import random +import string from playwright.sync_api import sync_playwright @@ -11,7 +13,13 @@ oracle_log_sender = LogSender() # proxy PROXY_SERVER = "http://gw.ntnt.io:5959" PROXY_PASSWORD = "94sY7zwBG13i" -SINGLE_PLAYWRIGHT_INSTANCE = sync_playwright().start() + +def get_random_id_number_for_proxy() -> str: + S = 8 # number of characters in the string. + ran = ''.join(random.choices(string.digits, k=S)) + id_number = str(ran) + print("The randomly generated string is : " + str(ran)) # print the random data + return id_number DEVICES = ['iPad (gen 6)', 'iPad (gen 6) landscape', 'iPad (gen 7)', 'iPad (gen 7) landscape', 'iPad Mini', 'iPad Mini landscape', 'iPad Pro 11', 'iPad Pro 11 landscape', 'iPhone 6', 'iPhone 6 landscape', diff --git a/workers/commandor_page.py b/workers/commandor_page.py index da84d83..0a42839 100644 --- a/workers/commandor_page.py +++ b/workers/commandor_page.py @@ -5,6 +5,8 @@ import threading import time from typing import Union +from playwright.sync_api import sync_playwright + import params from params import PROXY_SERVER, PROXY_PASSWORD from pojo.ReserveResultPojo import ReserveResultPojo, PublishType @@ -31,7 +33,7 @@ def get_random_wait_time() -> float: class Tls(threading.local): def __init__(self) -> None: - self.playwright = params.SINGLE_PLAYWRIGHT_INSTANCE + self.playwright = sync_playwright().start() class CommandorPage: @@ -74,7 +76,7 @@ class CommandorPage: first_page = None while first_page is None: first_page = self.start_browser(proxy, self.tls.playwright, devices) - proxy_username = "panleicim-res-fr-" + get_random_id_number_for_proxy() + proxy_username = "panleicim-res-fr-" + params.get_random_id_number_for_proxy() self.logger.info("proxy_username is " + proxy_username) proxy = { "server": params.PROXY_SERVER, @@ -264,14 +266,6 @@ class CommandorPage: self.browser.close() -def get_random_id_number_for_proxy() -> str: - S = 8 # number of characters in the string. - ran = ''.join(random.choices(string.digits, k=S)) - id_number = str(ran) - print("The randomly generated string is : " + str(ran)) # print the random data - return id_number - - def get_random_id_number() -> str: S = 8 # number of characters in the string. ran = ''.join(random.choices(string.digits, k=S)) @@ -285,7 +279,7 @@ def on_success(result: ReserveResultPojo): def launch_page(): - PROXY_USERNAME = "panleicim-res-fr-" + get_random_id_number_for_proxy() + PROXY_USERNAME = "panleicim-res-fr-" + params.get_random_id_number_for_proxy() print("proxy_username is " + PROXY_USERNAME) proxy = { "server": PROXY_SERVER,