panleicim/appointment_tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

can run playwright in multi-threads

Browse Source
This commit is contained in:
PAN Lei
2022-05-16 08:48:55 +02:00
parent 5ac4f6a0fb
commit 08ed7c8954
3 changed files with 36 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
check_results.py
+22 -22
View File
@@ -8,7 +8,6 @@ from typing import Union
from playwright.sync_api import sync_playwright
import params
from workers.commandor_page import get_random_id_number_for_proxy
from logs.LogSender import TYPE_EVENT_CHECK_RESULTS, LOG_SUBJECT_EVENT
from pojo.ReserveResultPojo import ReserveResultPojo, PublishType
@@ -25,7 +24,7 @@ class ResultEnum(Enum):
class TlsPlaywright(threading.local):
def __init__(self) -> None:
self.playwright = params.SINGLE_PLAYWRIGHT_INSTANCE
self.playwright = sync_playwright().start()
print("Create playwright instance in Thread", threading.current_thread().name)
@@ -76,7 +75,7 @@ class ResultChecker:
}
while content is None:
content = self.load_page(self.tls.playwright, proxy, url)
random_id_number = get_random_id_number_for_proxy()
random_id_number = params.get_random_id_number_for_proxy()
proxy_username = "panleicim-res-fr-" + random_id_number
print("proxy_username is " + proxy_username)
proxy = {
@@ -110,24 +109,25 @@ if __name__ == '__main__':
db_manager = params.firebase_store_manager
collection = db_manager.get_all_successful_items_for_day("2022-05-14", "landd")
count = 0
# result_pojo = ReserveResultPojo(type=PublishType.SUCCESS, phone="0649614591", email="panleicim@gmail.com",
# message="SUCCESS", firstName="Lei", lastName="PAN", url='https://api.ipify.org')
result_pojo = ReserveResultPojo(type=PublishType.SUCCESS, phone="0649614591", email="panleicim@gmail.com",
message="SUCCESS", firstName="Lei", lastName="PAN", url='https://api.ipify.org')
result_list = []
for appointment in collection.stream():
reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict())
result_list.append(reserve_pojo)
# result_list.append(result_pojo)
for result in result_list:
if result.accepted is None or ResultEnum.ACCEPTED.value == result.accepted:
ResultChecker().run(result, collection)
else:
print("status is " + result.accepted)
# for appointment in collection.stream():
# reserve_pojo = ReserveResultPojo.from_firestore_dict(appointment.to_dict())
# result_list.append(reserve_pojo)
result_list.append(result_pojo)
# for result in result_list:
# if result.accepted is None or ResultEnum.PENDING.value == result.accepted:
# ResultChecker().run(result, collection)
# else:
# print("status is " + result.accepted)
# with ThreadPoolExecutor(max_workers=1) as executor:
# for reserve in result_list:
# count = count + 1
# if reserve.accepted:
# print("status is " + reserve.accepted)
# if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted:
# executor.submit(ResultChecker().run, reserve, collection)
# print(count)
with ThreadPoolExecutor(max_workers=1) as executor:
for reserve in result_list:
count = count + 1
if reserve.accepted is None or ResultEnum.PENDING.value == reserve.accepted:
executor.submit(ResultChecker().run, reserve, collection)
else:
print("status is " + reserve.accepted)
print(count)
params.py
+9 -1
View File
@@ -1,4 +1,6 @@
import configparser
import random
import string
from playwright.sync_api import sync_playwright
@@ -11,7 +13,13 @@ oracle_log_sender = LogSender()
# proxy
PROXY_SERVER = "http://gw.ntnt.io:5959"
PROXY_PASSWORD = "94sY7zwBG13i"
SINGLE_PLAYWRIGHT_INSTANCE = sync_playwright().start()
def get_random_id_number_for_proxy() -> str:
S = 8 # number of characters in the string.
ran = ''.join(random.choices(string.digits, k=S))
id_number = str(ran)
print("The randomly generated string is : " + str(ran)) # print the random data
return id_number
DEVICES = ['iPad (gen 6)', 'iPad (gen 6) landscape', 'iPad (gen 7)', 'iPad (gen 7) landscape', 'iPad Mini',
'iPad Mini landscape', 'iPad Pro 11', 'iPad Pro 11 landscape', 'iPhone 6', 'iPhone 6 landscape',
workers/commandor_page.py
+5 -11
View File
@@ -5,6 +5,8 @@ import threading
import time
from typing import Union
from playwright.sync_api import sync_playwright
import params
from params import PROXY_SERVER, PROXY_PASSWORD
from pojo.ReserveResultPojo import ReserveResultPojo, PublishType
@@ -31,7 +33,7 @@ def get_random_wait_time() -> float:
class Tls(threading.local):
def __init__(self) -> None:
self.playwright = params.SINGLE_PLAYWRIGHT_INSTANCE
self.playwright = sync_playwright().start()
class CommandorPage:
@@ -74,7 +76,7 @@ class CommandorPage:
first_page = None
while first_page is None:
first_page = self.start_browser(proxy, self.tls.playwright, devices)
proxy_username = "panleicim-res-fr-" + get_random_id_number_for_proxy()
proxy_username = "panleicim-res-fr-" + params.get_random_id_number_for_proxy()
self.logger.info("proxy_username is " + proxy_username)
proxy = {
"server": params.PROXY_SERVER,
@@ -264,14 +266,6 @@ class CommandorPage:
self.browser.close()
def get_random_id_number_for_proxy() -> str:
S = 8 # number of characters in the string.
ran = ''.join(random.choices(string.digits, k=S))
id_number = str(ran)
print("The randomly generated string is : " + str(ran)) # print the random data
return id_number
def get_random_id_number() -> str:
S = 8 # number of characters in the string.
ran = ''.join(random.choices(string.digits, k=S))
@@ -285,7 +279,7 @@ def on_success(result: ReserveResultPojo):
def launch_page():
PROXY_USERNAME = "panleicim-res-fr-" + get_random_id_number_for_proxy()
PROXY_USERNAME = "panleicim-res-fr-" + params.get_random_id_number_for_proxy()
print("proxy_username is " + PROXY_USERNAME)
proxy = {
"server": PROXY_SERVER,