add BrightData proxy support

This commit is contained in:
Lei PAN
2022-06-13 11:42:54 +02:00
parent 5dcb0f6bbb
commit d1699c295f
7 changed files with 40 additions and 38 deletions
+10 -12
View File
@@ -3,14 +3,15 @@ import PySimpleGUI as sg
# First the window layout in 2 columns # First the window layout in 2 columns
from main import start_book from main import start_book
from pojo.ModeEnum import ModeEnum from pojo.ModeEnum import ModeEnum
from proxy.proxy_type import ProxyType
KEY_CHOOSE_STORE = "CHOOSE_STORE" KEY_CHOOSE_STORE = "CHOOSE_STORE"
KEY_START_NUMBER = "KEY_START_NUMBER" KEY_START_NUMBER = "KEY_START_NUMBER"
KEY_END_NUMBER = "KEY_END_NUMBER" KEY_END_NUMBER = "KEY_END_NUMBER"
KEY_MAX_WORKERS = "KEY_MAX_WORKERS" KEY_MAX_WORKERS = "KEY_MAX_WORKERS"
KEY_RANDOM = "KEY_RANDOM" KEY_RANDOM = "KEY_RANDOM"
KEY_PROXY_RES = "KEY_PROXY_RES" KEY_PROXY_BRIGHTDATA = "KEY_PROXY_BRIGHTDATA"
KEY_PROXY_CC = "KEY_PROXY_CC" KEY_PROXY_NETNUT = "KEY_PROXY_NETNUT"
KEY_FAUBOURG = "KEY_FAUBOURG" KEY_FAUBOURG = "KEY_FAUBOURG"
KEY_GEORGE = "KEY_GEORGE" KEY_GEORGE = "KEY_GEORGE"
KEY_SEVRES = "KEY_SEVRES" KEY_SEVRES = "KEY_SEVRES"
@@ -37,8 +38,8 @@ store_settings_column = [
] ]
proxy_settings_column = [ proxy_settings_column = [
[sg.Text("代理ip池")], [sg.Text("代理ip池")],
[sg.Radio('res(速度)', group_id=GROUP_PROXY, key=KEY_PROXY_RES, default=True)], [sg.Radio('亮数据', group_id=GROUP_PROXY, key=KEY_PROXY_BRIGHTDATA, default=True)],
[sg.Radio('cc(稳定)', group_id=GROUP_PROXY, key=KEY_PROXY_CC, default=False)], [sg.Radio('Netnut', group_id=GROUP_PROXY, key=KEY_PROXY_NETNUT, default=False)],
] ]
mode_settings_column = [ mode_settings_column = [
@@ -76,19 +77,16 @@ while True:
elif values[KEY_SEVRES]: elif values[KEY_SEVRES]:
store_type = 3 store_type = 3
proxy_type = 0 proxy_type = ProxyType.BRIGHT_DATA
if values[KEY_PROXY_CC]: if values[KEY_PROXY_NETNUT]:
proxy_type = 1 proxy_type = ProxyType.NETNUT
elif values[KEY_PROXY_RES]: elif values[KEY_PROXY_BRIGHTDATA]:
proxy_type = 0 proxy_type = ProxyType.BRIGHT_DATA
if values[KEY_AUTOMATIC]: if values[KEY_AUTOMATIC]:
mode = ModeEnum.AUTOMATIC mode = ModeEnum.AUTOMATIC
start_book(start_line, end_line, store_choose_state=store_type, max_workers=max_workers, proxy_type=proxy_type, start_book(start_line, end_line, store_choose_state=store_type, max_workers=max_workers, proxy_type=proxy_type,
mode=mode) mode=mode)
# except Exception as error:
# print("Not Integer: ")
# print(error)
elif event == "Exit" or event == sg.WIN_CLOSED: elif event == "Exit" or event == sg.WIN_CLOSED:
break break
+2 -5
View File
@@ -12,6 +12,7 @@ from notification.AcceptedResultPojo import get_accepted_result_from
from notification.mailer import Mailer from notification.mailer import Mailer
from pojo.ReserveResultPojo import ReserveResultPojo from pojo.ReserveResultPojo import ReserveResultPojo
from pojo.ResultEnum import ResultEnum from pojo.ResultEnum import ResultEnum
from proxy.proxy_type import ProxyType
SORRY_SENTENCE_FR = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci" SORRY_SENTENCE_FR = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci"
SORRY_SENTENCE_EN = "we are extremely sorry that we were not able to fulfill" SORRY_SENTENCE_EN = "we are extremely sorry that we were not able to fulfill"
@@ -72,11 +73,7 @@ class ResultChecker:
} }
while content is None: while content is None:
content = self.load_page(self.tls.playwright, proxy, url) content = self.load_page(self.tls.playwright, proxy, url)
proxy = { proxy = params.get_proxy(ProxyType.NETNUT)
"server": params.BRIGHT_DATA_PROXY_SERVER,
"username": params.BRIGHT_DATA_PROXY_USERNAME,
"password": params.BRIGHT_DATA_PROXY_PASSWORD
}
print(content) print(content)
self.browser.close() self.browser.close()
print("Stopped worker in ", threading.current_thread().name) print("Stopped worker in ", threading.current_thread().name)
+5 -4
View File
@@ -6,6 +6,7 @@ import params
from logs.AppLogging import init_logger from logs.AppLogging import init_logger
from pojo.ModeEnum import ModeEnum from pojo.ModeEnum import ModeEnum
from pojo.contact_pojo import ContactPojo from pojo.contact_pojo import ContactPojo
from proxy.proxy_type import ProxyType
from utils.excel_reader import ExcelHelper from utils.excel_reader import ExcelHelper
from workers.commandor_page import CommandorPage from workers.commandor_page import CommandorPage
@@ -15,7 +16,7 @@ logger = logging.getLogger()
logger.addHandler(logging.StreamHandler(stream=sys.stdout)) logger.addHandler(logging.StreamHandler(stream=sys.stdout))
def start_book(start_number, end_number, store_choose_state=0, max_workers=10, proxy_type=0, def start_book(start_number, end_number, store_choose_state=0, max_workers=10, proxy_type=ProxyType.BRIGHT_DATA,
mode: ModeEnum = ModeEnum.MANUAL): mode: ModeEnum = ModeEnum.MANUAL):
# read the contact, and contact the 2 objects together # read the contact, and contact the 2 objects together
excel_reader = ExcelHelper() excel_reader = ExcelHelper()
@@ -35,9 +36,9 @@ def start_book(start_number, end_number, store_choose_state=0, max_workers=10, p
def recheck_the_captcha_error_contacts(store_type=0, mode: ModeEnum = ModeEnum.MANUAL): def recheck_the_captcha_error_contacts(store_type=0, mode: ModeEnum = ModeEnum.MANUAL):
# get all the contacts in captcha error # get all the contacts in captcha error
list = params.local_db_manager.get_all_captcha_error_contacts() contact_list = params.local_db_manager.get_all_captcha_error_contacts()
with ThreadPoolExecutor(max_workers=10) as executor: with ThreadPoolExecutor(max_workers=10) as executor:
for errorContact in list: for errorContact in contact_list:
contact = ContactPojo.get_contact_from_error_contact(errorContact) contact = ContactPojo.get_contact_from_error_contact(errorContact)
proxy = get_proxy() proxy = get_proxy()
# start the task in thread # start the task in thread
@@ -46,7 +47,7 @@ def recheck_the_captcha_error_contacts(store_type=0, mode: ModeEnum = ModeEnum.M
proxy) proxy)
def get_proxy(proxy_type=0): def get_proxy(proxy_type=ProxyType.BRIGHT_DATA):
return params.get_proxy(proxy_type) return params.get_proxy(proxy_type)
+12 -14
View File
@@ -5,6 +5,7 @@ import definitions
from db.DbManager import DataManager from db.DbManager import DataManager
from db.local_db_manager import LocalDbManager from db.local_db_manager import LocalDbManager
from logs.LogSender import LogSender from logs.LogSender import LogSender
from proxy.proxy_type import ProxyType
firebase_store_manager = DataManager() firebase_store_manager = DataManager()
oracle_log_sender = LogSender() oracle_log_sender = LogSender()
@@ -22,33 +23,30 @@ PROXY_NAME_PREFIX_RES = "panleicim-res-fr-"
PROXY_NAME_PREFIX_CC = "panleicim-cc-fr-" PROXY_NAME_PREFIX_CC = "panleicim-cc-fr-"
def get_proxy_name_prefix(proxy_type=0) -> str: def get_proxy_name_prefix() -> str:
if proxy_type == 0:
return PROXY_NAME_PREFIX_RES return PROXY_NAME_PREFIX_RES
else:
return PROXY_NAME_PREFIX_CC
def get_proxy(proxy_type=0): def get_proxy(proxy_type: ProxyType):
if proxy_type == 0: if proxy_type == ProxyType.NETNUT:
random_id_number = get_random_id_number_for_proxy() random_id_number = get_random_id_number_for_proxy()
proxy_username = get_proxy_name_prefix(proxy_type) + random_id_number proxy_username = get_proxy_name_prefix() + random_id_number
proxy = { proxy = {
"server": PROXY_SERVER, "server": PROXY_SERVER,
"username": proxy_username, "username": proxy_username,
"password": PROXY_PASSWORD "password": PROXY_PASSWORD
} }
else: else:
# proxy = {
# "server": BRIGHT_DATA_PROXY_SERVER,
# "username": BRIGHT_DATA_PROXY_USERNAME,
# "password": BRIGHT_DATA_PROXY_PASSWORD
# }
proxy = { proxy = {
"server": BRIGHT_DATA_PROXY_SERVER, "server": BRIGHT_DATA_PROXY_SERVER,
"username": BRIGHT_DATA_MOBILE_PROXY_USERNAME, "username": BRIGHT_DATA_PROXY_USERNAME,
"password": BRIGHT_DATA_MOBILE_PROXY_PASSWORD "password": BRIGHT_DATA_PROXY_PASSWORD
} }
# proxy = {
# "server": BRIGHT_DATA_PROXY_SERVER,
# "username": BRIGHT_DATA_MOBILE_PROXY_USERNAME,
# "password": BRIGHT_DATA_MOBILE_PROXY_PASSWORD
# }
return proxy return proxy
View File
+6
View File
@@ -0,0 +1,6 @@
from enum import Enum
class ProxyType(Enum):
NETNUT = "NETNUT"
BRIGHT_DATA = "BRIGHT_DATA"
+4 -2
View File
@@ -13,6 +13,7 @@ from pojo.ReserveResultPojo import ReserveResultPojo, PublishType
from pojo.captcha_error_contact_pojo import ERROR_TYPE_CAPTCHA, \ from pojo.captcha_error_contact_pojo import ERROR_TYPE_CAPTCHA, \
TOO_MANY_REQUEST_ERROR TOO_MANY_REQUEST_ERROR
from pojo.contact_pojo import ContactPojo from pojo.contact_pojo import ContactPojo
from proxy.proxy_type import ProxyType
from utils.generate_random_passport_id import get_captcha_error_contact_from_contact from utils.generate_random_passport_id import get_captcha_error_contact_from_contact
from workers.SolveCaptch import SolveCaptcha from workers.SolveCaptch import SolveCaptcha
@@ -52,7 +53,8 @@ class Tls(threading.local):
class CommandorPage: class CommandorPage:
tls = Tls() tls = Tls()
def __init__(self, contact: ContactPojo, store_type=0, proxy_type=0, mode: ModeEnum = ModeEnum.MANUAL): def __init__(self, contact: ContactPojo, store_type=0, proxy_type=ProxyType.BRIGHT_DATA,
mode: ModeEnum = ModeEnum.MANUAL):
self.otp_value = None self.otp_value = None
self.logger = logging.getLogger("约会页面:" + str(contact.phone)) self.logger = logging.getLogger("约会页面:" + str(contact.phone))
self.is_finished = False self.is_finished = False
@@ -379,7 +381,7 @@ def launch_page():
first_name="xingzhen", first_name="xingzhen",
mail="ColbyPatel653@gmail.com", ccid="", position=0) mail="ColbyPatel653@gmail.com", ccid="", position=0)
page = CommandorPage(contact, store_type=1) page = CommandorPage(contact, store_type=1)
return page.start_page(params.get_proxy(0)) return page.start_page(params.get_proxy())
def wait_for_otp(event: threading.Event, commandor: CommandorPage): def wait_for_otp(event: threading.Event, commandor: CommandorPage):