From 302294fe06fd5bd875c7c6bc5df655b5de0f1416 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Sat, 6 Apr 2024 09:11:23 +0200 Subject: [PATCH] add ProxyManager --- proxy_manager/proxy_manager.py | 12 ++++++++++-- workers/link_validator.py | 2 +- workers/proxies_constants.py | 1 + 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/proxy_manager/proxy_manager.py b/proxy_manager/proxy_manager.py index 64c8a2c..9eb8c81 100644 --- a/proxy_manager/proxy_manager.py +++ b/proxy_manager/proxy_manager.py @@ -3,7 +3,15 @@ FR_ASOCKS_MOBILE_PROXY = { 'https': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046', } +FR_PROXY_RES_OXY = { + 'http': 'http://customer-rendezvous-cc-FR:Rdv202220212023@pr.oxylabs.io:7777', + 'https': 'http://customer-rendezvous-cc-FR:Rdv202220212023@pr.oxylabs.io:7777' +} + class ProxyManager: - def get_link_validate_proxy(self) -> list: - return [FR_ASOCKS_MOBILE_PROXY] + def get_link_validate_proxy(self, links_to_validate: list) -> list: + if len(links_to_validate) > 15: + return [FR_PROXY_RES_OXY] + else: + return [FR_ASOCKS_MOBILE_PROXY] diff --git a/workers/link_validator.py b/workers/link_validator.py index 0ff47f5..a2095f5 100644 --- a/workers/link_validator.py +++ b/workers/link_validator.py @@ -57,7 +57,7 @@ class LinkValidator(threading.Thread): 'Sec-Fetch-Site': 'same-origin', 'Sec-Fetch-Dest': 'document', 'Accept-Language': 'fr-FR,fr;q=0.6'} - _proxy_to_use = random.choice(self.proxy_manager.get_link_validate_proxy()) + _proxy_to_use = random.choice(self.proxy_manager.get_link_validate_proxy(self.link_to_validate_list)) print(_proxy_to_use) print("received cookie is " + str(self.cookie_str)) try: diff --git a/workers/proxies_constants.py b/workers/proxies_constants.py index 05788da..22bafe5 100644 --- a/workers/proxies_constants.py +++ b/workers/proxies_constants.py @@ -63,6 +63,7 @@ DE_PROXY_RES = { # PROXY_LIST = [FR_PROXY_MOBILE, FR_PROXY_RES, DE_PROXY_RES, DE_PROXY_MOBILE, ES_PROXY_MOBILE, IT_PROXY_MOBILE] # PROXY_LIST_FR = [FR_PROXY_MOBILE_2, FR_PROXY_MOBILE, FR_PROXY_RES, FR_PROXY_RES_4, FR_PROXY_RES_2] # PROXY_LIST_FR = [FR_PROXY_RES_OXY, FR_PROXY_MOBILE_3] +# PROXY_LIST_FR = [ FR_PROXY_MOBILE_3] PROXY_LIST_FR = [FR_PROXY_RES_OXY] MOBILE_PROXY_LIST_FR = [FR_PROXY_MOB_OXY] # PROXY_LIST = [FR_PROXY_MOBILE, FR_