From 640d609fb232214da9679af37bfdf94389596654 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Sat, 30 Mar 2024 16:35:10 +0100 Subject: [PATCH] add ProxyManager --- link_validator_executor.py | 49 ++-------------------------------- proxy_manager/__init__.py | 0 proxy_manager/proxy_manager.py | 9 +++++++ workers/link_validator.py | 17 +++++++----- 4 files changed, 21 insertions(+), 54 deletions(-) create mode 100644 proxy_manager/__init__.py create mode 100644 proxy_manager/proxy_manager.py diff --git a/link_validator_executor.py b/link_validator_executor.py index c63953c..0aee0eb 100644 --- a/link_validator_executor.py +++ b/link_validator_executor.py @@ -1,50 +1,5 @@ -import random -import time -from concurrent.futures import as_completed -from concurrent.futures.thread import ThreadPoolExecutor -from http.cookies import SimpleCookie - -from db.mongo_manager import MONGO_STORE_MANAGER -from excel_reader import read_contacts -from models.contact_pojo import ContactPojo -from queue_message.link_validator_receiver import LinkValidatorReceiver -from queue_message.appointmentrequestsender import AppointmentRequestSender, filter_contacts -from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER -from workers.sender import Sender - -IPFIY = 'http://api.ipify.org' -NGROK_TEST = "https://bcc6-193-164-156-53.ngrok-free.app" - -# def handle_cookie(cookie: str): -# contact_list = read_contacts('/Users/panlei/Desktop/yahoo_aol_valid_26-1.xlsx') -# # remove already booked contacts -# sub_contact_list = filter_contacts(contact_list) -# random.shuffle(sub_contact_list) -# # all_task = [] -# # executor = ThreadPoolExecutor(max_workers=2) -# # urls = [3, 2, 4] # 并不是真的url -# # all_task = [executor.submit(delegate_sender, (url)) for con in sub_contact_list] -# with ThreadPoolExecutor(max_workers=2) as executor: -# for con in sub_contact_list: -# sender = Sender(cookie) -# _can_continue = executor.submit(delegate_sender, sender, con) -# print("_can_continue is {}".format(_can_continue)) -# if not _can_continue: -# break -# print("fin") -# # for future in as_completed(all_task): -# # data = future.result() -# # print("in main: get page {}s success".format(data)) - +from workers.link_validator import validate_with_FR_ip if __name__ == '__main__': # link_list = MONGO_STORE_MANAGER.get_links_to_validate() - # # remove already booked contacts - # random.shuffle(link_list) - # for l in link_list: - # print(l) - - # contact_list = read_contacts('/Users/panlei/Desktop/real_name_contacts_13_01_2024.xlsx') - # contact_list = filter_contacts(contact_list) - receiver = LinkValidatorReceiver() - receiver.run() + validate_with_FR_ip() \ No newline at end of file diff --git a/proxy_manager/__init__.py b/proxy_manager/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/proxy_manager/proxy_manager.py b/proxy_manager/proxy_manager.py new file mode 100644 index 0000000..64c8a2c --- /dev/null +++ b/proxy_manager/proxy_manager.py @@ -0,0 +1,9 @@ +FR_ASOCKS_MOBILE_PROXY = { + 'http': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046', + 'https': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046', +} + + +class ProxyManager: + def get_link_validate_proxy(self) -> list: + return [FR_ASOCKS_MOBILE_PROXY] diff --git a/workers/link_validator.py b/workers/link_validator.py index a5f881c..0ff47f5 100644 --- a/workers/link_validator.py +++ b/workers/link_validator.py @@ -9,6 +9,7 @@ import requests from db.mongo_manager import MONGO_STORE_MANAGER from models.LinkPojo import LinkPojo from models.result_pojo import RequestResult +from proxy_manager.proxy_manager import ProxyManager from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials from workers.proxies_constants import PROXY_LIST_FR @@ -19,7 +20,7 @@ DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ce class LinkValidator(threading.Thread): def __init__(self, link_to_validate_list: list, - cookiesPublisher: CookiesPublisher, proxy_to_use, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR", + cookiesPublisher: CookiesPublisher, proxy_manager:ProxyManager, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR", limit=40): super().__init__() self.cookie = SimpleCookie() @@ -28,7 +29,7 @@ class LinkValidator(threading.Thread): self.queue_to_listen = queue_to_listen self.ip_country = ip_country self.filter_with_ip_country() - self.proxy_to_use = proxy_to_use + self.proxy_manager = proxy_manager self.limit = limit # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;' @@ -56,11 +57,12 @@ class LinkValidator(threading.Thread): 'Sec-Fetch-Site': 'same-origin', 'Sec-Fetch-Dest': 'document', 'Accept-Language': 'fr-FR,fr;q=0.6'} - print(self.proxy_to_use) + _proxy_to_use = random.choice(self.proxy_manager.get_link_validate_proxy()) + print(_proxy_to_use) print("received cookie is " + str(self.cookie_str)) try: - response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use, - timeout=15) + response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use, + timeout=10) print(response.status_code) if response.status_code == 200: _content = response.text @@ -151,10 +153,11 @@ def validate_with_FR_ip(): _link_list_to_click.append(_link) for _l in _link_list_to_click: print(_l.ip_country) - _fr_proxy_to_use = random.choice(PROXY_LIST_FR) + _proxy_manager = ProxyManager() + # _fr_proxy_to_use = random.choice(PROXY_LIST_FR) # random.shuffle(_link_list_to_click) receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher, - proxy_to_use=_fr_proxy_to_use, + proxy_manager=_proxy_manager, queue_to_listen=_queue_name, ip_country="FR", limit=50) print("will connect to queue") receiver.set_up_connection()