panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add ProxyManager

Browse Source
This commit is contained in:
Lei PAN
2024-03-30 16:35:10 +01:00
parent 4f48fc0366
commit 640d609fb2
4 changed files with 21 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
link_validator_executor.py
+2 -47
View File
@@ -1,50 +1,5 @@
import random
import time
from concurrent.futures import as_completed
from concurrent.futures.thread import ThreadPoolExecutor
from http.cookies import SimpleCookie
from db.mongo_manager import MONGO_STORE_MANAGER
from excel_reader import read_contacts
from models.contact_pojo import ContactPojo
from queue_message.link_validator_receiver import LinkValidatorReceiver
from queue_message.appointmentrequestsender import AppointmentRequestSender, filter_contacts
from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
from workers.sender import Sender
IPFIY = 'http://api.ipify.org'
NGROK_TEST = "https://bcc6-193-164-156-53.ngrok-free.app"
# def handle_cookie(cookie: str):
# contact_list = read_contacts('/Users/panlei/Desktop/yahoo_aol_valid_26-1.xlsx')
# # remove already booked contacts
# sub_contact_list = filter_contacts(contact_list)
# random.shuffle(sub_contact_list)
# # all_task = []
# # executor = ThreadPoolExecutor(max_workers=2)
# # urls = [3, 2, 4] # 并不是真的url
# # all_task = [executor.submit(delegate_sender, (url)) for con in sub_contact_list]
# with ThreadPoolExecutor(max_workers=2) as executor:
# for con in sub_contact_list:
# sender = Sender(cookie)
# _can_continue = executor.submit(delegate_sender, sender, con)
# print("_can_continue is {}".format(_can_continue))
# if not _can_continue:
# break
# print("fin")
# # for future in as_completed(all_task):
# # data = future.result()
# # print("in main: get page {}s success".format(data))
from workers.link_validator import validate_with_FR_ip
if __name__ == '__main__':
# link_list = MONGO_STORE_MANAGER.get_links_to_validate()
# # remove already booked contacts
# random.shuffle(link_list)
# for l in link_list:
# print(l)
# contact_list = read_contacts('/Users/panlei/Desktop/real_name_contacts_13_01_2024.xlsx')
# contact_list = filter_contacts(contact_list)
receiver = LinkValidatorReceiver()
receiver.run()
validate_with_FR_ip()
proxy_manager/__init__.py
View File
proxy_manager/proxy_manager.py
+9
View File
@@ -0,0 +1,9 @@
FR_ASOCKS_MOBILE_PROXY = {
'http': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046',
'https': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046',
}
class ProxyManager:
def get_link_validate_proxy(self) -> list:
return [FR_ASOCKS_MOBILE_PROXY]
workers/link_validator.py
+10 -7
View File
@@ -9,6 +9,7 @@ import requests
from db.mongo_manager import MONGO_STORE_MANAGER
from models.LinkPojo import LinkPojo
from models.result_pojo import RequestResult
from proxy_manager.proxy_manager import ProxyManager
from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST
from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
from workers.proxies_constants import PROXY_LIST_FR
@@ -19,7 +20,7 @@ DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ce
class LinkValidator(threading.Thread):
def __init__(self, link_to_validate_list: list,
cookiesPublisher: CookiesPublisher, proxy_to_use, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR",
cookiesPublisher: CookiesPublisher, proxy_manager:ProxyManager, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR",
limit=40):
super().__init__()
self.cookie = SimpleCookie()
@@ -28,7 +29,7 @@ class LinkValidator(threading.Thread):
self.queue_to_listen = queue_to_listen
self.ip_country = ip_country
self.filter_with_ip_country()
self.proxy_to_use = proxy_to_use
self.proxy_manager = proxy_manager
self.limit = limit
# self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
@@ -56,11 +57,12 @@ class LinkValidator(threading.Thread):
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Dest': 'document',
'Accept-Language': 'fr-FR,fr;q=0.6'}
print(self.proxy_to_use)
_proxy_to_use = random.choice(self.proxy_manager.get_link_validate_proxy())
print(_proxy_to_use)
print("received cookie is " + str(self.cookie_str))
try:
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
timeout=15)
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use,
timeout=10)
print(response.status_code)
if response.status_code == 200:
_content = response.text
@@ -151,10 +153,11 @@ def validate_with_FR_ip():
_link_list_to_click.append(_link)
for _l in _link_list_to_click:
print(_l.ip_country)
_fr_proxy_to_use = random.choice(PROXY_LIST_FR)
_proxy_manager = ProxyManager()
# _fr_proxy_to_use = random.choice(PROXY_LIST_FR)
# random.shuffle(_link_list_to_click)
receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
proxy_to_use=_fr_proxy_to_use,
proxy_manager=_proxy_manager,
queue_to_listen=_queue_name, ip_country="FR", limit=50)
print("will connect to queue")
receiver.set_up_connection()