From f9ce820b9f1fdd09be535c4eed14a26d37420a57 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Sat, 5 Oct 2024 11:21:20 +0200 Subject: [PATCH] try to use curl_cffi --- proxy_manager/proxy_manager.py | 16 +++++++++------- queue_message/appointmentrequestsender.py | 8 ++++---- workers/link_validator_with_provided_list.py | 8 ++++++-- workers/sender.py | 17 ++++++++++------- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/proxy_manager/proxy_manager.py b/proxy_manager/proxy_manager.py index 50f5c35..42df19d 100644 --- a/proxy_manager/proxy_manager.py +++ b/proxy_manager/proxy_manager.py @@ -49,6 +49,7 @@ MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_ FR_PROXY_RES_OXY_STICKY, FR_PROXY_DATA_IMPULSE_STICKY] + # MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY] @@ -61,13 +62,14 @@ class ProxyManager: self.logger = logger def get_link_validate_proxy(self, links_to_validate: list) -> list: - if len(links_to_validate) > 15: - return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES] - # return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY] - else: - - return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES] - # return [FR_PROXY_RES_OXY] + return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY] + # if len(links_to_validate) > 15: + # return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES] + # # return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY] + # else: + # + # return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES] + # return [FR_PROXY_RES_OXY] def get_result_link_proxy(self): return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY] diff --git a/queue_message/appointmentrequestsender.py b/queue_message/appointmentrequestsender.py index ddbd8b4..5b95a2f 100644 --- a/queue_message/appointmentrequestsender.py +++ b/queue_message/appointmentrequestsender.py @@ -101,8 +101,8 @@ class AppointmentRequestSender(threading.Thread): _received_object = body.decode("UTF-8") self.logger.info(f" [x] Received {_received_object}") # 同一个COOKIES保持用同一个IP - _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() - print("proxy to use is {}".format(_proxy_to_use)) + # _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() + # print("proxy to use is {}".format(_proxy_to_use)) if "glrd" in _received_object: _received_dict = json.loads(_received_object) js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'], @@ -124,8 +124,8 @@ class AppointmentRequestSender(threading.Thread): # self.contact_list = filter_contacts(self.contact_list) can_continue = None for con in self.contact_list: - # _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() - # print(_proxy_to_use) + _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() + print(_proxy_to_use) sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict, proxy_to_use=_proxy_to_use, logger=self.logger) diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index aaf8df5..2324203 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -7,7 +7,7 @@ from datetime import datetime from http.cookies import SimpleCookie import pika -import requests +from curl_cffi import requests from db.mongo_manager import MONGO_STORE_MANAGER from models.LinkPojo import LinkPojo @@ -56,6 +56,10 @@ class LinkValidatorWithProvidedList(threading.Thread): _ua = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36' if _received_dict is not None: _ua = _received_dict['ua'] + _model = _received_dict['model'] + _ua = "Mozilla/5.0 (Linux; Android 9; {}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.58 Mobile Safari/537.36".format( + _model) + logger.info("model: %s", _model) self.cookie.load(self.cookie_str) headers = { 'User-Agent': _ua, @@ -78,7 +82,7 @@ class LinkValidatorWithProvidedList(threading.Thread): print("will send request with ua {}".format(_ua)) print("will send request with cookie {}".format(self.cookie_str)) response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use, - timeout=60) + timeout=60, impersonate="chrome99_android") print(response.status_code) if response.status_code == 200: _content = response.text diff --git a/workers/sender.py b/workers/sender.py index 6991baf..d416ca9 100644 --- a/workers/sender.py +++ b/workers/sender.py @@ -6,7 +6,7 @@ import re import time from http.cookies import SimpleCookie from models.contact_pojo import ContactPojo -import requests +from curl_cffi import requests from db.mongo_manager import MONGO_STORE_MANAGER from models.ReserveResultPojo import ReserveResultPojo, PublishType @@ -118,6 +118,8 @@ class Sender: self._csrf = csrf _csrf = csrf self.logger.info("model is:{}".format(model)) + js_data.ua = "Mozilla/5.0 (Linux; Android 9; {}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.58 Mobile Safari/537.36".format( + model) _chrome_code = get_chrome_version_from_ua(js_data.ua) _chrome_version = None if _chrome_code == '128': @@ -128,11 +130,12 @@ class Sender: _chrome_version = "116.0.5845.114" elif _chrome_code == '114': _chrome_version = "114.0.5735.53" - - _chrome_ua_version = '"Chromium";v="{}", "Not;A=Brand";v="24.0.0.0", "Google Chrome";v="{}"'.format( - _chrome_version, _chrome_version) - _chrome_ua_header = '"Chromium";v="{}", "Not;A=Brand";v="24", "Google Chrome";v="{}"'.format(_chrome_code, - _chrome_code) + elif _chrome_code == '99': + _chrome_version = "99.0.4844.58" + # 99: " Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99" + # " Not A;Brand";v="99.0.0.0", "Chromium";v="99.0.4844.58", "Google Chrome";v="99.0.4844.58" + _chrome_ua_version = 'Not A;Brand";v="99.0.0.0", "Chromium";v="99.0.4844.58", "Google Chrome";v="99.0.4844.58"' + _chrome_ua_header = 'Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"' self.logger.info("chrome_ua_header is:{}".format(_chrome_ua_header)) self.logger.info("_chrome_ua_version is:{}".format(_chrome_ua_version)) # 默认为空，也就是随机 @@ -168,7 +171,7 @@ class Sender: proxy_to_use = self.proxy_to_use # print(proxy_to_use) response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data, - timeout=15, allow_redirects=False) + timeout=15, allow_redirects=False, impersonate="chrome99_android") self.logger.info(response.status_code) if response.status_code == 302: # add to mongodb