panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

try to use curl_cffi

Browse Source
This commit is contained in:
PAN Lei
2024-10-05 11:21:20 +02:00
parent c2db1035c4
commit f9ce820b9f
4 changed files with 29 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
proxy_manager/proxy_manager.py
+8 -6
View File
@@ -49,6 +49,7 @@ MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_
FR_PROXY_RES_OXY_STICKY,
FR_PROXY_DATA_IMPULSE_STICKY]
# MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY]
@@ -61,12 +62,13 @@ class ProxyManager:
self.logger = logger
def get_link_validate_proxy(self, links_to_validate: list) -> list:
if len(links_to_validate) > 15:
return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
# return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY]
else:
return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY]
# if len(links_to_validate) > 15:
# return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
# # return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY]
# else:
#
# return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
# return [FR_PROXY_RES_OXY]
def get_result_link_proxy(self):
queue_message/appointmentrequestsender.py
+4 -4
View File
@@ -101,8 +101,8 @@ class AppointmentRequestSender(threading.Thread):
_received_object = body.decode("UTF-8")
self.logger.info(f" [x] Received {_received_object}")
# 同一个COOKIES保持用同一个IP
_proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
print("proxy to use is {}".format(_proxy_to_use))
# _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
# print("proxy to use is {}".format(_proxy_to_use))
if "glrd" in _received_object:
_received_dict = json.loads(_received_object)
js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'],
@@ -124,8 +124,8 @@ class AppointmentRequestSender(threading.Thread):
# self.contact_list = filter_contacts(self.contact_list)
can_continue = None
for con in self.contact_list:
# _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
# print(_proxy_to_use)
_proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
print(_proxy_to_use)
sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher,
received_dict=_received_dict,
proxy_to_use=_proxy_to_use, logger=self.logger)
workers/link_validator_with_provided_list.py
+6 -2
View File
@@ -7,7 +7,7 @@ from datetime import datetime
from http.cookies import SimpleCookie
import pika
import requests
from curl_cffi import requests
from db.mongo_manager import MONGO_STORE_MANAGER
from models.LinkPojo import LinkPojo
@@ -56,6 +56,10 @@ class LinkValidatorWithProvidedList(threading.Thread):
_ua = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36'
if _received_dict is not None:
_ua = _received_dict['ua']
_model = _received_dict['model']
_ua = "Mozilla/5.0 (Linux; Android 9; {}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.58 Mobile Safari/537.36".format(
_model)
logger.info("model: %s", _model)
self.cookie.load(self.cookie_str)
headers = {
'User-Agent': _ua,
@@ -78,7 +82,7 @@ class LinkValidatorWithProvidedList(threading.Thread):
print("will send request with ua {}".format(_ua))
print("will send request with cookie {}".format(self.cookie_str))
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
timeout=60)
timeout=60, impersonate="chrome99_android")
print(response.status_code)
if response.status_code == 200:
_content = response.text
workers/sender.py
+10 -7
View File
@@ -6,7 +6,7 @@ import re
import time
from http.cookies import SimpleCookie
from models.contact_pojo import ContactPojo
import requests
from curl_cffi import requests
from db.mongo_manager import MONGO_STORE_MANAGER
from models.ReserveResultPojo import ReserveResultPojo, PublishType
@@ -118,6 +118,8 @@ class Sender:
self._csrf = csrf
_csrf = csrf
self.logger.info("model is:{}".format(model))
js_data.ua = "Mozilla/5.0 (Linux; Android 9; {}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.58 Mobile Safari/537.36".format(
model)
_chrome_code = get_chrome_version_from_ua(js_data.ua)
_chrome_version = None
if _chrome_code == '128':
@@ -128,11 +130,12 @@ class Sender:
_chrome_version = "116.0.5845.114"
elif _chrome_code == '114':
_chrome_version = "114.0.5735.53"
_chrome_ua_version = '"Chromium";v="{}", "Not;A=Brand";v="24.0.0.0", "Google Chrome";v="{}"'.format(
_chrome_version, _chrome_version)
_chrome_ua_header = '"Chromium";v="{}", "Not;A=Brand";v="24", "Google Chrome";v="{}"'.format(_chrome_code,
_chrome_code)
elif _chrome_code == '99':
_chrome_version = "99.0.4844.58"
# 99: " Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"
# " Not A;Brand";v="99.0.0.0", "Chromium";v="99.0.4844.58", "Google Chrome";v="99.0.4844.58"
_chrome_ua_version = 'Not A;Brand";v="99.0.0.0", "Chromium";v="99.0.4844.58", "Google Chrome";v="99.0.4844.58"'
_chrome_ua_header = 'Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"'
self.logger.info("chrome_ua_header is:{}".format(_chrome_ua_header))
self.logger.info("_chrome_ua_version is:{}".format(_chrome_ua_version))
# 默认为空，也就是随机
@@ -168,7 +171,7 @@ class Sender:
proxy_to_use = self.proxy_to_use
# print(proxy_to_use)
response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data,
timeout=15, allow_redirects=False)
timeout=15, allow_redirects=False, impersonate="chrome99_android")
self.logger.info(response.status_code)
if response.status_code == 302:
# add to mongodb