panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

try to use curl_cffi

Browse Source
This commit is contained in:
PAN Lei
2024-10-05 11:21:20 +02:00
parent c2db1035c4
commit f9ce820b9f
4 changed files with 29 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
proxy_manager/proxy_manager.py
+8 -6
View File
@@ -49,6 +49,7 @@ MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_
FR_PROXY_RES_OXY_STICKY, FR_PROXY_RES_OXY_STICKY,
FR_PROXY_DATA_IMPULSE_STICKY] FR_PROXY_DATA_IMPULSE_STICKY]
# MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY] # MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY]
@@ -61,12 +62,13 @@ class ProxyManager:
self.logger = logger self.logger = logger
def get_link_validate_proxy(self, links_to_validate: list) -> list: def get_link_validate_proxy(self, links_to_validate: list) -> list:
if len(links_to_validate) > 15: return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY]
return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES] # if len(links_to_validate) > 15:
# return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY] # return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
else: # # return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY]
# else:
return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES] #
# return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
# return [FR_PROXY_RES_OXY] # return [FR_PROXY_RES_OXY]
def get_result_link_proxy(self): def get_result_link_proxy(self):
queue_message/appointmentrequestsender.py
+4 -4
View File
@@ -101,8 +101,8 @@ class AppointmentRequestSender(threading.Thread):
_received_object = body.decode("UTF-8") _received_object = body.decode("UTF-8")
self.logger.info(f" [x] Received {_received_object}") self.logger.info(f" [x] Received {_received_object}")
# 同一个COOKIES保持用同一个IP # 同一个COOKIES保持用同一个IP
_proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() # _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
print("proxy to use is {}".format(_proxy_to_use)) # print("proxy to use is {}".format(_proxy_to_use))
if "glrd" in _received_object: if "glrd" in _received_object:
_received_dict = json.loads(_received_object) _received_dict = json.loads(_received_object)
js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'], js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'],
@@ -124,8 +124,8 @@ class AppointmentRequestSender(threading.Thread):
# self.contact_list = filter_contacts(self.contact_list) # self.contact_list = filter_contacts(self.contact_list)
can_continue = None can_continue = None
for con in self.contact_list: for con in self.contact_list:
# _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
# print(_proxy_to_use) print(_proxy_to_use)
sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher,
received_dict=_received_dict, received_dict=_received_dict,
proxy_to_use=_proxy_to_use, logger=self.logger) proxy_to_use=_proxy_to_use, logger=self.logger)
workers/link_validator_with_provided_list.py
+6 -2
View File
@@ -7,7 +7,7 @@ from datetime import datetime
from http.cookies import SimpleCookie from http.cookies import SimpleCookie
import pika import pika
import requests from curl_cffi import requests
from db.mongo_manager import MONGO_STORE_MANAGER from db.mongo_manager import MONGO_STORE_MANAGER
from models.LinkPojo import LinkPojo from models.LinkPojo import LinkPojo
@@ -56,6 +56,10 @@ class LinkValidatorWithProvidedList(threading.Thread):
_ua = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36' _ua = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36'
if _received_dict is not None: if _received_dict is not None:
_ua = _received_dict['ua'] _ua = _received_dict['ua']
_model = _received_dict['model']
_ua = "Mozilla/5.0 (Linux; Android 9; {}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.58 Mobile Safari/537.36".format(
_model)
logger.info("model: %s", _model)
self.cookie.load(self.cookie_str) self.cookie.load(self.cookie_str)
headers = { headers = {
'User-Agent': _ua, 'User-Agent': _ua,
@@ -78,7 +82,7 @@ class LinkValidatorWithProvidedList(threading.Thread):
print("will send request with ua {}".format(_ua)) print("will send request with ua {}".format(_ua))
print("will send request with cookie {}".format(self.cookie_str)) print("will send request with cookie {}".format(self.cookie_str))
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use, response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
timeout=60) timeout=60, impersonate="chrome99_android")
print(response.status_code) print(response.status_code)
if response.status_code == 200: if response.status_code == 200:
_content = response.text _content = response.text
workers/sender.py
+10 -7
View File
@@ -6,7 +6,7 @@ import re
import time import time
from http.cookies import SimpleCookie from http.cookies import SimpleCookie
from models.contact_pojo import ContactPojo from models.contact_pojo import ContactPojo
import requests from curl_cffi import requests
from db.mongo_manager import MONGO_STORE_MANAGER from db.mongo_manager import MONGO_STORE_MANAGER
from models.ReserveResultPojo import ReserveResultPojo, PublishType from models.ReserveResultPojo import ReserveResultPojo, PublishType
@@ -118,6 +118,8 @@ class Sender:
self._csrf = csrf self._csrf = csrf
_csrf = csrf _csrf = csrf
self.logger.info("model is:{}".format(model)) self.logger.info("model is:{}".format(model))
js_data.ua = "Mozilla/5.0 (Linux; Android 9; {}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.58 Mobile Safari/537.36".format(
model)
_chrome_code = get_chrome_version_from_ua(js_data.ua) _chrome_code = get_chrome_version_from_ua(js_data.ua)
_chrome_version = None _chrome_version = None
if _chrome_code == '128': if _chrome_code == '128':
@@ -128,11 +130,12 @@ class Sender:
_chrome_version = "116.0.5845.114" _chrome_version = "116.0.5845.114"
elif _chrome_code == '114': elif _chrome_code == '114':
_chrome_version = "114.0.5735.53" _chrome_version = "114.0.5735.53"
elif _chrome_code == '99':
_chrome_ua_version = '"Chromium";v="{}", "Not;A=Brand";v="24.0.0.0", "Google Chrome";v="{}"'.format( _chrome_version = "99.0.4844.58"
_chrome_version, _chrome_version) # 99: " Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"
_chrome_ua_header = '"Chromium";v="{}", "Not;A=Brand";v="24", "Google Chrome";v="{}"'.format(_chrome_code, # " Not A;Brand";v="99.0.0.0", "Chromium";v="99.0.4844.58", "Google Chrome";v="99.0.4844.58"
_chrome_code) _chrome_ua_version = 'Not A;Brand";v="99.0.0.0", "Chromium";v="99.0.4844.58", "Google Chrome";v="99.0.4844.58"'
_chrome_ua_header = 'Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"'
self.logger.info("chrome_ua_header is:{}".format(_chrome_ua_header)) self.logger.info("chrome_ua_header is:{}".format(_chrome_ua_header))
self.logger.info("_chrome_ua_version is:{}".format(_chrome_ua_version)) self.logger.info("_chrome_ua_version is:{}".format(_chrome_ua_version))
# 默认为空，也就是随机 # 默认为空，也就是随机
@@ -168,7 +171,7 @@ class Sender:
proxy_to_use = self.proxy_to_use proxy_to_use = self.proxy_to_use
# print(proxy_to_use) # print(proxy_to_use)
response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data, response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data,
timeout=15, allow_redirects=False) timeout=15, allow_redirects=False, impersonate="chrome99_android")
self.logger.info(response.status_code) self.logger.info(response.status_code)
if response.status_code == 302: if response.status_code == 302:
# add to mongodb # add to mongodb