panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

try to use curl_cffi

Browse Source
This commit is contained in:
PAN Lei
2024-10-05 11:21:20 +02:00
parent c2db1035c4
commit f9ce820b9f
4 changed files with 29 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/link_validator_with_provided_list.py
+6 -2
View File
@@ -7,7 +7,7 @@ from datetime import datetime
from http.cookies import SimpleCookie
import pika
import requests
from curl_cffi import requests
from db.mongo_manager import MONGO_STORE_MANAGER
from models.LinkPojo import LinkPojo
@@ -56,6 +56,10 @@ class LinkValidatorWithProvidedList(threading.Thread):
_ua = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36'
if _received_dict is not None:
_ua = _received_dict['ua']
_model = _received_dict['model']
_ua = "Mozilla/5.0 (Linux; Android 9; {}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.58 Mobile Safari/537.36".format(
_model)
logger.info("model: %s", _model)
self.cookie.load(self.cookie_str)
headers = {
'User-Agent': _ua,
@@ -78,7 +82,7 @@ class LinkValidatorWithProvidedList(threading.Thread):
print("will send request with ua {}".format(_ua))
print("will send request with cookie {}".format(self.cookie_str))
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
timeout=60)
timeout=60, impersonate="chrome99_android")
print(response.status_code)
if response.status_code == 200:
_content = response.text
workers/sender.py
+10 -7
View File
@@ -6,7 +6,7 @@ import re
import time
from http.cookies import SimpleCookie
from models.contact_pojo import ContactPojo
import requests
from curl_cffi import requests
from db.mongo_manager import MONGO_STORE_MANAGER
from models.ReserveResultPojo import ReserveResultPojo, PublishType
@@ -118,6 +118,8 @@ class Sender:
self._csrf = csrf
_csrf = csrf
self.logger.info("model is:{}".format(model))
js_data.ua = "Mozilla/5.0 (Linux; Android 9; {}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.58 Mobile Safari/537.36".format(
model)
_chrome_code = get_chrome_version_from_ua(js_data.ua)
_chrome_version = None
if _chrome_code == '128':
@@ -128,11 +130,12 @@ class Sender:
_chrome_version = "116.0.5845.114"
elif _chrome_code == '114':
_chrome_version = "114.0.5735.53"
_chrome_ua_version = '"Chromium";v="{}", "Not;A=Brand";v="24.0.0.0", "Google Chrome";v="{}"'.format(
_chrome_version, _chrome_version)
_chrome_ua_header = '"Chromium";v="{}", "Not;A=Brand";v="24", "Google Chrome";v="{}"'.format(_chrome_code,
_chrome_code)
elif _chrome_code == '99':
_chrome_version = "99.0.4844.58"
# 99: " Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"
# " Not A;Brand";v="99.0.0.0", "Chromium";v="99.0.4844.58", "Google Chrome";v="99.0.4844.58"
_chrome_ua_version = 'Not A;Brand";v="99.0.0.0", "Chromium";v="99.0.4844.58", "Google Chrome";v="99.0.4844.58"'
_chrome_ua_header = 'Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"'
self.logger.info("chrome_ua_header is:{}".format(_chrome_ua_header))
self.logger.info("_chrome_ua_version is:{}".format(_chrome_ua_version))
# 默认为空，也就是随机
@@ -168,7 +171,7 @@ class Sender:
proxy_to_use = self.proxy_to_use
# print(proxy_to_use)
response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data,
timeout=15, allow_redirects=False)
timeout=15, allow_redirects=False, impersonate="chrome99_android")
self.logger.info(response.status_code)
if response.status_code == 302:
# add to mongodb