panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of git.lpaconsulting.fr:panleicim/appointment_request

Browse Source
This commit is contained in:
PAN Lei
2025-09-18 15:36:17 +02:00
parent 474c9bf339 7eab86c910
commit d21f64bfc2
1 changed files with 15 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/link_validator_with_provided_list.py
+15 -17
View File
@@ -7,8 +7,9 @@ from datetime import datetime
from http.cookies import SimpleCookie from http.cookies import SimpleCookie
import pika import pika
from curl_cffi import requests # from curl_cffi import requests
import requests
from db.mongo_manager import MONGO_STORE_MANAGER from db.mongo_manager import MONGO_STORE_MANAGER
from models.LinkPojo import LinkPojo from models.LinkPojo import LinkPojo
from models.result_pojo import RequestResult from models.result_pojo import RequestResult
@@ -28,7 +29,8 @@ logger = logging.getLogger()
def filter_link_pojo_list_with_serial(_received_dict, link_to_validate_list): def filter_link_pojo_list_with_serial(_received_dict, link_to_validate_list):
_serial = _received_dict["serial"] _serial = _received_dict["serial"]
_model = _received_dict["model"] _model = _received_dict["model"]
_bit_browser_list = list(filter(lambda link_pojo: link_pojo.serial == "requests", link_to_validate_list)) # _bit_browser_list = list(filter(lambda link_pojo: link_pojo.serial == "requests", link_to_validate_list))
_bit_browser_list = link_to_validate_list
if len(_bit_browser_list) > 0: if len(_bit_browser_list) > 0:
return _bit_browser_list return _bit_browser_list
else: else:
@@ -77,7 +79,8 @@ class LinkValidatorWithProvidedList(threading.Thread):
print("will send request with ua {}".format(_ua)) print("will send request with ua {}".format(_ua))
print("will send request with cookie {}".format(self.cookie_str)) print("will send request with cookie {}".format(self.cookie_str))
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use, response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
timeout=60, impersonate="chrome99_android") timeout=60)
# timeout=60, impersonate="chrome99_android")
print(response.status_code) print(response.status_code)
if response.status_code == 200: if response.status_code == 200:
_content = response.text _content = response.text
@@ -181,7 +184,7 @@ class LinkValidatorWithProvidedList(threading.Thread):
else: else:
print("will ack") print("will ack")
print("will wait for 40s") print("will wait for 40s")
time.sleep(20) time.sleep(40)
ch.basic_ack(delivery_tag=method.delivery_tag) ch.basic_ack(delivery_tag=method.delivery_tag)
else: else:
print("empty list, will republish message") print("empty list, will republish message")
@@ -221,8 +224,8 @@ def validate_all_links(_contact_serial_list):
for _link in all_link_list: for _link in all_link_list:
# print("serial is "+_link.serial) # print("serial is "+_link.serial)
print("email is " + _link.email) print("email is " + _link.email)
if _link.serial == "requests": # if _link.serial == "requests":
link_to_validated.append(_link) link_to_validated.append(_link)
# get the first 50 links # get the first 50 links
if len(link_to_validated) == 0: if len(link_to_validated) == 0:
return return
@@ -230,19 +233,14 @@ def validate_all_links(_contact_serial_list):
# default_segment_number = 20 # default_segment_number = 20
_first_25_percent_links = link_to_validated[0:(int(len(all_link_list) / divided))] _first_25_percent_links = link_to_validated[0:(int(len(all_link_list) / divided))]
_first_25_percent_links = all_link_list _first_25_percent_links = all_link_list
# _queue_name = MORNING_DATA_CACHE # _queue_name = "REQUEST_DATA"
_queue_name = MORNING_DATA_CACHE_BAK _queue_name = MORNING_DATA_CACHE_BAK
# if len(all_link_list) > divided * default_segment_number: # _queue_name = MORNING_DATA_CACHE
# _segment_number = default_segment_number
# else:
# _first_25_percent_links = all_link_list
# if len(_first_25_percent_links) > divided:
# _segment_number = int(len(_first_25_percent_links) / divided)
# else:
# _segment_number = 1
_thread_list = [] _thread_list = []
if len(_first_25_percent_links) >= 20: if len(_first_25_percent_links) >= 100:
_segment_number = 20 _segment_number = 100
elif len(_first_25_percent_links) >= 40:
_segment_number = 40
else: else:
_segment_number = len(_first_25_percent_links) _segment_number = len(_first_25_percent_links)
for i in range(0, _segment_number): for i in range(0, _segment_number):