panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support bitbrowser

Browse Source
This commit is contained in:
PAN Lei
2024-12-26 11:25:31 +01:00
parent 32d2a0b52b
commit 293947fda8
1 changed files with 11 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/link_validator_with_provided_list.py
+11 -6
View File
@@ -13,7 +13,7 @@ from db.mongo_manager import MONGO_STORE_MANAGER
from models.LinkPojo import LinkPojo from models.LinkPojo import LinkPojo
from models.result_pojo import RequestResult from models.result_pojo import RequestResult
from proxy_manager.proxy_manager import ProxyManager from proxy_manager.proxy_manager import ProxyManager
from queue_message.CookiesPublisher import CookiesPublisher, MORNING_DATA_CACHE from queue_message.CookiesPublisher import CookiesPublisher, MORNING_DATA_CACHE, MORNING_DATA_CACHE_BAK
from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
from utils.AppLogging import init_logger from utils.AppLogging import init_logger
from utils.user_agent_helper import generate_headers_from_request_message from utils.user_agent_helper import generate_headers_from_request_message
@@ -28,8 +28,12 @@ logger = logging.getLogger()
def filter_link_pojo_list_with_serial(_received_dict, link_to_validate_list): def filter_link_pojo_list_with_serial(_received_dict, link_to_validate_list):
_serial = _received_dict["serial"] _serial = _received_dict["serial"]
_model = _received_dict["model"] _model = _received_dict["model"]
_to_return = filter(lambda link_pojo: link_pojo.serial == _serial, link_to_validate_list) _bit_browser_list = list(filter(lambda link_pojo: link_pojo.serial == "bitbrowser", link_to_validate_list))
return list(_to_return) if len(_bit_browser_list) > 0:
return _bit_browser_list
else:
_to_return = filter(lambda link_pojo: link_pojo.serial == _serial, link_to_validate_list)
return list(_to_return)
class LinkValidatorWithProvidedList(threading.Thread): class LinkValidatorWithProvidedList(threading.Thread):
@@ -194,7 +198,8 @@ class LinkValidatorWithProvidedList(threading.Thread):
def validate_links(cookiesPublisher, queue_name: str, link_list: list, _contact_serial_list): def validate_links(cookiesPublisher, queue_name: str, link_list: list, _contact_serial_list):
receiver = LinkValidatorWithProvidedList(cookiesPublisher=cookiesPublisher, link_list=link_list, receiver = LinkValidatorWithProvidedList(cookiesPublisher=cookiesPublisher, link_list=link_list,
queue_to_listen=queue_name, _contact_serial_list= _contact_serial_list, ip_country="FR", limit=0) queue_to_listen=queue_name, _contact_serial_list=_contact_serial_list,
ip_country="FR", limit=0)
print("{} set_up_connection".format(threading.currentThread().name)) print("{} set_up_connection".format(threading.currentThread().name))
receiver.set_up_connection() receiver.set_up_connection()
receiver.listen_to_queue(receiver.on_message) receiver.listen_to_queue(receiver.on_message)
@@ -213,7 +218,7 @@ def validate_all_links(_contact_serial_list):
# _first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))] # _first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))]
_first_25_percent_links = all_link_list _first_25_percent_links = all_link_list
_queue_name = MORNING_DATA_CACHE _queue_name = MORNING_DATA_CACHE
# # _queue_name = MORNING_DATA_CACHE_BAK # _queue_name = MORNING_DATA_CACHE_BAK
# if len(all_link_list) > divided * default_segment_number: # if len(all_link_list) > divided * default_segment_number:
# _segment_number = default_segment_number # _segment_number = default_segment_number
# else: # else:
@@ -223,7 +228,7 @@ def validate_all_links(_contact_serial_list):
# else: # else:
# _segment_number = 1 # _segment_number = 1
_thread_list = [] _thread_list = []
_segment_number =1 _segment_number = 1
for i in range(0, _segment_number): for i in range(0, _segment_number):
logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_25_percent_links))) logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_25_percent_links)))
logger.info("segment is {}".format(i)) logger.info("segment is {}".format(i))