diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index 153d68e..bae1f1e 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -13,7 +13,7 @@ from db.mongo_manager import MONGO_STORE_MANAGER from models.LinkPojo import LinkPojo from models.result_pojo import RequestResult from proxy_manager.proxy_manager import ProxyManager -from queue_message.CookiesPublisher import CookiesPublisher, MORNING_DATA_CACHE +from queue_message.CookiesPublisher import CookiesPublisher, MORNING_DATA_CACHE, MORNING_DATA_CACHE_BAK from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials from utils.AppLogging import init_logger from utils.user_agent_helper import generate_headers_from_request_message @@ -28,8 +28,12 @@ logger = logging.getLogger() def filter_link_pojo_list_with_serial(_received_dict, link_to_validate_list): _serial = _received_dict["serial"] _model = _received_dict["model"] - _to_return = filter(lambda link_pojo: link_pojo.serial == _serial, link_to_validate_list) - return list(_to_return) + _bit_browser_list = list(filter(lambda link_pojo: link_pojo.serial == "bitbrowser", link_to_validate_list)) + if len(_bit_browser_list) > 0: + return _bit_browser_list + else: + _to_return = filter(lambda link_pojo: link_pojo.serial == _serial, link_to_validate_list) + return list(_to_return) class LinkValidatorWithProvidedList(threading.Thread): @@ -194,7 +198,8 @@ class LinkValidatorWithProvidedList(threading.Thread): def validate_links(cookiesPublisher, queue_name: str, link_list: list, _contact_serial_list): receiver = LinkValidatorWithProvidedList(cookiesPublisher=cookiesPublisher, link_list=link_list, - queue_to_listen=queue_name, _contact_serial_list= _contact_serial_list, ip_country="FR", limit=0) + queue_to_listen=queue_name, _contact_serial_list=_contact_serial_list, + ip_country="FR", limit=0) print("{} set_up_connection".format(threading.currentThread().name)) receiver.set_up_connection() receiver.listen_to_queue(receiver.on_message) @@ -213,7 +218,7 @@ def validate_all_links(_contact_serial_list): # _first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))] _first_25_percent_links = all_link_list _queue_name = MORNING_DATA_CACHE - # # _queue_name = MORNING_DATA_CACHE_BAK + # _queue_name = MORNING_DATA_CACHE_BAK # if len(all_link_list) > divided * default_segment_number: # _segment_number = default_segment_number # else: @@ -223,7 +228,7 @@ def validate_all_links(_contact_serial_list): # else: # _segment_number = 1 _thread_list = [] - _segment_number =1 + _segment_number = 1 for i in range(0, _segment_number): logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_25_percent_links))) logger.info("segment is {}".format(i))