diff --git a/workers/result_link_checker.py b/workers/result_link_checker.py index 24bbf8f..13193ac 100644 --- a/workers/result_link_checker.py +++ b/workers/result_link_checker.py @@ -167,7 +167,7 @@ def start_check_links(_queue_name, link_list, logger): receiver.listen_to_queue(receiver.on_message) -def check_all_links(_segment_number=100): +def check_all_links(_segment_number=2): _queue_name = MORNING_DATA_CACHE_BAK _all_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day() _link_list = [] @@ -175,11 +175,13 @@ def check_all_links(_segment_number=100): if _item.url_validated is not None and _item.url_validated is True: _link_list.append(_item.url) print(_link_list) + # _link_list_filted = _link_list[4000:-1] + _link_list_filted = _link_list thread_list = [] for i in range(0, _segment_number): logger.info("segment is {}".format(i)) - _step = int(len(_link_list) / _segment_number) - _sublist = _link_list[i * _step:_step * (i + 1)] + _step = int(len(_link_list_filted) / _segment_number) + _sublist = _link_list_filted[i * _step:_step * (i + 1)] _thread1 = threading.Thread(target=start_check_links, args=(MORNING_DATA_CACHE_BAK, _sublist, logger)) thread_list.append(_thread1) _thread1.start()