panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

can check link with multi threads

Browse Source
This commit is contained in:
PAN Lei
2024-05-17 22:47:19 +02:00
parent 7cacb0aa56
commit d7aafb9954
1 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/result_link_checker.py
+5 -3
View File
@@ -167,7 +167,7 @@ def start_check_links(_queue_name, link_list, logger):
receiver.listen_to_queue(receiver.on_message) receiver.listen_to_queue(receiver.on_message)
def check_all_links(_segment_number=100): def check_all_links(_segment_number=2):
_queue_name = MORNING_DATA_CACHE_BAK _queue_name = MORNING_DATA_CACHE_BAK
_all_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day() _all_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
_link_list = [] _link_list = []
@@ -175,11 +175,13 @@ def check_all_links(_segment_number=100):
if _item.url_validated is not None and _item.url_validated is True: if _item.url_validated is not None and _item.url_validated is True:
_link_list.append(_item.url) _link_list.append(_item.url)
print(_link_list) print(_link_list)
# _link_list_filted = _link_list[4000:-1]
_link_list_filted = _link_list
thread_list = [] thread_list = []
for i in range(0, _segment_number): for i in range(0, _segment_number):
logger.info("segment is {}".format(i)) logger.info("segment is {}".format(i))
_step = int(len(_link_list) / _segment_number) _step = int(len(_link_list_filted) / _segment_number)
_sublist = _link_list[i * _step:_step * (i + 1)] _sublist = _link_list_filted[i * _step:_step * (i + 1)]
_thread1 = threading.Thread(target=start_check_links, args=(MORNING_DATA_CACHE_BAK, _sublist, logger)) _thread1 = threading.Thread(target=start_check_links, args=(MORNING_DATA_CACHE_BAK, _sublist, logger))
thread_list.append(_thread1) thread_list.append(_thread1)
_thread1.start() _thread1.start()