can check link with multi threads

workers/result_link_checker.py

@@ -167,7 +167,7 @@ def start_check_links(_queue_name, link_list, logger):
     receiver.listen_to_queue(receiver.on_message)
 
 
-def check_all_links(_segment_number=100):
+def check_all_links(_segment_number=2):
     _queue_name = MORNING_DATA_CACHE_BAK
     _all_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
     _link_list = []
@@ -175,11 +175,13 @@ def check_all_links(_segment_number=100):
         if _item.url_validated is not None and _item.url_validated is True:
             _link_list.append(_item.url)
     print(_link_list)
+    # _link_list_filted = _link_list[4000:-1]
+    _link_list_filted = _link_list
     thread_list = []
     for i in range(0, _segment_number):
         logger.info("segment is {}".format(i))
-        _step = int(len(_link_list) / _segment_number)
-        _sublist = _link_list[i * _step:_step * (i + 1)]
+        _step = int(len(_link_list_filted) / _segment_number)
+        _sublist = _link_list_filted[i * _step:_step * (i + 1)]
         _thread1 = threading.Thread(target=start_check_links, args=(MORNING_DATA_CACHE_BAK, _sublist, logger))
         thread_list.append(_thread1)
         _thread1.start()