log oxylabs res proxy

workers/link_validator_with_provided_list.py

@@ -220,7 +220,7 @@ def validate_all_links():
             _segment_number = int(len(_first_25_percent_links) / divided)
         else:
             _segment_number = 1
-    last_thread = None
+    _thread_list = []
     for i in range(0, _segment_number):
         logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_25_percent_links)))
         logger.info("segment is {}".format(i))
@@ -229,10 +229,10 @@ def validate_all_links():
         _step = int(len(_first_25_percent_links) / _segment_number)
         _sublist = _first_25_percent_links[i * _step:_step * (i + 1)]
         _thread1 = threading.Thread(target=validate_links, args=(_cookiesPublisher, MORNING_DATA_CACHE_BAK, _sublist))
-        last_thread = _thread1
+        _thread_list.append(_thread1)
         _thread1.start()
-    if last_thread is not None:
-        last_thread.join()
+    for _thread in _thread_list:
+        _thread.join()
 
 
 if __name__ == '__main__':
@@ -240,5 +240,6 @@ if __name__ == '__main__':
     while True:
         print("call validate_all_links()")
         validate_all_links()
-        print("wait for 30 seconds")
-        time.sleep(30)
+        delay = random.randint(10, 30)
+        print("wait for {} seconds".format(delay))
+        time.sleep(delay)