try to wait for more after 20h and before 9 h

workers/link_validator_with_provided_list.py

@@ -3,6 +3,7 @@ import logging
 import random
 import threading
 import time
+from datetime import datetime
 from http.cookies import SimpleCookie
 
 import pika
@@ -202,13 +203,15 @@ def validate_links(cookiesPublisher, queue_name: str, link_list: list):
     receiver.listen_to_queue(receiver.on_message)
 
 
+# default_segment_number 并发数，决定速度
+# divided = 4，越小，一次处理得越多
 def validate_all_links():
     print("will get all links")
     all_link_list = MONGO_STORE_MANAGER.get_links_to_validate()
     # get the first 50 links
     if len(all_link_list) == 0:
         return
-    divided = 4
+    divided = 3
     default_segment_number = 20
     _first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))]
     _queue_name = MORNING_DATA_CACHE_BAK
@@ -241,5 +244,11 @@ if __name__ == '__main__':
         print("call validate_all_links()")
         validate_all_links()
         delay = random.randint(10, 30)
+        current_time = datetime.now()
+        current_hour = current_time.hour
+        print("Current hour ", current_time.hour)
+        print("Current minute ", current_time.minute)
+        if current_hour > 20 or current_hour < 9:
+            delay = 3600
         print("wait for {} seconds".format(delay))
         time.sleep(delay)