diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index fc1b73b..318b72d 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -3,6 +3,7 @@ import logging import random import threading import time +from datetime import datetime from http.cookies import SimpleCookie import pika @@ -202,13 +203,15 @@ def validate_links(cookiesPublisher, queue_name: str, link_list: list): receiver.listen_to_queue(receiver.on_message) +# default_segment_number 并发数,决定速度 +# divided = 4,越小,一次处理得越多 def validate_all_links(): print("will get all links") all_link_list = MONGO_STORE_MANAGER.get_links_to_validate() # get the first 50 links if len(all_link_list) == 0: return - divided = 4 + divided = 3 default_segment_number = 20 _first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))] _queue_name = MORNING_DATA_CACHE_BAK @@ -241,5 +244,11 @@ if __name__ == '__main__': print("call validate_all_links()") validate_all_links() delay = random.randint(10, 30) + current_time = datetime.now() + current_hour = current_time.hour + print("Current hour ", current_time.hour) + print("Current minute ", current_time.minute) + if current_hour > 20 or current_hour < 9: + delay = 3600 print("wait for {} seconds".format(delay)) time.sleep(delay)