From ff20c1a003193b60fd898b2b1389b0d796fba25d Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Sat, 11 May 2024 11:47:11 +0200 Subject: [PATCH] same cookie use same ip address --- workers/link_validator_with_provided_list.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index f338eca..cd01e01 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -210,11 +210,14 @@ def validate_all_links(): divided = 4 _first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))] _queue_name = MORNING_DATA_CACHE_BAK - if len(_first_25_percent_links) > 40: + if len(all_link_list) > 40: _segment_number = 20 else: _first_25_percent_links = all_link_list - _segment_number = int(len(_first_25_percent_links) / divided) + if len(_first_25_percent_links) > divided: + _segment_number = int(len(_first_25_percent_links) / divided) + else: + _segment_number = 1 last_thread = None for i in range(0, _segment_number): logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_25_percent_links))) @@ -226,7 +229,8 @@ def validate_all_links(): _thread1 = threading.Thread(target=validate_links, args=(_cookiesPublisher, MORNING_DATA_CACHE_BAK, _sublist)) last_thread = _thread1 _thread1.start() - last_thread.join() + if last_thread is not None: + last_thread.join() if __name__ == '__main__':