From c3dd17cf1841f4211b46d326b88262516a515608 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 10 May 2024 14:33:19 +0200 Subject: [PATCH] update mails configurations --- queue_message/appointmentrequestsender.py | 2 +- workers/link_validator_with_provided_list.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/queue_message/appointmentrequestsender.py b/queue_message/appointmentrequestsender.py index fcf4880..2b4aadc 100644 --- a/queue_message/appointmentrequestsender.py +++ b/queue_message/appointmentrequestsender.py @@ -234,7 +234,7 @@ class AppointmentRequestSender(threading.Thread): self.logger.info("will get mail from " + mail.mail) _mail_list_filtered.append(mail) self.logger.info("will call find_links_to_validate_from_mail_list, size = " + str(len(_mail_list_filtered))) - find_links_to_validate_from_mail_list(_mail_list_filtered) + find_links_to_validate_from_mail_list(_mail_list_filtered, self.logger) self.already_read_emails = True else: self.logger.info("already read emails, is there any contacts to use") diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index 836344a..4b51c73 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -12,7 +12,8 @@ from db.mongo_manager import MONGO_STORE_MANAGER from models.LinkPojo import LinkPojo from models.result_pojo import RequestResult from proxy_manager.proxy_manager import ProxyManager -from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST, TEST_QUEUE, SHARED_OBJECT +from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST, TEST_QUEUE, SHARED_OBJECT, \ + MORNING_DATA_CACHE_BAK from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials from utils.AppLogging import init_logger from workers.proxies_constants import PROXY_LIST_FR @@ -204,18 +205,18 @@ def validate_links(cookiesPublisher, queue_name: str, link_list: list): def validate_all_links(): all_link_list = MONGO_STORE_MANAGER.get_links_to_validate() # get the first 50 links - _first_50_links = all_link_list[0:50] - _queue_name = TEST_QUEUE + _first_50_links = all_link_list[0:(int(len(all_link_list) / 4))] + _queue_name = MORNING_DATA_CACHE_BAK _segment_number = 20 last_thread = None for i in range(0, _segment_number): logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_50_links))) logger.info("segment is {}".format(i)) - _cookiesPublisher = CookiesPublisher(queue_name=TEST_QUEUE) + _cookiesPublisher = CookiesPublisher(queue_name=_queue_name) _cookiesPublisher.set_up_connection() _step = int(len(_first_50_links) / _segment_number) _sublist = _first_50_links[i * _step:_step * (i + 1)] - _thread1 = threading.Thread(target=validate_links, args=(_cookiesPublisher, SHARED_OBJECT, _sublist)) + _thread1 = threading.Thread(target=validate_links, args=(_cookiesPublisher, MORNING_DATA_CACHE_BAK, _sublist)) last_thread = _thread1 _thread1.start() last_thread.join() @@ -223,4 +224,7 @@ def validate_all_links(): if __name__ == '__main__': # generate test data - validate_all_links() + while True: + validate_all_links() + print("wait for 30 seconds") + time.sleep(30)