From 7e593120d706e8e23c8214b4d2b21ca24e8a218a Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Sat, 10 Dec 2022 11:53:04 +0100 Subject: [PATCH] can use old programme --- main.py | 2 +- src/mail/mail_reader.py | 24 +++++++++--------- src/params.py | 46 +++++++++++++++++++---------------- src/workers/commandor_page.py | 3 ++- 4 files changed, 40 insertions(+), 35 deletions(-) diff --git a/main.py b/main.py index 212e865..fb5ba0f 100644 --- a/main.py +++ b/main.py @@ -62,7 +62,7 @@ def get_proxy(proxy_type=ProxyType.RESIDENTIAL): if __name__ == '__main__': # 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机，传入1的时候是总店 - start_book(800, 1500, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=False, max_workers=5, + start_book(2, 2600, store_choose_state=0, mode=ModeEnum.AUTOMATIC, headless=False, max_workers=8, proxy_type=ProxyType.RESIDENTIAL) # start_book(828, 857, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=True) # start_book(1210, 1211, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=False, max_workers=3, diff --git a/src/mail/mail_reader.py b/src/mail/mail_reader.py index 5ff744c..cf1bf90 100644 --- a/src/mail/mail_reader.py +++ b/src/mail/mail_reader.py @@ -193,18 +193,18 @@ def read_mails(): mail_reader = MailReader(mail.mail, mail.password) executor.submit(mail_reader.read_emails, mails_messages) - # with ThreadPoolExecutor(max_workers=10) as executor: - for mail in mails_messages: - match = re.search(VALIDATION_URL_REGEX, mail.body) - if match: - url = match.group(0) - if need_to_valid_url(url, successful_items): - MONGO_STORE_MANAGER.save_links_to_validate(url) - # url_validator = LinkValidator(url) - # print("need to validate url: " + url) - # executor.submit(url_validator.start_page, params.get_proxy(ProxyType.OXYLABS), False) - else: - print("do not need to click url --> {}".format(mail.mail_address)) + with ThreadPoolExecutor(max_workers=10) as executor: + for mail in mails_messages: + match = re.search(VALIDATION_URL_REGEX, mail.body) + if match: + url = match.group(0) + if need_to_valid_url(url, successful_items): + MONGO_STORE_MANAGER.save_links_to_validate(url) + url_validator = LinkValidator(url) + # print("need to validate url: " + url) + executor.submit(url_validator.start_page, params.get_proxy(ProxyType.OXYLABS), True) + else: + print("do not need to click url --> {}".format(mail.mail_address)) # check whether the url has already been clicked diff --git a/src/params.py b/src/params.py index a83f3ea..8df40c1 100644 --- a/src/params.py +++ b/src/params.py @@ -28,27 +28,31 @@ def get_proxy_name_prefix() -> str: def get_proxy(proxy_type: ProxyType): - if proxy_type == ProxyType.RESIDENTIAL: - # random_id_number = get_random_id_number_for_proxy() - # proxy_username = get_proxy_name_prefix() + random_id_number - proxy = { - "server": BRIGHT_DATA_PROXY_SERVER, - "username": BRIGHT_DATA_PROXY_USERNAME, - "password": BRIGHT_DATA_PROXY_PASSWORD - } - elif proxy_type == ProxyType.OXYLABS: - proxy = { - "server": OXYLABS_PROXY_SERVER, - "username": OXYLABS_DATA_PROXY_USERNAME, - "password": OXYLABS_DATA_PROXY_PASSWORD - } - else: - proxy = { - "server": BRIGHT_DATA_PROXY_SERVER, - "username": BRIGHT_DATA_MOBILE_PROXY_USERNAME, - "password": BRIGHT_DATA_MOBILE_PROXY_PASSWORD - } - return proxy + # if proxy_type == ProxyType.RESIDENTIAL: + # # random_id_number = get_random_id_number_for_proxy() + # # proxy_username = get_proxy_name_prefix() + random_id_number + # proxy = { + # "server": BRIGHT_DATA_PROXY_SERVER, + # "username": BRIGHT_DATA_PROXY_USERNAME, + # "password": BRIGHT_DATA_PROXY_PASSWORD + # } + # elif proxy_type == ProxyType.OXYLABS: + # proxy = { + # "server": OXYLABS_PROXY_SERVER, + # "username": OXYLABS_DATA_PROXY_USERNAME, + # "password": OXYLABS_DATA_PROXY_PASSWORD + # } + # else: + # proxy = { + # "server": BRIGHT_DATA_PROXY_SERVER, + # "username": BRIGHT_DATA_MOBILE_PROXY_USERNAME, + # "password": BRIGHT_DATA_MOBILE_PROXY_PASSWORD + # } + return { + "server": OXYLABS_PROXY_SERVER, + "username": OXYLABS_DATA_PROXY_USERNAME, + "password": OXYLABS_DATA_PROXY_PASSWORD + } def get_random_id_number_for_proxy() -> str: diff --git a/src/workers/commandor_page.py b/src/workers/commandor_page.py index d9300af..242541a 100644 --- a/src/workers/commandor_page.py +++ b/src/workers/commandor_page.py @@ -36,6 +36,7 @@ BLANK_URL = "about:blank" CONFIRMED_MESSAGE = "Your request for a Leather Goods appointment has been registered" CONFIRMED_MESSAGE_FR = "Votre demande de rendez-vous Maroquinerie a bien été enregistrée et nous vous en remercions." MESSAGE_URL_VALIDATION_FR = "Nous avons envoyé un lien par e-mail." +MESSAGE_URL_VALIDATION = "Please click on the link we sent by email to confirm your request." DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today." DOUBLE_REQUEST_ERROR_MESSAGE_FR = "Une demande avec les données saisies a déjà été validée aujourd’hui." TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests" @@ -204,7 +205,7 @@ class CommandorPage: self.fill_fields() try: message = self.page.content() - if CONFIRMED_MESSAGE_FR in message or MESSAGE_URL_VALIDATION_FR in message: + if CONFIRMED_MESSAGE_FR in message or MESSAGE_URL_VALIDATION_FR in message or MESSAGE_URL_VALIDATION in message: # publish the successful message self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url) self.get_errors()