diff --git a/src/mail/mail_reader.py b/src/mail/mail_reader.py index 5e47d88..75eeffa 100644 --- a/src/mail/mail_reader.py +++ b/src/mail/mail_reader.py @@ -155,7 +155,7 @@ def read_mails(): mail_reader = MailReader(mail.mail, mail.password) executor.submit(mail_reader.read_emails, mails_messages) - with ThreadPoolExecutor(max_workers=10) as executor: + with ThreadPoolExecutor(max_workers=20) as executor: for mail in mails_messages: match = re.search(VALIDATION_URL_REGEX, mail.body) if match: diff --git a/src/utils/excel_reader.py b/src/utils/excel_reader.py index d53cb41..debc852 100644 --- a/src/utils/excel_reader.py +++ b/src/utils/excel_reader.py @@ -32,9 +32,10 @@ class ExcelHelper: user_agent_list = [] for user_agent_dict in user_agent_dict_list: user_agent_str = user_agent_dict['user_agent'] - if 'Android' in user_agent_str: - if 'Mac OS' not in user_agent_str: - user_agent_list.append(user_agent_dict['user_agent']) + if 'Mozilla' in user_agent_str: + if 'Android 5.1' in user_agent_str: + if 'Mac OS' not in user_agent_str: + user_agent_list.append(user_agent_dict['user_agent']) print(user_agent_list) def read_contacts(self, file_name=CONTACT_LIST_FILE) -> list: diff --git a/src/workers/commandor_page.py b/src/workers/commandor_page.py index 531b01c..9a5f362 100644 --- a/src/workers/commandor_page.py +++ b/src/workers/commandor_page.py @@ -120,24 +120,50 @@ class CommandorPage: def start_browser(self, proxy, pwright, device) -> Union[str, None]: try: self.browser = pwright.firefox.launch(headless=self.headless, timeout=PAGE_TIMEOUT, proxy=proxy) - userAgent = random.choice(params.firefox_user_agent_list) + # userAgent = random.choice(params.firefox_user_agent_list) + simulated_mobile = pwright.devices[device] + userAgent = simulated_mobile['user_agent'] print("user_agent is " + userAgent) # context = self.browser.new_context(**simulated_mobile, locale='fr-FR') context = self.browser.new_context(user_agent=userAgent, locale='fr-FR') self.page = context.new_page() # hide webdriver information self.page.add_init_script("""() => { - Object.defineProperty(navigator,'webdriver',{get: () => undefined}); - Object.defineProperty(navigator, 'platform', { - get: () => { - return "iPhone"; - }}); -} - """) + Object.defineProperty(navigator,'webdriver',{get: () => undefined}); + Object.defineProperty(navigator, 'platform', { + get: () => { + return "iPhone"; + }}); + } + """) self.page.on("load", self._on_page_loaded) self.page.on("response", self.handle_response) self.page.goto(RDV_URL, timeout=PAGE_TIMEOUT) return self.page.content() + # self.browser = pwright.webkit.launch(headless=self.headless, timeout=PAGE_TIMEOUT, proxy=proxy) + # self.logger.info("模拟设备: " + device) + # simulated_mobile = pwright.devices[device] + # context = self.browser.new_context(**simulated_mobile, locale='fr-FR') + # self.page = context.new_page() + # # hide webdriver information + # self.page.add_init_script("""() => { + # Object.defineProperty(navigator,'webdriver',{get: () => undefined}); + # Object.defineProperty(navigator, 'platform', { + # get: () => { + # return "iPhone"; + # }}); + # } + # """) + # self.page.on("load", self._on_page_loaded) + # self.page.on("response", self.handle_response) + # self.page.goto(RDV_URL, timeout=PAGE_TIMEOUT) + # captcha_url = "geo.captcha-delivery.com/captcha" + # if captcha_url in self.page.content(): + # self.logger.info("will close browser") + # self.browser.close() + # return None + # else: + # return self.page.content() except Exception as error: params.oracle_log_sender.send_error(str(error)) traceback.print_exc(*sys.exc_info()) diff --git a/src/workers/link_validator.py b/src/workers/link_validator.py index a948020..cb752e9 100644 --- a/src/workers/link_validator.py +++ b/src/workers/link_validator.py @@ -77,7 +77,13 @@ class LinkValidator: """) self.page.on("load", self._on_page_loaded) self.page.goto(self.link, timeout=PAGE_TIMEOUT) - return self.page.content() + captcha_url = "geo.captcha-delivery.com/captcha" + if captcha_url in self.page.content(): + self.logger.info("will close browser") + self.browser.close() + return None + else: + return self.page.content() except Exception as error: params.oracle_log_sender.send_error(str(error)) traceback.print_exc(*sys.exc_info())