diff --git a/appointment.py b/appointment.py index ff6b645..906c84d 100644 --- a/appointment.py +++ b/appointment.py @@ -9,8 +9,8 @@ KEY_START_NUMBER = "KEY_START_NUMBER" KEY_END_NUMBER = "KEY_END_NUMBER" KEY_MAX_WORKERS = "KEY_MAX_WORKERS" KEY_RANDOM = "KEY_RANDOM" -KEY_PROXY_BRIGHTDATA = "KEY_PROXY_BRIGHTDATA" -KEY_PROXY_NETNUT = "KEY_PROXY_NETNUT" +KEY_PROXY_MOBILE = "KEY_PROXY_MOBILE" +KEY_PROXY_RESIDENTIAL = "KEY_PROXY_RESIDENTIAL" KEY_FAUBOURG = "KEY_FAUBOURG" KEY_GEORGE = "KEY_GEORGE" KEY_SEVRES = "KEY_SEVRES" @@ -39,8 +39,8 @@ store_settings_column = [ ] proxy_settings_column = [ [sg.Text("代理ip池")], - [sg.Radio('亮数据', group_id=GROUP_PROXY, key=KEY_PROXY_BRIGHTDATA, default=True)], - [sg.Radio('Netnut', group_id=GROUP_PROXY, key=KEY_PROXY_NETNUT, default=False)], + [sg.Radio('住宅IP', group_id=GROUP_PROXY, key=KEY_PROXY_RESIDENTIAL, default=True)], + [sg.Radio('手机IP', group_id=GROUP_PROXY, key=KEY_PROXY_MOBILE, default=False)], ] mode_settings_column = [ @@ -82,11 +82,11 @@ while True: elif values[KEY_SEVRES]: store_type = 3 - proxy_type = ProxyType.BRIGHT_DATA - if values[KEY_PROXY_NETNUT]: - proxy_type = ProxyType.NETNUT - elif values[KEY_PROXY_BRIGHTDATA]: - proxy_type = ProxyType.BRIGHT_DATA + proxy_type = ProxyType.RESIDENTIAL + if values[KEY_PROXY_RESIDENTIAL]: + proxy_type = ProxyType.RESIDENTIAL + elif values[KEY_PROXY_MOBILE]: + proxy_type = ProxyType.MOBILE if values[KEY_AUTOMATIC]: mode = ModeEnum.AUTOMATIC diff --git a/requirements.txt b/requirements.txt index 872d413..6bf88bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,19 @@ -dataclasses_json==0.5.7 -firebase_admin==5.2.0 -pandas~=1.3.5 -playwright==1.25.0 -dataclasses~=0.6 -SpeechRecognition==3.8.1 -pymongo==4.1.1 -anticaptchaofficial==1.0.43 -oci~=2.54.1 -XlsxWriter~=3.0.3 -playwright-stealth==1.0.5 -boto3~=1.21.13 -openpyxl==3.0.9 -google-cloud-firestore==2.4.0 -PySimpleGUI==4.60.1 -SQLAlchemy~=1.4.37 -requests~=2.27.1 -sqlalchemy_utils~=0.38.2 +dataclasses_json==0.5.7 +firebase_admin==5.2.0 +pandas~=1.3.5 +playwright==1.25.0 +dataclasses~=0.6 +SpeechRecognition==3.8.1 +pymongo==4.1.1 +anticaptchaofficial==1.0.43 +oci~=2.54.1 +XlsxWriter~=3.0.3 +playwright-stealth==1.0.5 +boto3~=1.21.13 +openpyxl==3.0.9 +google-cloud-firestore==2.4.0 +PySimpleGUI==4.60.1 +SQLAlchemy~=1.4.37 +requests~=2.27.1 +sqlalchemy_utils~=0.38.2 Mako~=1.2.0 \ No newline at end of file diff --git a/src/mail/mail_reader.py b/src/mail/mail_reader.py index 5e56979..34dad3e 100644 --- a/src/mail/mail_reader.py +++ b/src/mail/mail_reader.py @@ -1,173 +1,173 @@ -import datetime -import email -import imaplib -import re -from concurrent.futures import ThreadPoolExecutor -from email.header import decode_header -from email.message import Message - -from builtins import list - -from src import params -from src.db.mongo_manager import MONGO_STORE_MANAGER -from src.pojo.mail.mail_pojo import MailPojo, MailAddress -from src.proxy.proxy_type import ProxyType -from src.workers.link_validator import LinkValidator - -AOL_IMAP_SERVER = "imap.aol.com" -IMAP_SERVER_163 = "imap.163.com" -VALIDATION_URL_SUBJECT = 'Validation de votre demande de rendez-vous' -VALIDATION_URL_REGEX = """https:\/\/rendezvousparis.hermes.com\/client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+""" -HERMES_EMAIL = "no-reply@hermes.com" - -DOMAIN_163 = "163.com" -date_format = "%d-%b-%Y" # DD-Mon-YYYY e.g., 3-Mar-2014 - - -class MailReader(): - def __init__(self, login, password): - self.login = login - self.password = password - - @staticmethod - def show_folders(imap): - for i in imap.list()[1]: - l = i.decode().split(' "/" ') - print(l[0] + " = " + l[1]) - - def read_emails(self, mails_messages: list) -> list: - # create an IMAP4 class with SSL - if DOMAIN_163 in self.login: - imap = imaplib.IMAP4_SSL(IMAP_SERVER_163) - else: - imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER) - # authenticate - imap.login(self.login, self.password) - mail_list = [] - print("read mails from {}".format(self.login)) - # self.show_folders(imap) - # total number of emails - # get mails from inbox - # (\Archive \HasNoChildren) = "Archive" - # (\Junk \HasNoChildren) = "Bulk" - # (\Drafts \HasNoChildren) = "Draft" - # (\HasNoChildren) = "Inbox" - # (\Sent \HasNoChildren) = "Sent" - # (\Trash \HasNoChildren) = "Trash" - mail_list.extend(self._get_messages_from_folder(imap)) - # mail_list.extend(self._get_messages_from_folder(imap, folder="Bulk")) - # close the connection and logout - imap.close() - imap.logout() - mails_messages.extend(mail_list) - return mail_list - - def _get_messages_from_folder(self, imap, folder="INBOX") -> list: - imap.select(folder) - mail_messages = [] - typ, data = imap.search(None, '(SUBJECT "{}" SINCE "{}")'.format(VALIDATION_URL_SUBJECT, - datetime.datetime.today().strftime( - date_format))) - for i in data[0].split(): - # fetch the email message by ID - res, msg = imap.fetch(i.decode("utf-8"), "(RFC822)") - body = '' - for response in msg: - if isinstance(response, tuple): - # parse a bytes email into a message object - msg = email.message_from_bytes(response[1]) - # decode the email subject - subject, subject_encoded = decode_header(msg["Subject"])[0] - received_date = msg["Date"] - if isinstance(subject, bytes): - # if it's a bytes, decode to str - subject = subject.decode(subject_encoded) - # decode email sender - from_address, subject_encoded = decode_header(msg.get("From"))[0] - if isinstance(from_address, bytes): - from_address = from_address.decode(subject_encoded) - print("From:", from_address) - print("Subject:", subject) - # if the email message is multipart - if msg.is_multipart(): - # iterate over email parts - for part in msg.walk(): - try: - # get the email body - payloads = part.get_payload() - if isinstance(payloads, list): - for payload in payloads: - if isinstance(payload, Message): - body = body + payload.get_payload(decode=True).decode("iso-8859-1") - # print(body) - except Exception as Error: - print(Error) - else: - body = msg.get_payload(decode=True).decode() - print(body) - if VALIDATION_URL_SUBJECT in subject: - mail = MailPojo(subject=subject, body=body, from_address=from_address) - mail_messages.append(mail) - return mail_messages - - -def clean(text): - # clean text for creating a folder - return "".join(c if c.isalnum() else "_" for c in text) - - -def need_to_valid_url(url: str, successful_items) -> bool: - print("url is :" + url) - parts = url.split('/') - id = parts[5] - if len(id) == 6: - for item in successful_items: - # if item.url_validated is not None: - # print("id:{}, status:{} ".format(id, str(item.url_validated))) - if item.id == id: - if item.url_validated is not None: - return not item.url_validated - else: - # if url_validated is None - return True - # return True by default - return False - else: - print("id not valid:{}".format(id)) - return False - - -def read_mails(): - # get email address - mail_list = MONGO_STORE_MANAGER.get_destination_emails() - # # mail_address1 = MailAddress(mail="appointment2022@aol.com", password="gyilpmvyyvlcaviq") - # # mail_address2 = MailAddress(mail="chenpeijun@aol.com", password="ytifuwguknzifqyb") - # # mail_address2 = MailAddress(mail="sdfgfhgf1986@aol.com", password="fjwcgvhxxlywqfwm") - # # mail_address3 = MailAddress(mail="ciyuexie@aol.com", password="czezlmmyypokdfce") - # # mail_address4 = MailAddress(mail="hongjiang176@aol.com", password="ftzpscgzvwneelmn") - # mail_address4 = MailAddress(mail="ribka_puchkova@aol.com", password="aqvsozoyifbixtdc") - # mail_list = [mail_address3, mail_address2, mail_address1, mail_address4] - # mail_list = [mail_address4] - successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day() - mails_messages = [] - with ThreadPoolExecutor(max_workers=20) as executor: - for mail in mail_list: - mail_reader = MailReader(mail.mail, mail.password) - executor.submit(mail_reader.read_emails, mails_messages) - - with ThreadPoolExecutor(max_workers=20) as executor: - for mail in mails_messages: - match = re.search(VALIDATION_URL_REGEX, mail.body) - if match: - url = match.group(0) - if need_to_valid_url(url, successful_items): - url_validator = LinkValidator(url) - print("need to validate url: " + url) - executor.submit(url_validator.start_page, params.get_proxy(ProxyType.BRIGHT_DATA), False) - else: - print("do not need to click url --> {}".format(mail.mail_address)) - - -# check whether the url has already been clicked -if __name__ == '__main__': - read_mails() +import datetime +import email +import imaplib +import re +from concurrent.futures import ThreadPoolExecutor +from email.header import decode_header +from email.message import Message + +from builtins import list + +from src import params +from src.db.mongo_manager import MONGO_STORE_MANAGER +from src.pojo.mail.mail_pojo import MailPojo +from src.proxy.proxy_type import ProxyType +from src.workers.link_validator import LinkValidator + +AOL_IMAP_SERVER = "imap.aol.com" +IMAP_SERVER_163 = "imap.163.com" +VALIDATION_URL_SUBJECT = 'Validation de votre demande de rendez-vous' +VALIDATION_URL_REGEX = """https:\/\/rendezvousparis.hermes.com\/client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+""" +HERMES_EMAIL = "no-reply@hermes.com" + +DOMAIN_163 = "163.com" +date_format = "%d-%b-%Y" # DD-Mon-YYYY e.g., 3-Mar-2014 + + +class MailReader(): + def __init__(self, login, password): + self.login = login + self.password = password + + @staticmethod + def show_folders(imap): + for i in imap.list()[1]: + l = i.decode().split(' "/" ') + print(l[0] + " = " + l[1]) + + def read_emails(self, mails_messages: list) -> list: + # create an IMAP4 class with SSL + if DOMAIN_163 in self.login: + imap = imaplib.IMAP4_SSL(IMAP_SERVER_163) + else: + imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER) + # authenticate + imap.login(self.login, self.password) + mail_list = [] + print("read mails from {}".format(self.login)) + # self.show_folders(imap) + # total number of emails + # get mails from inbox + # (\Archive \HasNoChildren) = "Archive" + # (\Junk \HasNoChildren) = "Bulk" + # (\Drafts \HasNoChildren) = "Draft" + # (\HasNoChildren) = "Inbox" + # (\Sent \HasNoChildren) = "Sent" + # (\Trash \HasNoChildren) = "Trash" + mail_list.extend(self._get_messages_from_folder(imap)) + # mail_list.extend(self._get_messages_from_folder(imap, folder="Bulk")) + # close the connection and logout + imap.close() + imap.logout() + mails_messages.extend(mail_list) + return mail_list + + def _get_messages_from_folder(self, imap, folder="INBOX") -> list: + imap.select(folder) + mail_messages = [] + typ, data = imap.search(None, '(SUBJECT "{}" SINCE "{}")'.format(VALIDATION_URL_SUBJECT, + datetime.datetime.today().strftime( + date_format))) + for i in data[0].split(): + # fetch the email message by ID + res, msg = imap.fetch(i.decode("utf-8"), "(RFC822)") + body = '' + for response in msg: + if isinstance(response, tuple): + # parse a bytes email into a message object + msg = email.message_from_bytes(response[1]) + # decode the email subject + subject, subject_encoded = decode_header(msg["Subject"])[0] + received_date = msg["Date"] + if isinstance(subject, bytes): + # if it's a bytes, decode to str + subject = subject.decode(subject_encoded) + # decode email sender + from_address, subject_encoded = decode_header(msg.get("From"))[0] + if isinstance(from_address, bytes): + from_address = from_address.decode(subject_encoded) + print("From:", from_address) + print("Subject:", subject) + # if the email message is multipart + if msg.is_multipart(): + # iterate over email parts + for part in msg.walk(): + try: + # get the email body + payloads = part.get_payload() + if isinstance(payloads, list): + for payload in payloads: + if isinstance(payload, Message): + body = body + payload.get_payload(decode=True).decode("iso-8859-1") + # print(body) + except Exception as Error: + print(Error) + else: + body = msg.get_payload(decode=True).decode() + print(body) + if VALIDATION_URL_SUBJECT in subject: + mail = MailPojo(subject=subject, body=body, from_address=from_address) + mail_messages.append(mail) + return mail_messages + + +def clean(text): + # clean text for creating a folder + return "".join(c if c.isalnum() else "_" for c in text) + + +def need_to_valid_url(url: str, successful_items) -> bool: + print("url is :" + url) + parts = url.split('/') + id = parts[5] + if len(id) == 6: + for item in successful_items: + # if item.url_validated is not None: + # print("id:{}, status:{} ".format(id, str(item.url_validated))) + if item.id == id: + if item.url_validated is not None: + return not item.url_validated + else: + # if url_validated is None + return True + # return True by default + return False + else: + print("id not valid:{}".format(id)) + return False + + +def read_mails(): + # get email address + mail_list = MONGO_STORE_MANAGER.get_destination_emails() + # # mail_address1 = MailAddress(mail="appointment2022@aol.com", password="gyilpmvyyvlcaviq") + # # mail_address2 = MailAddress(mail="chenpeijun@aol.com", password="ytifuwguknzifqyb") + # # mail_address2 = MailAddress(mail="sdfgfhgf1986@aol.com", password="fjwcgvhxxlywqfwm") + # # mail_address3 = MailAddress(mail="ciyuexie@aol.com", password="czezlmmyypokdfce") + # # mail_address4 = MailAddress(mail="hongjiang176@aol.com", password="ftzpscgzvwneelmn") + # mail_address4 = MailAddress(mail="ribka_puchkova@aol.com", password="aqvsozoyifbixtdc") + # mail_list = [mail_address3, mail_address2, mail_address1, mail_address4] + # mail_list = [mail_address4] + successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day() + mails_messages = [] + with ThreadPoolExecutor(max_workers=20) as executor: + for mail in mail_list: + mail_reader = MailReader(mail.mail, mail.password) + executor.submit(mail_reader.read_emails, mails_messages) + + with ThreadPoolExecutor(max_workers=20) as executor: + for mail in mails_messages: + match = re.search(VALIDATION_URL_REGEX, mail.body) + if match: + url = match.group(0) + if need_to_valid_url(url, successful_items): + url_validator = LinkValidator(url) + print("need to validate url: " + url) + executor.submit(url_validator.start_page, params.get_proxy(ProxyType.RESIDENTIAL), True) + else: + print("do not need to click url --> {}".format(mail.mail_address)) + + +# check whether the url has already been clicked +if __name__ == '__main__': + read_mails() diff --git a/src/params.py b/src/params.py index d957eae..b4a245c 100644 --- a/src/params.py +++ b/src/params.py @@ -1,259 +1,60 @@ -import random -import string - -from src.logs.LogSender import LogSender -from src.proxy.proxy_type import ProxyType - -oracle_log_sender = LogSender() - -# proxy -PROXY_SERVER = "http://gw.ntnt.io:5959" -PROXY_PASSWORD = "94sY7zwBG13i" - -BRIGHT_DATA_PROXY_SERVER = "http://zproxy.lum-superproxy.io:22225" -BRIGHT_DATA_PROXY_USERNAME = "lum-customer-c_daabba94-zone-residential-country-fr" -BRIGHT_DATA_MOBILE_PROXY_USERNAME = "lum-customer-c_daabba94-zone-mobile-country-fr-mobile" -BRIGHT_DATA_PROXY_PASSWORD = "9dwmh54u3bbh" -BRIGHT_DATA_MOBILE_PROXY_PASSWORD = "fk5f7c2z2c19" -PROXY_NAME_PREFIX_RES = "panleicim-res-fr-" -PROXY_NAME_PREFIX_CC = "panleicim-cc-fr-" - - -def get_proxy_name_prefix() -> str: - return PROXY_NAME_PREFIX_RES - - -def get_proxy(proxy_type: ProxyType): - if proxy_type == ProxyType.NETNUT: - random_id_number = get_random_id_number_for_proxy() - proxy_username = get_proxy_name_prefix() + random_id_number - proxy = { - "server": PROXY_SERVER, - "username": proxy_username, - "password": PROXY_PASSWORD - } - else: - proxy = { - "server": BRIGHT_DATA_PROXY_SERVER, - "username": BRIGHT_DATA_PROXY_USERNAME, - "password": BRIGHT_DATA_PROXY_PASSWORD - } - return proxy - - -def get_random_id_number_for_proxy() -> str: - S = 8 # number of characters in the string. - ran = ''.join(random.choices(string.digits, k=S)) - id_number = str(ran) - print("The randomly generated string is : " + str(ran)) # print the random data - return id_number - - -DEVICES = ['iPad (gen 6)', 'iPad (gen 6) landscape', 'iPad (gen 7)', 'iPad (gen 7) landscape', 'iPad Mini', - 'iPad Mini landscape', 'iPad Pro 11', 'iPad Pro 11 landscape', 'iPhone 6', 'iPhone 6 landscape', - 'iPhone 6 Plus', 'iPhone 6 Plus landscape', 'iPhone 7', 'iPhone 7 landscape', 'iPhone 7 Plus', - 'iPhone 7 Plus landscape', 'iPhone 8', 'iPhone 8 landscape', 'iPhone 8 Plus', 'iPhone 8 Plus landscape', - 'iPhone SE', 'iPhone SE landscape', 'iPhone X', 'iPhone X landscape', 'iPhone XR', 'iPhone XR landscape', - 'iPhone 11', 'iPhone 11 landscape', 'iPhone 11 Pro', 'iPhone 11 Pro landscape', 'iPhone 11 Pro Max', - 'iPhone 11 Pro Max landscape', 'iPhone 12', 'iPhone 12 landscape', 'iPhone 12 Pro', - 'iPhone 12 Pro landscape', 'iPhone 12 Pro Max', 'iPhone 12 Pro Max landscape', 'iPhone 12 Mini', 'iPhone 13', - 'iPhone 13 landscape', 'iPhone 13 Pro', 'iPhone 13 Pro landscape', 'iPhone 13 Pro Max', 'iPhone 13 Mini'] - -firefox_user_agent_list = [ - 'Mozilla/5.0 (Linux; Android 5.1.1; GT-I9100 Build/LMY48G) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; m8 Build/LMY48B) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; GT-I9301I Build/LMY48G) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G360T1 Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-N9005 Build/LMY48B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; S6 Build/G920TU) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; S6 Build/G920TU) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; D6653 Build/23.4.A.0.546) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; wx_un_do Build/LMY48G) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; wx_un_do Build/LMY48G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; LGLS665 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; C2004 Build/LMY48G) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G530T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; VS880 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.93 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-G900T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/2.1 Chrome/34.0.1847.76 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; LG-H634 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Spice Mi-498 Build/LMY48K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.89 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920P Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; C6603 Build/10.7.A.0.222) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; U; Android 5.1.1; en-us; SM-G360T1 Build/LMY47X) AppleWebKit/537.16 (KHTML, like Gecko) Version/4.0 Mobile Safari/537.16', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-A800F Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.3 Chrome/38.0.2125.102 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; MI 3W Build/LVY48F) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G360T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920P Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; GT-I9300 Build/LMY47Z) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 6 Build/LYZ28E) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-G928T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.4 Chrome/38.0.2125.102 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; LG-H345 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY47V; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G925W8-parrot Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.2 Chrome/38.0.2125.102 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-N9005 Build/LVY48F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-G530T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.3 Chrome/38.0.2125.102 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-N920T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SGP511 Build/23.4.A.0.546) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.93 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920F Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.93 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Moto G Build/LVY48F; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36 bdbrowser/4.9.2.2', - 'Mozilla/5.0 (Linux; Android 5.1.1; Moto G Build/LVY48F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; VS880 Build/LMY47V; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36/TansoDL', - 'Mozilla/5.0 (Linux; Android 5.1.1; Moto G Build/LVY48F) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; U; Android 5.1.1; en-US; Nexus 4 Build/LMY48I) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 UCBrowser/10.7.0.636 U3/0.8.0 Mobile Safari/534.30', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-J500N0 Build/LMY48B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; loudnetgoCR18/13plus Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.133 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; loudnetgoCR18/13plus Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920F Build/LMY47X; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/45.0.2454.87 Mobile Safari/537.36 BingWeb/5.5.25151058', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920K Build/LMY47X; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36 NAVER(inapp; search; 390; 6.4.2)', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 4 Build/LMY48G) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY48G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.89 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G530P Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Moto G Build/LVY48F; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/45.0.2454.87 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Moto G Build/LMY48G; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 5 Build/LMY48M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; D6603 Build/23.4.A.0.546; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-N915P Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.133 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G530R4 Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G925T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG-SM-N920A Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.93 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-T537V Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LYZ28E) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 5 Build/LMY48B; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-N915P Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920R4 Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920I Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Micromax AQ4501 Build/LMY48K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G928T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY48G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920F Build/LMY47X; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-N920P Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-N920T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.133 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-N915P Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/2.1 Chrome/34.0.1847.76 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-N910P Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-G920T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.2 Chrome/38.0.2125.102 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 6 Build/LMY48I; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/45.0.2454.87 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920V Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 10 Build/LMY48M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 10 Build/LMY48I) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920P Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; GT-I9300 Build/LMY47Z) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 6 Build/LYZ28E) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-G928T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.4 Chrome/38.0.2125.102 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; LG-H345 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY47V; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G925W8-parrot Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.2 Chrome/38.0.2125.102 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-N9005 Build/LVY48F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-G530T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.3 Chrome/38.0.2125.102 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-N920T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.93 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920F Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.93 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Moto G Build/LVY48F; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36 bdbrowser/4.9.2.2', - 'Mozilla/5.0 (Linux; Android 5.1.1; Moto G Build/LVY48F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Moto G Build/LVY48F) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; U; Android 5.1.1; en-US; Nexus 4 Build/LMY48I) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 UCBrowser/10.7.0.636 U3/0.8.0 Mobile Safari/534.30', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-J500N0 Build/LMY48B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920F Build/LMY47X; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/45.0.2454.87 Mobile Safari/537.36 BingWeb/5.5.25151058', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920K Build/LMY47X; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36 NAVER(inapp; search; 390; 6.4.2)', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 4 Build/LMY48G) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY48G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.89 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G530P Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Moto G Build/LVY48F; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/45.0.2454.87 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Moto G Build/LMY48G; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 5 Build/LMY48M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-N915P Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.133 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G530R4 Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G925T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG-SM-N920A Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.93 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-T537V Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 5 Build/LMY48B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2499.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; XT1526 Build/LPI29-18-S.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.133 Mobile Safari/537.36', - 'Mozilla/5.0 (Android 5.1.1; Mobile; rv:41.0) Gecko/41.0 Firefox/97.0.1', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 10 Build/LMY48I) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.133 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; D6603 Build/23.4.A.0.546) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/40.0.2214.89 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; HTC Desire 626s Build/LMY47O; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/42.0.2311.137 Mobile Safari/537.36 MinuteMaid', - 'Mozilla/5.0 (Linux; Android 5.1; XT1528 Build/LPI29-17) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.133 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 6 Build/LMY48I) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920T1 Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.133 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; m2 note Build/LMY47D) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/40.0.2214.114 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 4 Build/LMY48I) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 4 Build/LMY48I; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/45.0.2454.87 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/44.0.0.26.142;]', - 'Mozilla/5.0 (Linux; Android 5.1; LG-H815 Build/LMY47E) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/38.0.2125.102 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; Nexus 6 Build/LMY47E; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; LGLS665 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; A0001 Build/LMY48B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G900T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; LG-H345 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-G360T1 Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.3 Chrome/38.0.2125.102 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; LG-H631 Build/LMY47V; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; LG-H631 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; VS986 Build/LMY47D) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 9 Build/LMY47X; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; A0001 Build/LMY48G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; LG-D855 Build/LMY48G) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; XT1096 Build/LPE23.32-25-3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-T710 Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.93 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 4 Build/LMY47V; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/44.0.0.26.142;]', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920V Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.93 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; Nexus 4 Build/LMY47O) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; XT1526 Build/LPI29-18-S.2; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; HTC One M9 Build/LMY47O) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-G360T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.3 Chrome/38.0.2125.102 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 5 Build/LMY48B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; Android SDK built for Build/LKY45) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Crosswalk/15.44.384.6 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; D6503 Build/23.4.A.0.546) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.133 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; D6503 Build/23.4.A.0.546) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G925F Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920F Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G925V Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.133 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-T330NU Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; D6603 Build/23.4.A.0.546) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; LG-H811 Build/LMY47D) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; XT1526 Build/LPI29-18-S.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.78 Mobile Safari/537.36 OPR/30.0.1856.95530', - 'Mozilla/5.0 (Linux; Android 5.1; XT1254 Build/SU3TL-39) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-N910P Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/2.1 Chrome/34.0.1847.76 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; GT-I9300 Build/LVY48C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Xperia S Build/LVY48E) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.78 Mobile Safari/537.36 OPR/30.0.1856.95530', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY48I) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; D6503 Build/23.4.A.0.546) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/40.0.2214.89 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; XT1528 Build/LPI29-17) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 9 Build/LMY48I) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.89 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G925V Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; D6603 Build/23.4.A.0.570) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G928T Build/LMY47X; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/43.0.2357.121 Mobile Safari/537.36 [Pinterest/Android]', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G900T1 Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; XT1254 Build/SU3TL-39; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36 Mobile/1 EtsyInc/4.20.0 Android/1', - 'Mozilla/5.0 (Linux; Android 5.1; Nexus 5 Build/LMY47I) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/40.0.0.0 Mobile Safari/537.36; DailymotionEmbedSDK 1.0', - 'Mozilla/5.0 (Linux; Android 5.1.1; D5803 Build/23.4.A.1.210) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Xperia Z1 Compact Build/LMY48G) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 5 Build/LMY48I) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; D6603 Build/23.4.A.0.570; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; U; Android 5.1.1; en-US; SM-E500H Build/LMY47X) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 UCBrowser/10.7.0.636 U3/0.8.0 Mobile Safari/534.30', - 'Mozilla/5.0 (Linux; Android 5.1.1; LGLS665 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.133 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G925P Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; ToughMobile Build/SDP3_3.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; Elephone P8000 Build/LMY47D) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LYZ28E) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Linux; U; Android 5.1.1; en-us; SM-G360T1 Build/LMY47X) AppleWebKit/537.16 (KHTML, like Gecko) Version/4.0 Mobile Safari/537.16', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-A800F Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.3 Chrome/38.0.2125.102 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; MI 3W Build/LVY48F) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G360T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920P Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; GT-I9300 Build/LMY47Z) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 6 Build/LYZ28E) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-G928T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.4 Chrome/38.0.2125.102 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; LG-H345 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY47V; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G925W8-parrot Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/ Chrome/38.0.2125.102 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-N9005 Build/LVY48F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-G530T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.3 Chrome/38.0.2125.102 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-N920T Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1; MotoE2(4G-LTE) Build/LPI29-18.5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.93 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SGP511 Build/23.4.A.0.546) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.93 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G920F Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.93 Mobile Safari/537.36', - 'Mozilla/5.0 (Linux; Android 5.1.1; Moto G Build/LVY48F; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.117 Mobile Safari/537.36 bdbrowser/4.9.', - 'Mozilla/5.0 (Linux; Android 5.1.1; SM-T537V Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Safari/537.36'] +import random +import string + +from src.logs.LogSender import LogSender +from src.proxy.proxy_type import ProxyType + +oracle_log_sender = LogSender() + +# proxy +PROXY_SERVER = "http://gw.ntnt.io:5959" +PROXY_PASSWORD = "94sY7zwBG13i" + +BRIGHT_DATA_PROXY_SERVER = "http://zproxy.lum-superproxy.io:22225" +BRIGHT_DATA_PROXY_USERNAME = "lum-customer-c_daabba94-zone-residential-country-fr" +BRIGHT_DATA_MOBILE_PROXY_USERNAME = "lum-customer-c_daabba94-zone-mobile-country-fr-mobile" +BRIGHT_DATA_PROXY_PASSWORD = "9dwmh54u3bbh" +BRIGHT_DATA_MOBILE_PROXY_PASSWORD = "fk5f7c2z2c19" +PROXY_NAME_PREFIX_RES = "panleicim-res-fr-" +PROXY_NAME_PREFIX_CC = "panleicim-cc-fr-" + + +def get_proxy_name_prefix() -> str: + return PROXY_NAME_PREFIX_RES + + +def get_proxy(proxy_type: ProxyType): + if proxy_type == ProxyType.RESIDENTIAL: + # random_id_number = get_random_id_number_for_proxy() + # proxy_username = get_proxy_name_prefix() + random_id_number + proxy = { + "server": BRIGHT_DATA_PROXY_SERVER, + "username": BRIGHT_DATA_PROXY_USERNAME, + "password": BRIGHT_DATA_PROXY_PASSWORD + } + else: + proxy = { + "server": BRIGHT_DATA_PROXY_SERVER, + "username": BRIGHT_DATA_MOBILE_PROXY_USERNAME, + "password": BRIGHT_DATA_MOBILE_PROXY_PASSWORD + } + return proxy + + +def get_random_id_number_for_proxy() -> str: + S = 8 # number of characters in the string. + ran = ''.join(random.choices(string.digits, k=S)) + id_number = str(ran) + print("The randomly generated string is : " + str(ran)) # print the random data + return id_number + + +DEVICES = ['iPad (gen 6)', 'iPad (gen 6) landscape', 'iPad (gen 7)', 'iPad (gen 7) landscape', 'iPad Mini', + 'iPad Mini landscape', 'iPad Pro 11', 'iPad Pro 11 landscape', 'iPhone 6', 'iPhone 6 landscape', + 'iPhone 6 Plus', 'iPhone 6 Plus landscape', 'iPhone 7', 'iPhone 7 landscape', 'iPhone 7 Plus', + 'iPhone 7 Plus landscape', 'iPhone 8', 'iPhone 8 landscape', 'iPhone 8 Plus', 'iPhone 8 Plus landscape', + 'iPhone SE', 'iPhone SE landscape', 'iPhone X', 'iPhone X landscape', 'iPhone XR', 'iPhone XR landscape', + 'iPhone 11', 'iPhone 11 landscape', 'iPhone 11 Pro', 'iPhone 11 Pro landscape', 'iPhone 11 Pro Max', + 'iPhone 11 Pro Max landscape', 'iPhone 12', 'iPhone 12 landscape', 'iPhone 12 Pro', + 'iPhone 12 Pro landscape', 'iPhone 12 Pro Max', 'iPhone 12 Pro Max landscape', 'iPhone 12 Mini', 'iPhone 13', + 'iPhone 13 landscape', 'iPhone 13 Pro', 'iPhone 13 Pro landscape', 'iPhone 13 Pro Max', 'iPhone 13 Mini'] \ No newline at end of file diff --git a/src/proxy/proxy_type.py b/src/proxy/proxy_type.py index ca78fa3..fccfb6f 100644 --- a/src/proxy/proxy_type.py +++ b/src/proxy/proxy_type.py @@ -2,5 +2,5 @@ from enum import Enum class ProxyType(Enum): - NETNUT = "NETNUT" - BRIGHT_DATA = "BRIGHT_DATA" + RESIDENTIAL = "RESIENTIAL" + MOBILE = "MOBILE" diff --git a/src/workers/GeoCaptchSolver.py b/src/workers/GeoCaptchSolver.py index 920dc3d..8360648 100644 --- a/src/workers/GeoCaptchSolver.py +++ b/src/workers/GeoCaptchSolver.py @@ -6,7 +6,9 @@ import wget WAV_FILE_REGEX = "https:[\/a-z0-9.-]+.wav" -number_text_dict = {'eight': 8, 'zero': 0, 'one': 1, 'to': 2, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6, +number_text_dict = {'eight': 8, 'zero': 0, 'one': 1, 'won': 1, 'to': 2, 'two': 2, 'three': 3, 'four': 4, 'for': 4, + 'five': 5, + 'six': 6, 'seven': 7, 'nine': 9} @@ -22,54 +24,65 @@ class GeoCaptchaSolver: self.iframe = iframe print(type(iframe)) print("url is " + iframe.url) - r = requests.get(iframe.url) - body = r.text + # r = requests.get(iframe.url) + audio_tag = iframe.query_selector('.audio-captcha-track').inner_html() + print("audio_tag " + audio_tag) + wav_url = self.get_wav_src() + print("src " + wav_url) + # body = r.text # print(body) - match = re.search(WAV_FILE_REGEX, body) - if match: - wav_url = match.group(0) - print("wav file " + wav_url) - response = requests.get(wav_url) - file_name = wav_url.split("/")[-1] - wget.download(wav_url, file_name) - open(file_name, "wb").write(response.content) - r = sr.Recognizer() - with sr.AudioFile(file_name) as source: - audio = r.record(source) # read the entire audio file - # recognize speech using Sphinx - try: - text = r.recognize_sphinx(audio) - number_list = text.split(" ")[-6:] - print(number_list) - number_to_type = [] - for number in number_list: - number_to_type.append(number_text_dict[number]) - print(number_to_type) - if len(number_to_type) == 6: - self.input_numbers(number_to_type) - print("Sphinx thinks you said " + text) - except sr.UnknownValueError: - print("Sphinx could not understand audio") - except sr.RequestError as e: - print("Sphinx error; {0}".format(e)) - else: - print("wav file not found") + # match = re.search(WAV_FILE_REGEX, body) + # if match: + # wav_url = match.group(0) + # print("wav file " + wav_url) + response = requests.get(wav_url) + file_name = wav_url.split("/")[-1] + wget.download(wav_url, file_name) + open(file_name, "wb").write(response.content) + r = sr.Recognizer() + with sr.AudioFile(file_name) as source: + audio = r.record(source) # read the entire audio file + # recognize speech using Sphinx + try: + text = r.recognize_sphinx(audio) + number_list = text.split(" ")[-6:] + print(number_list) + number_to_type = [] + for number in number_list: + number_to_type.append(number_text_dict[number]) + print(number_to_type) + if len(number_to_type) == 6: + self.input_numbers(number_to_type) + print("Sphinx thinks you said " + text) + except sr.UnknownValueError: + print("Sphinx could not understand audio") + except sr.RequestError as e: + print("Sphinx error; {0}".format(e)) def input_numbers(self, number_to_type): # use javascript to input self.input_by_js(number_to_type) - # input_elements = self.page.query_selector('.audio-captcha-inputs') - # print(input_elements) - # download file + def get_wav_src(self): + src = self.iframe.evaluate("""()=>{ + return document.querySelector('.audio-captcha-track').src; + } + """) + return src def input_by_js(self, number_to_type): index = 0 + # self.iframe.query_selector('.audio-captcha-inputs').evaluate( + # + # ) for number in number_to_type: - index = index + 1 - (self.iframe.page).evaluate("""(info) =>{ + (self.iframe).evaluate("""(info) =>{ let input_fileds = document.getElementsByClassName("audio-captcha-inputs"); - console.log(input_fileds) + input_fileds[info.index].focus(); input_fileds[info.index].value = info.number;} """, {'index': index, 'number': number}) + index = index + 1 + + self.iframe.query_selector(".audio-captcha-verify-button").evaluate("""(btn)=>{ + btn.click()};""") diff --git a/src/workers/commandor_page.py b/src/workers/commandor_page.py index 071437b..7005ca7 100644 --- a/src/workers/commandor_page.py +++ b/src/workers/commandor_page.py @@ -2,14 +2,14 @@ import datetime import logging import random import re + +import requests import sys import threading import time import traceback from typing import Union -from playwright_stealth import stealth_sync - from src import params, definitions from src.db.mongo_manager import MONGO_STORE_MANAGER from src.pojo.ModeEnum import ModeEnum @@ -26,6 +26,8 @@ RDV_URL = "https://rendezvousparis.hermes.com/client/register" # RDV_URL = "https://api.ipify.org" # RDV_URL ="https://bot.sannysoft.com/" REGEX_RDV_URL = "https:\/\/rendezvousparis\.hermes\.com\/client\/register\/[A-Z0-9]+" +BLOCKED_MESSAGE_FR = "Pourquoi ce blocage" +BLOCKED_MESSAGE_EN = "You have been blocked" otp_value = None OTP_FIELD_ID = "#sms_code" MESSAGE_FIELD_CLASS = ".message" @@ -53,7 +55,7 @@ def get_random_wait_time() -> float: class CommandorPage: tls = TlsPlaywright() - def __init__(self, contact: ContactPojo, store_type=0, proxy_type=ProxyType.BRIGHT_DATA, + def __init__(self, contact: ContactPojo, store_type=0, proxy_type=ProxyType.RESIDENTIAL, mode: ModeEnum = ModeEnum.MANUAL, headless=False): self.otp_value = None self.is_finished = False @@ -97,6 +99,12 @@ class CommandorPage: self.on_success_listener = on_success # reset otp_value to None self.otp_value = None + # ios_devices_keys = self.tls.playwright.devices.keys() + # ios_keys = [] + # for key in ios_devices_keys: + # if "iPhone" in key or "iPad" in key: + # ios_keys.append(key) + # print(ios_keys) devices = random.choice(params.DEVICES) first_page = None while first_page is None: @@ -145,7 +153,6 @@ class CommandorPage: def create_and_config_page(self, context): self.page = context.new_page() - stealth_sync(self.page) # hide webdriver information self.page.add_init_script("""() => { Object.defineProperty(navigator,'webdriver',{get: () => undefined}); @@ -171,36 +178,11 @@ class CommandorPage: def solve_datadome_captcha(self): print("solve_datadome_captcha") - solver = GeoCaptchaSolver(self.page) # time.sleep(2) - # self._click_audio_btn() - # time.sleep(1) + self._click_audio_btn() + time.sleep(20) + solver = GeoCaptchaSolver(self.page) solver.solve() - # solver = antigateTask() - # solver.set_verbose(1) - # solver.set_key("ede6a69396fc961af351e7c8ffda9059") - # solver.set_website_url(RDV_URL) - # solver.set_template_name("Anti-bot screen bypass") - # solver.set_variables({ - # "css_selector": ".captcha__human__container" - # }) - # result = solver.solve_and_return_solution() - # if result != 0: - # cookies, localStorage, fingerprint, url, domain = result["cookies"], result["localStorage"], result[ - # "fingerprint"], result["url"], result["domain"] - # print("cookies: ", cookies) - # print("localStorage: ", localStorage) - # print("fingerprint: ", fingerprint) - # print("url: " + url) - # print("domain: " + domain) - # # add cookies to playwright - # cookie_list = [] - # cookie_list.append(SetCookieParam(name='datadome', value=cookies['datadome'], url=url)) - # self.page.context.add_cookies(cookie_list) - # self.config_page_with_fingerprint(fingerprint) - # self.page.reload() - # else: - # print("task finished with error " + solver.error_code) def _on_page_loaded(self): # time.sleep(40000) @@ -218,6 +200,10 @@ class CommandorPage: self.get_errors() except Exception as error: self.logger.error(error) + # else: + # if self.check_is_blocked(): + # print("we are blocked") + # # self.termine() else: self.solve_datadome_captcha() @@ -264,10 +250,15 @@ class CommandorPage: def _click_audio_btn(self): time.sleep(get_random_wait_time()) - self.page.evaluate("""{ - let surname = document.getElementById('captcha__audio__button'); - surname.click();} - """) + iframe = self.page.query_selector('iframe').content_frame() + iframe.query_selector('#captcha__audio__button').click() + time.sleep(2) + iframe.query_selector('.audio-captcha-play-button').evaluate("""(btn)=>{btn.click()}""") + + # self.page.evaluate("""{ + # let surname = document.getElementById('captcha__audio__button'); + # surname.click();} + # """) def get_errors(self): # send error result @@ -406,16 +397,18 @@ class CommandorPage: MONGO_STORE_MANAGER.insert_captcha_error_contact(self.contact) params.oracle_log_sender.send_captcha_error(self.contact) - def config_page_with_fingerprint(self, fingerprint): - self.page.add_init_script("""() => { - Object.defineProperty(navigator,'webdriver',{get: () => undefined}); - Object.defineProperty(navigator, 'platform', { - get: () => { - return "iPhone"; - }}); - - } - """) + def check_is_blocked(self) -> bool: + iframe = self.page.query_selector('iframe').content_frame() + self.iframe = iframe + print(type(iframe)) + print("url is " + iframe.url) + r = requests.get(iframe.url) + body = r.text + # print("iframe content is " + body) + if BLOCKED_MESSAGE_FR in body or BLOCKED_MESSAGE_EN in body: + return True + else: + return False def on_success(result: ReserveResultPojo): @@ -427,7 +420,7 @@ def launch_page(): first_name="xingzhen", mail="ColbyPatel653@gmail.com", ccid="", position=0) page = CommandorPage(contact, store_type=1) - return page.start_page(params.get_proxy(ProxyType.BRIGHT_DATA)) + return page.start_page(params.get_proxy(ProxyType.RESIDENTIAL)) def wait_for_otp(event: threading.Event, commandor: CommandorPage): diff --git a/src/workers/link_validator.py b/src/workers/link_validator.py index dc1ae06..7489f7f 100644 --- a/src/workers/link_validator.py +++ b/src/workers/link_validator.py @@ -1,135 +1,135 @@ -import logging -import random -import traceback -from typing import Union - -import sys -import time - -from src import params -from src.db.mongo_manager import MONGO_STORE_MANAGER -from src.pojo.ReserveResultPojo import PublishType -from src.proxy.proxy_type import ProxyType -from src.workers.TlsPlaywright import TlsPlaywright - -OTP_FIELD_ID = "#sms_code" -TIME_OUT = 10 * 60 * 1000 # 10 mins -PAGE_TIMEOUT = 40000 -CONFIRMED_MESSAGE_FR = "Votre demande de rendez-vous Maroquinerie a bien été enregistrée et nous vous en remercions." -SORRY_SENTENCE_FR = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci" -DOUBLE_REQUEST_ERROR_MESSAGE_FR = "Une demande avec les données saisies a déjà été validée aujourd’hui." - - -class LinkValidator: - tls = TlsPlaywright() - - def __init__(self, link: str, proxy_type=ProxyType.BRIGHT_DATA, headless=False): - self.is_finished = False - self.link = link - self.proxy_type = proxy_type - self.is_event_sent = False - self.is_captcha_in_error = False - self.is_filling_fields = False - self.headless = headless - self.logger = logging.getLogger("LinkValidator") - - def on_success(self): - self.logger.info("on_success called.") - self.is_finished = True - if not self.is_event_sent: - self.logger.info("will send successful event") - params.oracle_log_sender.send_url_validation_result() - self.is_event_sent = True - - def timeout_occurred(self): - params.oracle_log_sender.send_timeout_log(self.link) - self.logger.info("will close timeout modem") - self.termine() - - def _run(self, proxy): - self.logger.info("will start browser") - # reset otp_value to None - devices = random.choice(params.DEVICES) - first_page = None - while first_page is None: - first_page = self.start_browser(proxy, self.tls.playwright, devices) - proxy = params.get_proxy(self.proxy_type) - otp_input = self.page.locator(OTP_FIELD_ID) - otp_input.wait_for(state='visible', timeout=TIME_OUT) - self.logger.info("timeout") - self.termine() - - def start_browser(self, proxy, pwright, device) -> Union[str, None]: - try: - self.browser = pwright.webkit.launch(headless=self.headless, timeout=PAGE_TIMEOUT, proxy=proxy) - self.logger.info("模拟设备: " + device) - simulated_mobile = pwright.devices[device] - context = self.browser.new_context(**simulated_mobile, locale='fr-FR') - self.page = context.new_page() - # hide webdriver information - self.page.add_init_script("""() => { - Object.defineProperty(navigator,'webdriver',{get: () => undefined}); - Object.defineProperty(navigator, 'platform', { - get: () => { - return "iPhone"; - }}); -} - """) - self.page.on("load", self._on_page_loaded) - self.page.goto(self.link, timeout=PAGE_TIMEOUT) - captcha_url = "geo.captcha-delivery.com/captcha" - if captcha_url in self.page.content(): - self.logger.info("will close browser") - self.browser.close() - return None - else: - return self.page.content() - except Exception as error: - params.oracle_log_sender.send_error(str(error)) - traceback.print_exc(*sys.exc_info()) - self.logger.exception(error) - self.logger.info("will close browser") - self.browser.close() - return None - - def start_page(self, proxy, headless=False): - self.headless = headless - self._run(proxy) - - def _on_page_loaded(self): - self.logger.info("页面加载完毕") - self.logger.info("url is " + self.page.url) - try: - message = self.page.content() - if CONFIRMED_MESSAGE_FR in message: - # publish the successful message - self.publish_message_to_queue(PublishType.SUCCESS) - elif SORRY_SENTENCE_FR in message: - # publish the successful message - self.publish_message_to_queue(PublishType.SUCCESS) - elif DOUBLE_REQUEST_ERROR_MESSAGE_FR in message: - # publish the successful message - self.publish_message_to_queue(PublishType.DUPLICATED) - except Exception as error: - self.logger.error(error) - - def on_document_loaded(self): - self.logger.info("on_document_loaded called") - - def _handle_errors(self, erro_content: str): - pass - - def termine(self): - self.logger.info("will close browser") - time.sleep(1) - self.browser.close() - - def publish_message_to_queue(self, status: PublishType): - # create the message - if status is PublishType.SUCCESS: - MONGO_STORE_MANAGER.link_validated_for_result(self.page.url) - else: - MONGO_STORE_MANAGER.link_validated_for_result(self.page.url, state=status.name) - self.on_success() - time.sleep(2) - self.browser.close() +import logging +import random +import traceback +from typing import Union + +import sys +import time + +from src import params +from src.db.mongo_manager import MONGO_STORE_MANAGER +from src.pojo.ReserveResultPojo import PublishType +from src.proxy.proxy_type import ProxyType +from src.workers.TlsPlaywright import TlsPlaywright + +OTP_FIELD_ID = "#sms_code" +TIME_OUT = 10 * 60 * 1000 # 10 mins +PAGE_TIMEOUT = 40000 +CONFIRMED_MESSAGE_FR = "Votre demande de rendez-vous Maroquinerie a bien été enregistrée et nous vous en remercions." +SORRY_SENTENCE_FR = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci" +DOUBLE_REQUEST_ERROR_MESSAGE_FR = "Une demande avec les données saisies a déjà été validée aujourd’hui." + + +class LinkValidator: + tls = TlsPlaywright() + + def __init__(self, link: str, proxy_type=ProxyType.RESIDENTIAL, headless=False): + self.is_finished = False + self.link = link + self.proxy_type = proxy_type + self.is_event_sent = False + self.is_captcha_in_error = False + self.is_filling_fields = False + self.headless = headless + self.logger = logging.getLogger("LinkValidator") + + def on_success(self): + self.logger.info("on_success called.") + self.is_finished = True + if not self.is_event_sent: + self.logger.info("will send successful event") + params.oracle_log_sender.send_url_validation_result() + self.is_event_sent = True + + def timeout_occurred(self): + params.oracle_log_sender.send_timeout_log(self.link) + self.logger.info("will close timeout modem") + self.termine() + + def _run(self, proxy): + self.logger.info("will start browser") + # reset otp_value to None + devices = random.choice(params.DEVICES) + first_page = None + while first_page is None: + first_page = self.start_browser(proxy, self.tls.playwright, devices) + proxy = params.get_proxy(self.proxy_type) + otp_input = self.page.locator(OTP_FIELD_ID) + otp_input.wait_for(state='visible', timeout=TIME_OUT) + self.logger.info("timeout") + self.termine() + + def start_browser(self, proxy, pwright, device) -> Union[str, None]: + try: + self.browser = pwright.webkit.launch(headless=self.headless, timeout=PAGE_TIMEOUT, proxy=proxy) + self.logger.info("模拟设备: " + device) + simulated_mobile = pwright.devices[device] + context = self.browser.new_context(**simulated_mobile, locale='fr-FR') + self.page = context.new_page() + # hide webdriver information + self.page.add_init_script("""() => { + Object.defineProperty(navigator,'webdriver',{get: () => undefined}); + Object.defineProperty(navigator, 'platform', { + get: () => { + return "iPhone"; + }}); +} + """) + self.page.on("load", self._on_page_loaded) + self.page.goto(self.link, timeout=PAGE_TIMEOUT) + captcha_url = "geo.captcha-delivery.com/captcha" + if captcha_url in self.page.content(): + self.logger.info("will close browser") + self.browser.close() + return None + else: + return self.page.content() + except Exception as error: + params.oracle_log_sender.send_error(str(error)) + traceback.print_exc(*sys.exc_info()) + self.logger.exception(error) + self.logger.info("will close browser") + self.browser.close() + return None + + def start_page(self, proxy, headless=False): + self.headless = headless + self._run(proxy) + + def _on_page_loaded(self): + self.logger.info("页面加载完毕") + self.logger.info("url is " + self.page.url) + try: + message = self.page.content() + if CONFIRMED_MESSAGE_FR in message: + # publish the successful message + self.publish_message_to_queue(PublishType.SUCCESS) + elif SORRY_SENTENCE_FR in message: + # publish the successful message + self.publish_message_to_queue(PublishType.SUCCESS) + elif DOUBLE_REQUEST_ERROR_MESSAGE_FR in message: + # publish the successful message + self.publish_message_to_queue(PublishType.DUPLICATED) + except Exception as error: + self.logger.error(error) + + def on_document_loaded(self): + self.logger.info("on_document_loaded called") + + def _handle_errors(self, erro_content: str): + pass + + def termine(self): + self.logger.info("will close browser") + time.sleep(1) + self.browser.close() + + def publish_message_to_queue(self, status: PublishType): + # create the message + if status is PublishType.SUCCESS: + MONGO_STORE_MANAGER.link_validated_for_result(self.page.url) + else: + MONGO_STORE_MANAGER.link_validated_for_result(self.page.url, state=status.name) + self.on_success() + time.sleep(2) + self.browser.close()