From 0b203b00eefad67603f46a31d10e13c683e556ab Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 1 Aug 2022 21:11:45 +0200 Subject: [PATCH] can read email and click on the link --- main.py | 5 +- src/Test/index.html | 48 +++++++++++ src/check_results.py | 6 +- src/db/DbManager.py | 10 --- src/db/mongo_manager.py | 10 ++- src/logs/LogSender.py | 4 + src/mail/__init__.py | 0 src/mail/mail_reader.py | 139 +++++++++++++++++++++++++++++++ src/pojo/MailPojo.py | 14 ---- src/pojo/ReserveResultPojo.py | 5 ++ src/pojo/mail/__init__.py | 0 src/pojo/{ => mail}/mail_pojo.py | 13 ++- src/utils/excel_reader.py | 16 ++-- src/utils/real_contacts_52.xlsx | Bin 0 -> 11829 bytes src/workers/commandor_page.py | 5 +- src/workers/link_validator.py | 118 ++++++++++++++++++++++++++ validation_url_executor.py | 5 ++ 17 files changed, 355 insertions(+), 43 deletions(-) create mode 100644 src/Test/index.html create mode 100644 src/mail/__init__.py create mode 100644 src/mail/mail_reader.py delete mode 100644 src/pojo/MailPojo.py create mode 100644 src/pojo/mail/__init__.py rename src/pojo/{ => mail}/mail_pojo.py (58%) create mode 100644 src/utils/real_contacts_52.xlsx create mode 100644 src/workers/link_validator.py create mode 100644 validation_url_executor.py diff --git a/main.py b/main.py index 9d810ea..f1ce910 100644 --- a/main.py +++ b/main.py @@ -61,5 +61,6 @@ def get_proxy(proxy_type=ProxyType.BRIGHT_DATA): if __name__ == '__main__': # 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机，传入1的时候是总店 - # start_book(1, 100, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=True) - recheck_the_captcha_error_contacts(store_type=1, mode=ModeEnum.AUTOMATIC, on_no_contact_found=lambda: None, headless=True) + start_book(744, 792, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=False) + # start_book(1172, 1324, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=False) + # recheck_the_captcha_error_contacts(store_type=1, mode=ModeEnum.AUTOMATIC, on_no_contact_found=lambda: None, headless=False) diff --git a/src/Test/index.html b/src/Test/index.html new file mode 100644 index 0000000..d19b96e --- /dev/null +++ b/src/Test/index.html @@ -0,0 +1,48 @@ + + + + + + + +
+

 

+
+


+Post-scriptum La Poste

+ +

Ce message est confidentiel. Sous reserve de tout accord conclu par
+ecrit entre vous et La Poste, son contenu ne represente en aucun cas un engagement de la part de La Poste. Toute publication, utilisation ou diffusion, meme partielle, doit etre autorisee prealablement. Si vous n'etes pas destinataire de ce message, merci d'en avertir immediatement
+l'expediteur.

+ diff --git a/src/check_results.py b/src/check_results.py index 0bfed53..761fb1c 100644 --- a/src/check_results.py +++ b/src/check_results.py @@ -112,10 +112,10 @@ def check_results(headless=False): reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day() print("size is " + str(len(reserve_list))) start_check(reserve_list, firestore_collection, headless, need_send_email=False) - # reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day() - # start_check(reserve_list, firestore_collection, headless, need_send_email=True) + reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day() + start_check(reserve_list, firestore_collection, headless, need_send_email=True) # copy the accepted info to the accepted collection - # migre_accepted_appointment(str(datetime.date.today())) + migre_accepted_appointment(str(datetime.date.today())) def start_check(reserve_list, firestore_collection, headless: bool, need_send_email: bool): diff --git a/src/db/DbManager.py b/src/db/DbManager.py index d4ae20d..019cb7d 100644 --- a/src/db/DbManager.py +++ b/src/db/DbManager.py @@ -5,7 +5,6 @@ import firebase_admin from firebase_admin import credentials, firestore from src import params, config -from src.pojo.MailPojo import MailPojo from src.pojo.ReserveResultPojo import ReserveResultPojo from src.pojo.ResultEnum import ResultEnum from src.pojo.SimInfoPojo import SimInfoPojo @@ -104,12 +103,3 @@ class DataManager: params.oracle_log_sender.send_read_db_event("read_contacts_from_db") contact_collection = self._db.collection(CONTACT_COLLECTION_NAME) return contact_collection - - def get_mail_list(self) -> list: - params.oracle_log_sender.send_read_db_event("get_mail_list") - mail_collection = self._db.collection(MAIL_COLLECTION_NAME) - mail_list = [] - for mail in mail_collection.stream(): - mail_pojo = MailPojo.from_firestore_dict(mail.to_dict()) - mail_list.append(mail_pojo) - return mail_list diff --git a/src/db/mongo_manager.py b/src/db/mongo_manager.py index 903542e..cbbf0de 100644 --- a/src/db/mongo_manager.py +++ b/src/db/mongo_manager.py @@ -9,7 +9,7 @@ from src.pojo.ResultEnum import ResultEnum from src.pojo.accepted_appointment_pojo import AcceptedAppointmentPojo from src.pojo.black_contact import BlackContactPojo from src.pojo.contact_pojo import ContactPojo -from src.pojo.mail_pojo import Mail +from src.pojo.mail.mail_pojo import MailAddress MONGO_DB_URL = "mongo.lpaconsulting.fr" CAPTCHA_ERROR_COLLECTION_PREFIX = "CAPTCHA_ERROR_" @@ -36,7 +36,7 @@ class MongoDbManager: except Exception as Error: self.logger.info(Error) - def insert_email(self, reserve: Mail): + def insert_email(self, reserve: MailAddress): try: collection_to_use = self.db[EMAIL_LIST] collection_to_use.replace_one(filter={'_id': reserve.mail, }, replacement=reserve.to_firestore_dict(), @@ -161,6 +161,12 @@ class MongoDbManager: except Exception as error: self.logger.info(error) + def link_validated_for_result(self, link: str): + id = link.split("/")[-1] + collection_name = str(datetime.date.today()) + collection = self.db[collection_name] + collection.find_one_and_update({'_id': id}, {"$set": {"url_validated": "True"}}, upsert=False) + MONGO_STORE_MANAGER = MongoDbManager() diff --git a/src/logs/LogSender.py b/src/logs/LogSender.py index 3c6a0af..e68e1b0 100644 --- a/src/logs/LogSender.py +++ b/src/logs/LogSender.py @@ -30,6 +30,7 @@ LOG_APPOINTMENT_ERROR = "APPOINTMENT_ERROR" LOG_APPOINTMENT_TIMEOUT = "TIMEOUT" LOG_APPOINTMENT_CONTACT_NOT_FOUND = "CONTACT_NOT_FOUND" LOG_APPOINTMENT_SUCCESS = "SUCCESS" +URL_VALIDATION_SUCCESS = "URL_VALIDATION_SUCCESS" custom_retry_strategy = oci.retry.RetryStrategyBuilder( # Make up to 10 service calls @@ -90,6 +91,9 @@ class LogSender: msg = "{}, email: {}".format(result.message, result.email) self.send_log(msg, type=LOG_APPOINTMENT_ERROR) + def send_url_validation_result(self): + self.send_log(msg='', type=URL_VALIDATION_SUCCESS) + def send_error(self, msg: str): self.send_log(msg=msg, type=LOG_ERROR) diff --git a/src/mail/__init__.py b/src/mail/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mail/mail_reader.py b/src/mail/mail_reader.py new file mode 100644 index 0000000..ed338eb --- /dev/null +++ b/src/mail/mail_reader.py @@ -0,0 +1,139 @@ +import email +import imaplib +import re +from concurrent.futures import ThreadPoolExecutor +from email.header import decode_header +from email.message import Message + +from builtins import list + +from src import params +from src.db.mongo_manager import MONGO_STORE_MANAGER +from src.pojo.mail.mail_pojo import MailPojo +from src.proxy.proxy_type import ProxyType +from src.workers.link_validator import LinkValidator + +AOL_IMAP_SERVER = "imap.aol.com" +VALIDATION_URL_SUBJECT = 'Validation de votre demande de rendez-vous' +VALIDATION_URL_REGEX = """https:\/\/rendezvousparis.hermes.com\/client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+""" + + +class MailReader(): + def __init__(self, login, password): + self.login = login + self.password = password + + def read_emails(self, email_number=0) -> list: + # create an IMAP4 class with SSL + mail_list = [] + imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER) + # authenticate + imap.login(self.login, password) + status, messages = imap.select("INBOX") + # total number of emails + messages = int(messages[0]) + for i in range(messages, 0, -1): + # fetch the email message by ID + res, msg = imap.fetch(str(i), "(RFC822)") + # res, msg = imap.fetch(str(i)) + body = '' + for response in msg: + if isinstance(response, tuple): + # parse a bytes email into a message object + msg = email.message_from_bytes(response[1]) + # decode the email subject + subject, subject_encoded = decode_header(msg["Subject"])[0] + received_date = msg["Date"] + if isinstance(subject, bytes): + # if it's a bytes, decode to str + subject = subject.decode(subject_encoded) + # decode email sender + from_address, subject_encoded = decode_header(msg.get("From"))[0] + if isinstance(from_address, bytes): + from_address = from_address.decode(subject_encoded) + print("Subject:", subject) + print("From:", from_address) + # if the email message is multipart + if msg.is_multipart(): + # iterate over email parts + for part in msg.walk(): + try: + # get the email body + payloads = part.get_payload() + if isinstance(payloads, list): + for payload in payloads: + if isinstance(payload, Message): + body = body + payload.get_payload(decode=True).decode("iso-8859-1") + # print(body) + except Exception as Error: + print(Error) + else: + body = msg.get_payload(decode=True).decode() + print(body) + if VALIDATION_URL_SUBJECT in subject: + mail = MailPojo(subject=subject, body=body, from_address=from_address) + mail_list.append(mail) + # close the connection and logout + imap.close() + imap.logout() + return mail_list + + +hermes_email = "no-reply@hermes.com" +# account credentials +# username = "appointment2022@aol.com" +# password = "gyilpmvyyvlcaviq" + + +username = "chenpeijun@aol.com" +password = "ytifuwguknzifqyb" + + +def clean(text): + # clean text for creating a folder + return "".join(c if c.isalnum() else "_" for c in text) + + +def need_to_valid_url(url: str, successful_items) -> bool: + print("url is :" + url) + parts = url.split('/') + id = parts[5] + if len(id) == 6: + for item in successful_items: + if item.url_validated is not None: + print("id:{}, status:{} ".format(id, str(item.url_validated))) + if item.id == id: + if item.url_validated is not None: + return not item.url_validated + else: + # if url_validated is None + return True + # return True by default + return False + else: + print("id not valid:{}".format(id)) + return False + + +# check whether the url has already been clicked + + +if __name__ == '__main__': + mail_reader = MailReader(username, password) + successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day() + list = mail_reader.read_emails() + with ThreadPoolExecutor(max_workers=10) as executor: + for mail in list: + match = re.search(VALIDATION_URL_REGEX, mail.body) + if match: + url = match.group(0) + if need_to_valid_url(url, successful_items): + url_validator = LinkValidator(url) + print("need to validate url: " + url) + # .start_page(params.get_proxy(ProxyType.BRIGHT_DATA)) + executor.submit(url_validator.start_page, params.get_proxy(ProxyType.BRIGHT_DATA)) + else: + print("do not need to click url --> {}".format(mail)) + + # find link from mails + print(list) diff --git a/src/pojo/MailPojo.py b/src/pojo/MailPojo.py deleted file mode 100644 index 73364d1..0000000 --- a/src/pojo/MailPojo.py +++ /dev/null @@ -1,14 +0,0 @@ -class MailPojo: - email: str - - def __init__(self, email: str): - self.email = email - - @staticmethod - def from_firestore_dict(source): - email = source['email'] - result = MailPojo(email=email) - return result - - def __repr__(self): - return "email = " + self.email diff --git a/src/pojo/ReserveResultPojo.py b/src/pojo/ReserveResultPojo.py index 317ffe6..2e75dad 100644 --- a/src/pojo/ReserveResultPojo.py +++ b/src/pojo/ReserveResultPojo.py @@ -31,6 +31,7 @@ class ReserveResultPojo: ccid: str = "" source_from: str = config.LOG_SOURCE store_type = 0 + url_validated = None @staticmethod def from_firestore_dict(source): @@ -68,6 +69,9 @@ class ReserveResultPojo: if 'store_type' in source: store_type = source['store_type'] result.store_type = store_type + if 'url_validated' in source: + url_validated = source['url_validated'] + result.url_validated = bool(url_validated) result.id = id return result @@ -88,6 +92,7 @@ class ReserveResultPojo: u'source_from': self.source_from, u'store_type': self.store_type, u'accepted': self.accepted, + u'url_validated': self.url_validated, } return dest diff --git a/src/pojo/mail/__init__.py b/src/pojo/mail/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/pojo/mail_pojo.py b/src/pojo/mail/mail_pojo.py similarity index 58% rename from src/pojo/mail_pojo.py rename to src/pojo/mail/mail_pojo.py index 7e12524..231025d 100644 --- a/src/pojo/mail_pojo.py +++ b/src/pojo/mail/mail_pojo.py @@ -1,4 +1,4 @@ -class Mail: +class MailAddress: def __init__(self, mail, password): self.mail = mail self.password = password @@ -12,3 +12,14 @@ class Mail: u'password': self.password } return dest + + +class MailPojo: + from_address: str + body: str + subject: str + + def __init__(self, from_address, body, subject): + self.body = body + self.subject = subject + self.from_address = from_address diff --git a/src/utils/excel_reader.py b/src/utils/excel_reader.py index edbe05a..3854935 100644 --- a/src/utils/excel_reader.py +++ b/src/utils/excel_reader.py @@ -8,7 +8,7 @@ import xlsxwriter from src.config import CONTACT_LIST_FILE from src.db.mongo_manager import MONGO_STORE_MANAGER from src.pojo.contact_pojo import ContactPojo -from src.pojo.mail_pojo import Mail +from src.pojo.mail.mail_pojo import MailAddress from src.utils.generate_random_passport_id import get_random_passport_id_number phone_number_prefix = ['6'] @@ -55,7 +55,7 @@ class ExcelHelper: if contact_dict['mail']: mail = contact_dict['mail'].strip() pwd = contact_dict['password'] - contact = Mail(mail, pwd) + contact = MailAddress(mail, pwd) contact_list.append(contact) return contact_list @@ -124,7 +124,7 @@ def get_random_id_number() -> str: return ran -def write_new_contacts_to_excel(valid_contacts: list): +def write_new_contacts_to_excel(valid_contacts: list, generate_passport = True): row = 0 col = 0 # Create a workbook and add a worksheet. @@ -151,9 +151,9 @@ def write_new_contacts_to_excel(valid_contacts: list): if __name__ == '__main__': excel_reader = ExcelHelper() - # contacts = excel_reader.read_names("/Users/lpan/Downloads/real_contacts.xlsx") - # print(contacts) - # write_new_contacts_to_excel(valid_contacts=contacts) - for mail in excel_reader.read_mails_and_pwd(): - MONGO_STORE_MANAGER.insert_email(mail) + contacts = excel_reader.read_names("/Users/lpan/Downloads/real_contacts_31.xls") + print(contacts) + write_new_contacts_to_excel(valid_contacts=contacts) + # for mail in excel_reader.read_mails_and_pwd(): + # MONGO_STORE_MANAGER.insert_email(mail) diff --git a/src/utils/real_contacts_52.xlsx b/src/utils/real_contacts_52.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..0274f8415710a6ea73eafcab8234f769817e77e9 GIT binary patch literal 11829 zcma)i1yo$iwk__S;4Y23yIXJ%?(Xi|Xt3b!?(Xgc2p&AR2e+VsPm*`;xhMC&H-3!) z-DB61s##U5SM4*j_1cCX|pf;2DZzwM9f&f^~S#mKd#i=kT#L?xL)) zskUbJU<7n8$NA0*FTjt%+Cq`;;IJpsY?&bM&(MsNCW6}Z(pj2fa7xr(P zhbN0Msp7KM9~{i_C%!AldWzt4ToY-AB{n`Rubf$XM> zbagqjUDIeiJ+(f7{S#i2=2I&CS9noh;YIy-c#Z5FO@87V6Mm1Mu6EdMx0UOo8%hF5J2m*2r8V>1x~cLPGL?Am2`{L0=Bm#~#u>7IexM z>l?6N;Tae4Zo|-`?{bIdq%aO(gP`}=xTBOgu@;S23fHjc*5QCF=$Q(Ddc#|UJ&4Kf zKl?lhWNIa8goDO)OB?ERm3DwL^E{hFK~On z17nSeoW|)!QD`k|F1-vy+xBZV<~}3P>q=%=_p-jtg<^z1Dwu|hBn1nY`s%Ed4S(m# zn?naN(cESQoi7H8xw%o*(QU*qAKB+gg$Dz70_T3=`i=K(uo#wvWun9sd;4zjV=e@2gn+0=R zuCct;+xQk?)g{e?`mP`q#<-?*d4_1>Ec$t(?qF zOq~CW5?|ElhPPlqKm;g3K(PKp_LmCNuic!sV!I;#8ZF?j@gdgUZ(Wj2*XSf@L(olu z34L6M6kQP}Q%p4H@p+uEf6v2HAIshvhW}DOX17o1xC49U_f)q&zVtpD)pzVK`q``DgSfQ(_= zy+ez3;wag7J#)XmR>1#~HRJm#%nL6c!^E)ix=oST<0^^ttYz(4Q|Q34z1u?)Ur2HB zN!vpIgUdRhc>PG}x(n%fwqDcll}R&0y4|v!)3>$F3QPiIMcl!Y;&6YvZk_Cqt8>Id|;Cx;dh_dNk+1#Q|&8^H1 z*<_*r0SQ#<_}T!$FjF~vqGi|Tk|kpC#G`}R9U4LZRrSXi0|a4_(w%#n%|r323rAD` zY4(Y!w}<4noy+?5(A(_CldCZp_j^3|kOhZkzU%Jqc0Z6Xc*$mi=lhXdgS*Wh>5^=i zSKRc9s?SV^E7IP3=$e%~mWEqH;#TTSZr3zpsz@FsnYnU#H;$?DRqLl}Ao#6oBUz+W zA?cl5@}_%5z@yk|ch-z;($Fznn%p<4!aH43O}62-4?oVH^9M>n4^UYQ@`eWQ8-+F( zw9)i*v3(N|0(pku^^=@9pKTt#_Pv1Oue1?;BTh*$9fFGeamqmSlmx^7n>+DB5+?Ek zx-N`;LGX$ZxAe9ZU_o<=oZ2?gfDir^2q|kjEK?ybV0q38(r+GCKh1GgQ7jM97s3pzmMZIvP!jV6$m%-KA$C(Se`pYQ% zUq(l$o~Foy9O7sqm3t)`R&ua}3i?_@%8k>6#8m6xgF{dPf?=SwU>

zRU9Y!(phh-x=D=}7sR3{e805-|>i-Z|fqR2Hs8ToCmqxy0*8O(P{Hsu3oCqF%t)dLzk+$Mcbm--GtBdB*>R597elKmvVu!r<2F*&Fo5TL4e3#h6{$A?B4mQluHs6uNc zmsgaDoi-v~Xe6shN3umUF+|z}S_?FpE40x8GkM?CBdF!Ca`bIrj0&&p)ei8RuhQ;+ zSiEaA)<~sBuyvrTdHC2=Ncq;r26q2d3O^lbRB&D7>H%yCmAB zQR;m;FCcK7bbg3Yo%*eA?G)E5Qf`_bq}i~Q#pX3t@U3-JOaMF~O~jFjmLKZHwchV> z6sK8^tm(+S4t`5>Skw5BTF)a}&e~8m5?M?66$5kp*nx-Dcl^ZZk)?U!uO^O}r8kWN z(3qBAeUyPn^n#xKE#GvJ4LBQ1!e0N@-gTTV%vF z^=I1`eNbe&9p4Nvd)quGO7P%`E5HS!ro;rMpU!MQ@dZia+{UnF#J}h5pE4rB{{EOU z8k{LRVZuCPx_Hf)Ww04H#0RE-84fNI{{f_44MV2iksg{O)4~veGbMDB0PSvBup-Qr zJ2EGZGAJ|Zb*AwK3A@T9DIb=I)cPSk2AQ=*dGN(KH$WCJRf#)n9ZrHGW>I+BP`eve zP>{R|@!%N0xpL_T1){SezDdlOXr$y(f9<_rU=Nx0B#iU0B&HS~`;MAjgY%@P;D!O9 z@TMC@T9C6F_)%tqupUOhg2%DE0XM%f(1=qt>R=dA*FQcEB9R1y-E7zaDUNb&?z7Vq z;xH|XbE?KLk+7oROlUsTeLX$fy6xjRm7QA2wSS zY#YqYXr3l8oDGYp?`iBJm^UsR(I!o80Dimr!^ z`58o=+ST>iS`B8-KtY1_Xeu^S})Xd5=joPhe^%YERz`k16 zZOmN0A4ZBk({sxrrEKTCS!@ZgfaGHADEq2yIuH-9*i3_|TSTI{)PPe|R@gDE;M(9? zMrt7&=7+u;6^mHuQ90Ui>U5m-xZXB<_1!+}WMhubUBpUv?rt4AS>RlsHhb9>ck*F7g>{p_9G`PRPY2vb*A_IbjjumX_}6T=(z`=>MK%{k&wo zCRxtrCN?I2T(YjO;nr=TKtT9#e`l`$68k0mD|0&0khfc5f7^|`=tlIK%u+PrLc4@7 z)}~j=;ttq`5?G`rsVnCR&b;%%UEcNw?Qb(k25moKw5FK?Yj@5j*9xr!e?b2(DOHCx zCOrd%J3kjHO5*4Jvc65~abzf>9cH0SPk$!s>%=;A?RwD z>U`OpDWH$_yFw%>k(U>9F29h7?99q&*ezDakkmKO><rXQ-hrkQx@$0tVzpOB8u zY?D4ZaH}tQ-S_+c0%5*DmhaW&1?H~1#=y%%KSlS`lLr?HvpMl}1Vi@I_v4X>YW^QT z+J^z>HUSc1cQN9kd0IO#_~$UrEFlg63Q~tI^nMg&Gf1QWSJ=ZWIQ)%DE>n{hB!?)s z_ktIc;JVqXZpQdQhDGME`?v->Ah(=m`1DeU_99o*>h7EofsT8IY6JNC-s(f`5k|v} zof=@OEbvmp-&bQe4NqC{Xz*fEC=2%Q5@OI!P8Gi`x_o>9O7klo5QyQ+NXs%8kW$28 zM7UZ&_@*WwB2>9r*Slf5aMH|rhT^qJO2T}W3L-??OS$0AqQXmz#oG!)NA!JDUL*6V zSNl#(Tb}ydSrAL2lPR{&dfW!|0VBf~T%?oQ<&hv=f?kP|n=aqujgzkE>A3~RfTB_e z{uC#zp+Jl>jAG>XR_eer2M&>Io;%QCnpK}OkaV)xh4bX;Z;O2Rc2Y zvYdHV^Y-k$k&%t7Cr}x!gYKIoI%Fcep4q5RdoS%_BuH(lrVbX84f%6Vxwv4$tfbH6<4g-=5Yuv9cI3h+gAh+Q-W4Y zuihD8^sL5I2O&Nx$1fS*_oY|LFmb?I+AABJbQRPnL!%MTAtmJ^BC;fgfJCrS(7b4Z zfA&;_Ta@%eq?+d-!g^+->WQwh8Zj33R&{nY5M7z*Dk@Bve2W@r?;Rv=O^(Faq8Cbi zz*2-}OCcjR#bBRURB2Q6bt{8h=K^eOK%pPxR?hY(5S#td=xr+eE3-u z75i-KXL!ewmz9iqRvFFyQhEF-5Lmd!SU5=HM};Jr>rCN_q*I+FGHKYtu(1KAadPXQAAkH<+C= z{yr&2yQF&JSZzt2T*M*UGpr05h2_q_=}u zYGVfaP@v{Tag*iILD;TD#22Iyy|)=k7NHL+?;*~HH-1Zt;GM>7QDuCS zW|M}VeR28~pMf>qlE#ydoYDSx>e$4{U<9kHrNQvLt94;%Z#{+b+Ys_4p&`y8?>TB; z_pSWLjRX9at!yR$oelW%!yhkJfQ28`XkT9p)X0N?ApZUpb8_~uHgWoSrJvLGvddpY zf9Y201~*)`Y_>G)x$Bk5SYm=*O)NSri7!j!Rt>Vr$l2Wl%W`P?=62SKOuxT%H2fJV z(1@yuDfH?gbJ(Y6ZbcT)B0G>4dDb{DJvpQba)ehWXk=UT<+0O^X}+=JJU*DP@EzlH zcA&PO*N>HT?cG7Ax8B}xK;b}XvU^cK>hQTEU50D{$l7eZqH>}Dy(9~aM@a8+0|t)8!DkC4{F;@nqf&^%u6+t8Pa z*{$)j#TCNF_kNx?qb;nFp=ayXKfWKfUoOap>n)+vFRrXKUxNwodRN~{jl^K>nXAW( zh*UIkAfMmNijABE;FquvN%*->B_LRryZRJnc_r|cY=Zs}XL~O6)yAyKkYe$SJeu3) z@9fendbg`tj@$*1sH(oyYnb$fC}H`4zh{wPstLLQx;{htfe`u_Vc}z+2FA>6))OuC z6w_#zY_#I13*~LbcPx%I!znc{EROSsi?C`Ku>~{R{E>m$?eJaZe$qw)!~zolpC$;* z_sdPO#Cl%IQ3n>GbPb`?r5Uk&6Xb2fFS;tN*sgxr+M2?+d`cb`$$)XA>7qKodnCs! zi;jwY;R-w;>^|q^r*bYQTc=~M9T&7{y<~bh47hF}>TTLK4oD(q<0gTM=b1PML*yvm zS3EaPPJ9la808WQSt#j~WcRqMD4xVN?CghqP#InJ%nCP33bYghcSNY$6!)^}V{gG; zCM@TtI`YJGW^-+!538=8zSH`}I%-JxI#^*>BAsIvI5lyKvOhX=eHg=U4Co;(c;GJTV29j7t%| zSwuWI?oDE&PhXn(Y16U~LW$>TjCOZb4!?Pm7U14mHS4VUMk8-&i%Nm|tK0Hc=lDy* zh+xZGCBND8C8R_(Yd%X(siryJN@*^?1XL|%r*>j72`|JZnH5vp!SNDwyALn#&{)4k zYAHIM8{T9hF&n7!NY_KsMq7C6jI!!WjzsI|iS^cj)SGT@zwuz0L1%K*d~1O~h>`n} zLhcJe=mikFm!mXLMZ}mQ_r#;^b3sTpaU$TIYrp=u(?h_O>co{6GAb&@odz+$P$-ZV zLgm#_72(Vm0c@+9bFP(I2xMJpBOp^e@Qm40*XcE(7Co*hr4&IRhu|4buXoS0Dn_&a zj(Y_%*hGblLW$tvzqRsNT~ofT#h53Uj4fHJ*e@MxgG^j~RN095fi*>>;K~-|zI&~3 z4lJ6QpDf-U6lKtXUJL?Ggh2lSja!cHayju#1&kx=qp|o^B4s__7iB5qcPR^}T`@jA(j0VD_qb3h ztI24l;XKp;almIE*VzPhNi0WaWy#17u6=ZLMMZNA`b z-P_`zEumuYP6~di|FoAB^zN<>I(~MZJD2~M`h=glJw4D~d1>h~Dk9ngNvCj0LnV%& z5|1#|Y#PpGA;DOLhtM)hO#IAN8FsF)p+LojX#Q3uQ`uE`>U{#5lp^?MgG3*cDa7Il ziFFnDV4ZPOow^qzg;Ha9UF$NRJLQt-jrw(-&^VoQc7boMIAT=hUM)gX)a z5~)$Yl8|JA5>-G+mVnip7!0X3CBbG3i4C2!+Ik%%3mhq8fS!o@8EDS&k?AJPW?4dw zOa#6tsbxPPzpp1(Hb0FN>|K_Zf@gv;?Kt(m(9-6KDz5s<=>bUWXVnaFxuvLdvjvA7 z;|FvU^EXY#L%Vn&LIXQo2Je~7`Rq`8C+O{R_vF%;fVw8~=;lo+Kq3HnI~y9s($z7q zwQDoS;bx(0(|Pp5*KUguofTT-s?IY@AEL2@F9<&>x^r+9Tr0Qpi@LI|<`%Zz7fDkS z!9>;7y{GEVzhf>0JQcIT`+>sbi9J>8&95h31+>#%P8<4+Jk9yk1jGugRMguk3td6Y ze(I+MB5TwjaEOw0KyQ^JNM68(iFyg|;1h#&w`;Gi;ZTjPt{=LZQIXX=|j z>9$X;jgN#Nln{P?+JF}2$nkprEvP9x#&|4z)Bhn)E{sd~(hTq2w`v$0*M0L=#E>Jk zH)xtpGQwvcORoBa;dG5tKDbYx9J&a@TTsCC@~6bCg=td^EC&S1@R*M=5_i5Sk^9u} zrsLBNHCo)-oH9l4GOVs~5iIHqZpu+mWHM21Kcr~B^Xt9io=e@Fp0iLTqAFU=IZ26u zmdrgBTn0)??~vD8iZfezRPvA}v*)_JWklhShA(=zqGd$=kY?4tR!MPG-{51q!eee; ztOHo{THp6dZp0+z9cd-Q*ad@lAP{X#LvhqSpi&4<%2ZxRp7$is_Jt=5I{%~4fixnOr7_bSCtM=N4 zpWEowj)Y@A^?=2VuR-msRrVeIQg^D3{Kxk)_Rn=37%s^`ho|co+U5NpPomnkm!ZjOs zKdE%lU=k@fuXNrEtEfMl|ctEYm863XC)KeXL+Lr2SO zQ}pHR#ChlISI_yu#gu4x|1-SNqrn4+t|8Z4LUT;cttg5_&0jq7MXNFhavTZ#O6jJx5w0AN1X1B}NM~HCflLWNwQ^Z~X1*Fv$#iZZ7QrB3n&ly}PpH@K zdWA4gmCw+07;4-O>cdz#GdK8sc-`-Ls2O*^klw_KRohy^e&u6w_p{?^#`;G4vW~UY z+2i`;#Mj&qxi;h?=qOSNBc8*H;>rnaJE5d1sB;~!BmgYWQNhg0crU_ozO7Jm^1hr- z)}u5X52wIY^1j*Z*{zvR)K;&^p|)K`w!pAhiR<&ZX-wPlQ}7Bmuxc-#c05HATYe9` zoy3T0OqreVfb6u1kNPai_N62$TT}o~hT~;8^xdr-qL;;a){|!O_f?iNB4ev&8u4oM=mF8>)QmOt81463If2-9U+R5s zyttWa)pYrR72lC$>BLe(An)E~ttR7^G-JqL(@sh&e|gUTI2*j2n-EIwnY+MtFfIU2 zi*Kij5nuclvw$|$#u@HL|28h?F&GWM|H^fz5$a4 zW5~H3D0~J*#{N6|S=}U){0cV{nTuNX_yR6qT6HRb3thP2&|BY)4kkYmGR$#pi(jjx zXP9kWs32MJ^@q@+iAbt0Xg>q0{9?v%no~+NaBbLlz4b9HH%OKEGuDKAnj)GYbZa`1 zM?c8X=Ui-P`0#K6bjQ z=$v8WJ|u4KSdj?Ias}{*yy^w9gO)IrAI78nPccBp-RZH$z{tNhieBHU$7n4Ln~EcS z9K(}y7y**?eu2?k`BJBJZfCvqq3H(BhHC!En^-hjNtKQ_(3w#@e+Oc2!K4HO*33f@ z0=6sUORteTjZPWmki4W=2m^5#q=uhl|J}W4*pw+ zPKrhtzNE0o7!;;Ni!s!w!fST+Tb`x0`A=$OtZ#r2JG6yYQ>7e$^}PC*mWKzA)oLa2 znTAgywpBQ4@3p21+06Hh+K}Uq!*vG1KAY=8EPkzQnR7hOu}C~Bh(5iUrL&}OR}F`i z8v$zRFqNx&WeW))?qbpPxHZ$Lu@!SoO40+rwQF6KdBrr&pI2zJqMg|-WKgHRJ?z7H zA8EcmU|z;jNXgi^4W-_SUX>4*C+TvKjcH^!-7bEwPP#ByKLh&9dS%*| z-1?$m_v9=0E4gejZHiVDb7e5I#+Ay{FH%jF))ic$_zUA78p@Fpb9f5hVVI7UiO~U; zKL&b$jvEd~tmdWIzxm8-s964mpq@@j*4pFzCIK4MmX?#%wkzNu6!v)CJP8&DuLPId zdakV3(%+mWGK#>pU2@B#q4}~gj$!`b;1*20g)*GB0kD{^`GGKln@e<@*f%vb!-~Kd zmpoCq5RXl?9Aw))q(fh5?`g6i$AN!2_h`D&bIEwR96*Kf66_1top)U56DRMG9;Txk z-b(O@PCjp50V@sUErNio?lbg12^>T6u#S|knfFROpP zvp9)G?Fj-)!&;5IQeayQXRJj+l(jt4LM^?9CdiN=774`C8m}iyy;7qDCp9>H>929A zr1ox@D;0Yq5hg)LGJ+iC2s)}-$$B~*kz#+U+EDOV5nzG36|aJ70;)`H6S^Rd^f~)$ z|2vDcHTif1_j|9MdF5%n_VefTL|U|Hm-feW0EQo1h=~&(3-bo-jJZ+hC#iGGE6secFwE+C$3mmVCKCky_Y%Zw zi2+2<(Zu>s`i2*$YsbrL-});H3I1O)^gksl<0hpB7*U3P^1JrBWZlqFmz)_$j%AH~ z{8MCJrpVvKX`FO*3MtZlX@mZldcT*x?rq$TyGo?XR21C-bWsEf+I@2EQatx|S6>B6 zR&>sRd#Kp=L9~CnG<*yS+~ZAHrYDQl@m88*4&|YWx0pUuQ4q}LVG}Fn5-=hSWbd?9 zupPY=AIc_BkmVg*qYm?GhjQQy?zp$JD?6Il;7iRHnPj2aQHjlTv5rBH4D`-l>_)i} zgM#9kbade#J9Co?t*A7mLln~vPEij3x~tvoK$|+OEk_s*$Qi#EoSoh8>y4e(Y8aE` z4g%yIy^t{v59}m)@U!VjI$qM1yZ!H2{!cmQzrneCo`-ev3S-+VoH(zauZhiXv7h{z zzjpIq=)SVM5QfeII|HYE9A=`J#4~k=auzDRK#XYo%nhNXwbGljKjmuVI45f(WCwMh zQH-suww`@KCbu+VR50eEM4*3Q{{q#8bcW236J~SX6ak7DDOp2vcGg6=sF5W7r2(AE z0oFLjfIzpAd_muiDgqqpP=)CxNvuc{c^=;;$}@g{lKaPvzvE`<_cg)gM+${dhnSt0 zP9xPur1iYVQ>ODo7}+S<0|BlPcd{Y&xld3SyE0)ydZ8ZL=pn;$x=uwY*OhnUX&1CS zeOx|qp4<^y2!m+|#VOHWh%P@nP{mqHb7Nw4-Z||4QS&Of&~!3;J?Mz98Q@R0mX?T} zt+R=(v!05Fy@`{~&jVkXFeV4a^fu&{ysN)Ji$&SqAksj8)1;Ja3;fIX9@gumEa3U~ zEGm64p@7KrciboU-7im_2;wwW;Q-MuYjYKPH|n5ro`tIwriAgyms#j3LCSsZ&}>@Z zb<^h$mW&$EJ-{;j`HFW`y!O#`pfoj?pm^glJ`E((j1%rCj`XHIc3z!4^qtZhN~xNN z?0CGrDzq}xS?{mCw#k4(YP_}>O%-V$P$H8^VwJm`zxgZK!lJA0u`M3>a~KC+V%VKZ@F~D^AVH_DeG2^Uc8W$w>{E8QcGDHx z)J@xQ)q7>c{sg5U4GQ-5R|fu@^7-e=yeUq|Nua`y8~{%${={cQinYW=6bKY6V`g@5~__;x1g3Q&Iz^v_84)6Z{|wEkV~KjPY-cK*ET{cQ(G@XP&wT>SpD@MjI;ZwoZ9 zcgLSs$A48M{{QE0@> Union[str, None]: + try: + self.browser = pwright.webkit.launch(headless=self.headless, timeout=PAGE_TIMEOUT, proxy=proxy) + self.logger.info("模拟设备: " + device) + simulated_mobile = pwright.devices[device] + context = self.browser.new_context(**simulated_mobile, locale='fr-FR') + self.page = context.new_page() + # hide webdriver information + self.page.add_init_script("""() => { + Object.defineProperty(navigator,'webdriver',{get: () => undefined}); + Object.defineProperty(navigator, 'platform', { + get: () => { + return "iPhone"; + }}); +} + """) + self.page.on("load", self._on_page_loaded) + self.page.goto(self.link, timeout=PAGE_TIMEOUT) + return self.page.content() + except Exception as error: + params.oracle_log_sender.send_error(str(error)) + traceback.print_exc(*sys.exc_info()) + self.logger.exception(error) + self.logger.info("will close browser") + self.browser.close() + return None + + def start_page(self, proxy): + self._run(proxy) + + def _on_page_loaded(self): + self.logger.info("页面加载完毕") + self.logger.info("url is " + self.page.url) + try: + message = self.page.content() + if CONFIRMED_MESSAGE_FR in message: + # publish the successful message + self.publish_message_to_queue(PublishType.SUCCESS) + elif SORRY_SENTENCE_FR in message: + # publish the successful message + self.publish_message_to_queue(PublishType.SUCCESS) + except Exception as error: + self.logger.error(error) + + def on_document_loaded(self): + self.logger.info("on_document_loaded called") + + def _handle_errors(self, erro_content: str): + pass + + def termine(self): + self.logger.info("will close browser") + time.sleep(1) + self.browser.close() + + def publish_message_to_queue(self, status: PublishType): + # create the message + MONGO_STORE_MANAGER.link_validated_for_result(self.page.url) + if status is PublishType.SUCCESS: + self.on_success() + time.sleep(2) + self.browser.close() diff --git a/validation_url_executor.py b/validation_url_executor.py new file mode 100644 index 0000000..bdead3b --- /dev/null +++ b/validation_url_executor.py @@ -0,0 +1,5 @@ +from src.mail.mail_reader import MailReader + +if __name__ == '__main__': + # read emails + mail_reader = MailReader() \ No newline at end of file