panleicim/appointment_tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

can read email and click on the link

Browse Source
This commit is contained in:
PAN Lei
2022-08-01 21:11:45 +02:00
parent f890454e25
commit 0b203b00ee
17 changed files with 355 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.py
+3 -2
View File
@@ -61,5 +61,6 @@ def get_proxy(proxy_type=ProxyType.BRIGHT_DATA):
if __name__ == '__main__': if __name__ == '__main__':
# 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机，传入1的时候是总店 # 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机，传入1的时候是总店
# start_book(1, 100, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=True) start_book(744, 792, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=False)
recheck_the_captcha_error_contacts(store_type=1, mode=ModeEnum.AUTOMATIC, on_no_contact_found=lambda: None, headless=True) # start_book(1172, 1324, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=False)
# recheck_the_captcha_error_contacts(store_type=1, mode=ModeEnum.AUTOMATIC, on_no_contact_found=lambda: None, headless=False)
src/Test/index.html
+48
View File
@@ -0,0 +1,48 @@
<html xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:DengXian;
panose-1:2 1 6 0 3 1 1 1 1 1;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:"\@DengXian";
panose-1:2 1 6 0 3 1 1 1 1 1;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0cm;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
span.EmailStyle17
{mso-style-type:personal-compose;
font-family:"Calibri",sans-serif;
color:windowtext;}
.MsoChpDefault
{mso-style-type:export-only;
font-family:"Calibri",sans-serif;}
@page WordSection1
{size:612.0pt 792.0pt;
margin:72.0pt 72.0pt 72.0pt 72.0pt;}
div.WordSection1
{page:WordSection1;}
--></style>
</head>
<body lang="en-FR" link="#0563C1" vlink="#954F72" style="word-wrap:break-word">
<div class="WordSection1">
<p class="MsoNormal"><o:p>&nbsp;</o:p></p>
</div>
<p><br>
Post-scriptum La Poste</p>
<p>Ce message est confidentiel. Sous reserve de tout accord conclu par<br>
ecrit entre vous et La Poste, son contenu ne represente en aucun cas un engagement de la part de La Poste. Toute publication, utilisation ou diffusion, meme partielle, doit etre autorisee prealablement. Si vous n'etes pas destinataire de ce message, merci d'en avertir immediatement<br>
l'expediteur.</p></body>
</html>
src/check_results.py
+3 -3
View File
@@ -112,10 +112,10 @@ def check_results(headless=False):
reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day() reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
print("size is " + str(len(reserve_list))) print("size is " + str(len(reserve_list)))
start_check(reserve_list, firestore_collection, headless, need_send_email=False) start_check(reserve_list, firestore_collection, headless, need_send_email=False)
# reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day() reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
# start_check(reserve_list, firestore_collection, headless, need_send_email=True) start_check(reserve_list, firestore_collection, headless, need_send_email=True)
# copy the accepted info to the accepted collection # copy the accepted info to the accepted collection
# migre_accepted_appointment(str(datetime.date.today())) migre_accepted_appointment(str(datetime.date.today()))
def start_check(reserve_list, firestore_collection, headless: bool, need_send_email: bool): def start_check(reserve_list, firestore_collection, headless: bool, need_send_email: bool):
src/db/DbManager.py
-10
View File
@@ -5,7 +5,6 @@ import firebase_admin
from firebase_admin import credentials, firestore from firebase_admin import credentials, firestore
from src import params, config from src import params, config
from src.pojo.MailPojo import MailPojo
from src.pojo.ReserveResultPojo import ReserveResultPojo from src.pojo.ReserveResultPojo import ReserveResultPojo
from src.pojo.ResultEnum import ResultEnum from src.pojo.ResultEnum import ResultEnum
from src.pojo.SimInfoPojo import SimInfoPojo from src.pojo.SimInfoPojo import SimInfoPojo
@@ -104,12 +103,3 @@ class DataManager:
params.oracle_log_sender.send_read_db_event("read_contacts_from_db") params.oracle_log_sender.send_read_db_event("read_contacts_from_db")
contact_collection = self._db.collection(CONTACT_COLLECTION_NAME) contact_collection = self._db.collection(CONTACT_COLLECTION_NAME)
return contact_collection return contact_collection
def get_mail_list(self) -> list:
params.oracle_log_sender.send_read_db_event("get_mail_list")
mail_collection = self._db.collection(MAIL_COLLECTION_NAME)
mail_list = []
for mail in mail_collection.stream():
mail_pojo = MailPojo.from_firestore_dict(mail.to_dict())
mail_list.append(mail_pojo)
return mail_list
src/db/mongo_manager.py
+8 -2
View File
@@ -9,7 +9,7 @@ from src.pojo.ResultEnum import ResultEnum
from src.pojo.accepted_appointment_pojo import AcceptedAppointmentPojo from src.pojo.accepted_appointment_pojo import AcceptedAppointmentPojo
from src.pojo.black_contact import BlackContactPojo from src.pojo.black_contact import BlackContactPojo
from src.pojo.contact_pojo import ContactPojo from src.pojo.contact_pojo import ContactPojo
from src.pojo.mail_pojo import Mail from src.pojo.mail.mail_pojo import MailAddress
MONGO_DB_URL = "mongo.lpaconsulting.fr" MONGO_DB_URL = "mongo.lpaconsulting.fr"
CAPTCHA_ERROR_COLLECTION_PREFIX = "CAPTCHA_ERROR_" CAPTCHA_ERROR_COLLECTION_PREFIX = "CAPTCHA_ERROR_"
@@ -36,7 +36,7 @@ class MongoDbManager:
except Exception as Error: except Exception as Error:
self.logger.info(Error) self.logger.info(Error)
def insert_email(self, reserve: Mail): def insert_email(self, reserve: MailAddress):
try: try:
collection_to_use = self.db[EMAIL_LIST] collection_to_use = self.db[EMAIL_LIST]
collection_to_use.replace_one(filter={'_id': reserve.mail, }, replacement=reserve.to_firestore_dict(), collection_to_use.replace_one(filter={'_id': reserve.mail, }, replacement=reserve.to_firestore_dict(),
@@ -161,6 +161,12 @@ class MongoDbManager:
except Exception as error: except Exception as error:
self.logger.info(error) self.logger.info(error)
def link_validated_for_result(self, link: str):
id = link.split("/")[-1]
collection_name = str(datetime.date.today())
collection = self.db[collection_name]
collection.find_one_and_update({'_id': id}, {"$set": {"url_validated": "True"}}, upsert=False)
MONGO_STORE_MANAGER = MongoDbManager() MONGO_STORE_MANAGER = MongoDbManager()
src/logs/LogSender.py
+4
View File
@@ -30,6 +30,7 @@ LOG_APPOINTMENT_ERROR = "APPOINTMENT_ERROR"
LOG_APPOINTMENT_TIMEOUT = "TIMEOUT" LOG_APPOINTMENT_TIMEOUT = "TIMEOUT"
LOG_APPOINTMENT_CONTACT_NOT_FOUND = "CONTACT_NOT_FOUND" LOG_APPOINTMENT_CONTACT_NOT_FOUND = "CONTACT_NOT_FOUND"
LOG_APPOINTMENT_SUCCESS = "SUCCESS" LOG_APPOINTMENT_SUCCESS = "SUCCESS"
URL_VALIDATION_SUCCESS = "URL_VALIDATION_SUCCESS"
custom_retry_strategy = oci.retry.RetryStrategyBuilder( custom_retry_strategy = oci.retry.RetryStrategyBuilder(
# Make up to 10 service calls # Make up to 10 service calls
@@ -90,6 +91,9 @@ class LogSender:
msg = "{}, email: {}".format(result.message, result.email) msg = "{}, email: {}".format(result.message, result.email)
self.send_log(msg, type=LOG_APPOINTMENT_ERROR) self.send_log(msg, type=LOG_APPOINTMENT_ERROR)
def send_url_validation_result(self):
self.send_log(msg='', type=URL_VALIDATION_SUCCESS)
def send_error(self, msg: str): def send_error(self, msg: str):
self.send_log(msg=msg, type=LOG_ERROR) self.send_log(msg=msg, type=LOG_ERROR)
src/mail/__init__.py
View File
src/mail/mail_reader.py
+139
View File
@@ -0,0 +1,139 @@
import email
import imaplib
import re
from concurrent.futures import ThreadPoolExecutor
from email.header import decode_header
from email.message import Message
from builtins import list
from src import params
from src.db.mongo_manager import MONGO_STORE_MANAGER
from src.pojo.mail.mail_pojo import MailPojo
from src.proxy.proxy_type import ProxyType
from src.workers.link_validator import LinkValidator
AOL_IMAP_SERVER = "imap.aol.com"
VALIDATION_URL_SUBJECT = 'Validation de votre demande de rendez-vous'
VALIDATION_URL_REGEX = """https:\/\/rendezvousparis.hermes.com\/client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+"""
class MailReader():
def __init__(self, login, password):
self.login = login
self.password = password
def read_emails(self, email_number=0) -> list:
# create an IMAP4 class with SSL
mail_list = []
imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER)
# authenticate
imap.login(self.login, password)
status, messages = imap.select("INBOX")
# total number of emails
messages = int(messages[0])
for i in range(messages, 0, -1):
# fetch the email message by ID
res, msg = imap.fetch(str(i), "(RFC822)")
# res, msg = imap.fetch(str(i))
body = ''
for response in msg:
if isinstance(response, tuple):
# parse a bytes email into a message object
msg = email.message_from_bytes(response[1])
# decode the email subject
subject, subject_encoded = decode_header(msg["Subject"])[0]
received_date = msg["Date"]
if isinstance(subject, bytes):
# if it's a bytes, decode to str
subject = subject.decode(subject_encoded)
# decode email sender
from_address, subject_encoded = decode_header(msg.get("From"))[0]
if isinstance(from_address, bytes):
from_address = from_address.decode(subject_encoded)
print("Subject:", subject)
print("From:", from_address)
# if the email message is multipart
if msg.is_multipart():
# iterate over email parts
for part in msg.walk():
try:
# get the email body
payloads = part.get_payload()
if isinstance(payloads, list):
for payload in payloads:
if isinstance(payload, Message):
body = body + payload.get_payload(decode=True).decode("iso-8859-1")
# print(body)
except Exception as Error:
print(Error)
else:
body = msg.get_payload(decode=True).decode()
print(body)
if VALIDATION_URL_SUBJECT in subject:
mail = MailPojo(subject=subject, body=body, from_address=from_address)
mail_list.append(mail)
# close the connection and logout
imap.close()
imap.logout()
return mail_list
hermes_email = "no-reply@hermes.com"
# account credentials
# username = "appointment2022@aol.com"
# password = "gyilpmvyyvlcaviq"
username = "chenpeijun@aol.com"
password = "ytifuwguknzifqyb"
def clean(text):
# clean text for creating a folder
return "".join(c if c.isalnum() else "_" for c in text)
def need_to_valid_url(url: str, successful_items) -> bool:
print("url is :" + url)
parts = url.split('/')
id = parts[5]
if len(id) == 6:
for item in successful_items:
if item.url_validated is not None:
print("id:{}, status:{} ".format(id, str(item.url_validated)))
if item.id == id:
if item.url_validated is not None:
return not item.url_validated
else:
# if url_validated is None
return True
# return True by default
return False
else:
print("id not valid:{}".format(id))
return False
# check whether the url has already been clicked
if __name__ == '__main__':
mail_reader = MailReader(username, password)
successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
list = mail_reader.read_emails()
with ThreadPoolExecutor(max_workers=10) as executor:
for mail in list:
match = re.search(VALIDATION_URL_REGEX, mail.body)
if match:
url = match.group(0)
if need_to_valid_url(url, successful_items):
url_validator = LinkValidator(url)
print("need to validate url: " + url)
# .start_page(params.get_proxy(ProxyType.BRIGHT_DATA))
executor.submit(url_validator.start_page, params.get_proxy(ProxyType.BRIGHT_DATA))
else:
print("do not need to click url --> {}".format(mail))
# find link from mails
print(list)
src/pojo/MailPojo.py
-14
View File
@@ -1,14 +0,0 @@
class MailPojo:
email: str
def __init__(self, email: str):
self.email = email
@staticmethod
def from_firestore_dict(source):
email = source['email']
result = MailPojo(email=email)
return result
def __repr__(self):
return "email = " + self.email
src/pojo/ReserveResultPojo.py
+5
View File
@@ -31,6 +31,7 @@ class ReserveResultPojo:
ccid: str = "" ccid: str = ""
source_from: str = config.LOG_SOURCE source_from: str = config.LOG_SOURCE
store_type = 0 store_type = 0
url_validated = None
@staticmethod @staticmethod
def from_firestore_dict(source): def from_firestore_dict(source):
@@ -68,6 +69,9 @@ class ReserveResultPojo:
if 'store_type' in source: if 'store_type' in source:
store_type = source['store_type'] store_type = source['store_type']
result.store_type = store_type result.store_type = store_type
if 'url_validated' in source:
url_validated = source['url_validated']
result.url_validated = bool(url_validated)
result.id = id result.id = id
return result return result
@@ -88,6 +92,7 @@ class ReserveResultPojo:
u'source_from': self.source_from, u'source_from': self.source_from,
u'store_type': self.store_type, u'store_type': self.store_type,
u'accepted': self.accepted, u'accepted': self.accepted,
u'url_validated': self.url_validated,
} }
return dest return dest
src/pojo/mail/__init__.py
View File
src/pojo/mail_pojo.py → src/pojo/mail/mail_pojo.py
+12 -1
View File
@@ -1,4 +1,4 @@
class Mail: class MailAddress:
def __init__(self, mail, password): def __init__(self, mail, password):
self.mail = mail self.mail = mail
self.password = password self.password = password
@@ -12,3 +12,14 @@ class Mail:
u'password': self.password u'password': self.password
} }
return dest return dest
class MailPojo:
from_address: str
body: str
subject: str
def __init__(self, from_address, body, subject):
self.body = body
self.subject = subject
self.from_address = from_address
src/utils/excel_reader.py
+8 -8
View File
@@ -8,7 +8,7 @@ import xlsxwriter
from src.config import CONTACT_LIST_FILE from src.config import CONTACT_LIST_FILE
from src.db.mongo_manager import MONGO_STORE_MANAGER from src.db.mongo_manager import MONGO_STORE_MANAGER
from src.pojo.contact_pojo import ContactPojo from src.pojo.contact_pojo import ContactPojo
from src.pojo.mail_pojo import Mail from src.pojo.mail.mail_pojo import MailAddress
from src.utils.generate_random_passport_id import get_random_passport_id_number from src.utils.generate_random_passport_id import get_random_passport_id_number
phone_number_prefix = ['6'] phone_number_prefix = ['6']
@@ -55,7 +55,7 @@ class ExcelHelper:
if contact_dict['mail']: if contact_dict['mail']:
mail = contact_dict['mail'].strip() mail = contact_dict['mail'].strip()
pwd = contact_dict['password'] pwd = contact_dict['password']
contact = Mail(mail, pwd) contact = MailAddress(mail, pwd)
contact_list.append(contact) contact_list.append(contact)
return contact_list return contact_list
@@ -124,7 +124,7 @@ def get_random_id_number() -> str:
return ran return ran
def write_new_contacts_to_excel(valid_contacts: list): def write_new_contacts_to_excel(valid_contacts: list, generate_passport = True):
row = 0 row = 0
col = 0 col = 0
# Create a workbook and add a worksheet. # Create a workbook and add a worksheet.
@@ -151,9 +151,9 @@ def write_new_contacts_to_excel(valid_contacts: list):
if __name__ == '__main__': if __name__ == '__main__':
excel_reader = ExcelHelper() excel_reader = ExcelHelper()
# contacts = excel_reader.read_names("/Users/lpan/Downloads/real_contacts.xlsx") contacts = excel_reader.read_names("/Users/lpan/Downloads/real_contacts_31.xls")
# print(contacts) print(contacts)
# write_new_contacts_to_excel(valid_contacts=contacts) write_new_contacts_to_excel(valid_contacts=contacts)
for mail in excel_reader.read_mails_and_pwd(): # for mail in excel_reader.read_mails_and_pwd():
MONGO_STORE_MANAGER.insert_email(mail) # MONGO_STORE_MANAGER.insert_email(mail)
src/utils/real_contacts_52.xlsx
BIN
View File
Binary file not shown.
src/workers/commandor_page.py
+2 -3
View File
@@ -8,8 +8,6 @@ import time
import traceback import traceback
from typing import Union from typing import Union
from playwright.sync_api import sync_playwright
from src import params, definitions from src import params, definitions
from src.db.mongo_manager import MONGO_STORE_MANAGER from src.db.mongo_manager import MONGO_STORE_MANAGER
from src.pojo.ModeEnum import ModeEnum from src.pojo.ModeEnum import ModeEnum
@@ -31,6 +29,7 @@ MESSAGE_FIELD_CLASS = ".message"
BLANK_URL = "about:blank" BLANK_URL = "about:blank"
CONFIRMED_MESSAGE = "Your request for a Leather Goods appointment has been registered" CONFIRMED_MESSAGE = "Your request for a Leather Goods appointment has been registered"
CONFIRMED_MESSAGE_FR = "Votre demande de rendez-vous Maroquinerie a bien été enregistrée et nous vous en remercions." CONFIRMED_MESSAGE_FR = "Votre demande de rendez-vous Maroquinerie a bien été enregistrée et nous vous en remercions."
MESSAGE_URL_VALIDATION_FR = "Nous avons envoyé un lien par e-mail."
DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today." DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today."
DOUBLE_REQUEST_ERROR_MESSAGE_FR = "Une demande avec les données saisies a déjà été validée aujourd’hui." DOUBLE_REQUEST_ERROR_MESSAGE_FR = "Une demande avec les données saisies a déjà été validée aujourd’hui."
TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests" TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests"
@@ -162,7 +161,7 @@ class CommandorPage:
self.fill_fields() self.fill_fields()
try: try:
message = self.page.content() message = self.page.content()
if CONFIRMED_MESSAGE in message or CONFIRMED_MESSAGE_FR in message: if CONFIRMED_MESSAGE_FR in message or MESSAGE_URL_VALIDATION_FR in message:
# publish the successful message # publish the successful message
self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url) self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url)
self.get_errors() self.get_errors()
src/workers/link_validator.py
+118
View File
@@ -0,0 +1,118 @@
import logging
import random
import traceback
from typing import Union
import sys
import time
from src import params
from src.db.mongo_manager import MONGO_STORE_MANAGER
from src.pojo.ReserveResultPojo import PublishType
from src.proxy.proxy_type import ProxyType
from src.workers.TlsPlaywright import TlsPlaywright
OTP_FIELD_ID = "#sms_code"
TIME_OUT = 10 * 60 * 1000 # 10 mins
PAGE_TIMEOUT = 40000
CONFIRMED_MESSAGE_FR = "Votre demande de rendez-vous Maroquinerie a bien été enregistrée et nous vous en remercions."
SORRY_SENTENCE_FR = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci"
class LinkValidator:
tls = TlsPlaywright()
def __init__(self, link: str, proxy_type=ProxyType.BRIGHT_DATA, headless=False):
self.is_finished = False
self.link = link
self.proxy_type = proxy_type
self.is_event_sent = False
self.is_captcha_in_error = False
self.is_filling_fields = False
self.headless = headless
self.logger = logging.getLogger("LinkValidator")
def on_success(self):
self.logger.info("on_success called.")
self.is_finished = True
if not self.is_event_sent:
self.logger.info("will send successful event")
params.oracle_log_sender.send_url_validation_result()
self.is_event_sent = True
def timeout_occurred(self):
params.oracle_log_sender.send_timeout_log(self.link)
self.logger.info("will close timeout modem")
self.termine()
def _run(self, proxy):
self.logger.info("will start browser")
# reset otp_value to None
devices = random.choice(params.DEVICES)
first_page = None
while first_page is None:
first_page = self.start_browser(proxy, self.tls.playwright, devices)
proxy = params.get_proxy(self.proxy_type)
def start_browser(self, proxy, pwright, device) -> Union[str, None]:
try:
self.browser = pwright.webkit.launch(headless=self.headless, timeout=PAGE_TIMEOUT, proxy=proxy)
self.logger.info("模拟设备: " + device)
simulated_mobile = pwright.devices[device]
context = self.browser.new_context(**simulated_mobile, locale='fr-FR')
self.page = context.new_page()
# hide webdriver information
self.page.add_init_script("""() => {
Object.defineProperty(navigator,'webdriver',{get: () => undefined});
Object.defineProperty(navigator, 'platform', {
get: () => {
return "iPhone";
}});
}
""")
self.page.on("load", self._on_page_loaded)
self.page.goto(self.link, timeout=PAGE_TIMEOUT)
return self.page.content()
except Exception as error:
params.oracle_log_sender.send_error(str(error))
traceback.print_exc(*sys.exc_info())
self.logger.exception(error)
self.logger.info("will close browser")
self.browser.close()
return None
def start_page(self, proxy):
self._run(proxy)
def _on_page_loaded(self):
self.logger.info("页面加载完毕")
self.logger.info("url is " + self.page.url)
try:
message = self.page.content()
if CONFIRMED_MESSAGE_FR in message:
# publish the successful message
self.publish_message_to_queue(PublishType.SUCCESS)
elif SORRY_SENTENCE_FR in message:
# publish the successful message
self.publish_message_to_queue(PublishType.SUCCESS)
except Exception as error:
self.logger.error(error)
def on_document_loaded(self):
self.logger.info("on_document_loaded called")
def _handle_errors(self, erro_content: str):
pass
def termine(self):
self.logger.info("will close browser")
time.sleep(1)
self.browser.close()
def publish_message_to_queue(self, status: PublishType):
# create the message
MONGO_STORE_MANAGER.link_validated_for_result(self.page.url)
if status is PublishType.SUCCESS:
self.on_success()
time.sleep(2)
self.browser.close()
validation_url_executor.py
+5
View File
@@ -0,0 +1,5 @@
from src.mail.mail_reader import MailReader
if __name__ == '__main__':
# read emails
mail_reader = MailReader()