diff --git a/main.py b/main.py
index 9d810ea..f1ce910 100644
--- a/main.py
+++ b/main.py
@@ -61,5 +61,6 @@ def get_proxy(proxy_type=ProxyType.BRIGHT_DATA):
if __name__ == '__main__':
# 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机，传入1的时候是总店
- # start_book(1, 100, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=True)
- recheck_the_captcha_error_contacts(store_type=1, mode=ModeEnum.AUTOMATIC, on_no_contact_found=lambda: None, headless=True)
+ start_book(744, 792, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=False)
+ # start_book(1172, 1324, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=False)
+ # recheck_the_captcha_error_contacts(store_type=1, mode=ModeEnum.AUTOMATIC, on_no_contact_found=lambda: None, headless=False)
diff --git a/src/Test/index.html b/src/Test/index.html
new file mode 100644
index 0000000..d19b96e
--- /dev/null
+++ b/src/Test/index.html
@@ -0,0 +1,48 @@
+
+
+
+
+
+
+
+
+
+Post-scriptum La Poste
+
+Ce message est confidentiel. Sous reserve de tout accord conclu par
+ecrit entre vous et La Poste, son contenu ne represente en aucun cas un engagement de la part de La Poste. Toute publication, utilisation ou diffusion, meme partielle, doit etre autorisee prealablement. Si vous n'etes pas destinataire de ce message, merci d'en avertir immediatement
+l'expediteur.
+
diff --git a/src/check_results.py b/src/check_results.py
index 0bfed53..761fb1c 100644
--- a/src/check_results.py
+++ b/src/check_results.py
@@ -112,10 +112,10 @@ def check_results(headless=False):
reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
print("size is " + str(len(reserve_list)))
start_check(reserve_list, firestore_collection, headless, need_send_email=False)
- # reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
- # start_check(reserve_list, firestore_collection, headless, need_send_email=True)
+ reserve_list = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+ start_check(reserve_list, firestore_collection, headless, need_send_email=True)
# copy the accepted info to the accepted collection
- # migre_accepted_appointment(str(datetime.date.today()))
+ migre_accepted_appointment(str(datetime.date.today()))
def start_check(reserve_list, firestore_collection, headless: bool, need_send_email: bool):
diff --git a/src/db/DbManager.py b/src/db/DbManager.py
index d4ae20d..019cb7d 100644
--- a/src/db/DbManager.py
+++ b/src/db/DbManager.py
@@ -5,7 +5,6 @@ import firebase_admin
from firebase_admin import credentials, firestore
from src import params, config
-from src.pojo.MailPojo import MailPojo
from src.pojo.ReserveResultPojo import ReserveResultPojo
from src.pojo.ResultEnum import ResultEnum
from src.pojo.SimInfoPojo import SimInfoPojo
@@ -104,12 +103,3 @@ class DataManager:
params.oracle_log_sender.send_read_db_event("read_contacts_from_db")
contact_collection = self._db.collection(CONTACT_COLLECTION_NAME)
return contact_collection
-
- def get_mail_list(self) -> list:
- params.oracle_log_sender.send_read_db_event("get_mail_list")
- mail_collection = self._db.collection(MAIL_COLLECTION_NAME)
- mail_list = []
- for mail in mail_collection.stream():
- mail_pojo = MailPojo.from_firestore_dict(mail.to_dict())
- mail_list.append(mail_pojo)
- return mail_list
diff --git a/src/db/mongo_manager.py b/src/db/mongo_manager.py
index 903542e..cbbf0de 100644
--- a/src/db/mongo_manager.py
+++ b/src/db/mongo_manager.py
@@ -9,7 +9,7 @@ from src.pojo.ResultEnum import ResultEnum
from src.pojo.accepted_appointment_pojo import AcceptedAppointmentPojo
from src.pojo.black_contact import BlackContactPojo
from src.pojo.contact_pojo import ContactPojo
-from src.pojo.mail_pojo import Mail
+from src.pojo.mail.mail_pojo import MailAddress
MONGO_DB_URL = "mongo.lpaconsulting.fr"
CAPTCHA_ERROR_COLLECTION_PREFIX = "CAPTCHA_ERROR_"
@@ -36,7 +36,7 @@ class MongoDbManager:
except Exception as Error:
self.logger.info(Error)
- def insert_email(self, reserve: Mail):
+ def insert_email(self, reserve: MailAddress):
try:
collection_to_use = self.db[EMAIL_LIST]
collection_to_use.replace_one(filter={'_id': reserve.mail, }, replacement=reserve.to_firestore_dict(),
@@ -161,6 +161,12 @@ class MongoDbManager:
except Exception as error:
self.logger.info(error)
+ def link_validated_for_result(self, link: str):
+ id = link.split("/")[-1]
+ collection_name = str(datetime.date.today())
+ collection = self.db[collection_name]
+ collection.find_one_and_update({'_id': id}, {"$set": {"url_validated": "True"}}, upsert=False)
+
MONGO_STORE_MANAGER = MongoDbManager()
diff --git a/src/logs/LogSender.py b/src/logs/LogSender.py
index 3c6a0af..e68e1b0 100644
--- a/src/logs/LogSender.py
+++ b/src/logs/LogSender.py
@@ -30,6 +30,7 @@ LOG_APPOINTMENT_ERROR = "APPOINTMENT_ERROR"
LOG_APPOINTMENT_TIMEOUT = "TIMEOUT"
LOG_APPOINTMENT_CONTACT_NOT_FOUND = "CONTACT_NOT_FOUND"
LOG_APPOINTMENT_SUCCESS = "SUCCESS"
+URL_VALIDATION_SUCCESS = "URL_VALIDATION_SUCCESS"
custom_retry_strategy = oci.retry.RetryStrategyBuilder(
# Make up to 10 service calls
@@ -90,6 +91,9 @@ class LogSender:
msg = "{}, email: {}".format(result.message, result.email)
self.send_log(msg, type=LOG_APPOINTMENT_ERROR)
+ def send_url_validation_result(self):
+ self.send_log(msg='', type=URL_VALIDATION_SUCCESS)
+
def send_error(self, msg: str):
self.send_log(msg=msg, type=LOG_ERROR)
diff --git a/src/mail/__init__.py b/src/mail/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/mail/mail_reader.py b/src/mail/mail_reader.py
new file mode 100644
index 0000000..ed338eb
--- /dev/null
+++ b/src/mail/mail_reader.py
@@ -0,0 +1,139 @@
+import email
+import imaplib
+import re
+from concurrent.futures import ThreadPoolExecutor
+from email.header import decode_header
+from email.message import Message
+
+from builtins import list
+
+from src import params
+from src.db.mongo_manager import MONGO_STORE_MANAGER
+from src.pojo.mail.mail_pojo import MailPojo
+from src.proxy.proxy_type import ProxyType
+from src.workers.link_validator import LinkValidator
+
+AOL_IMAP_SERVER = "imap.aol.com"
+VALIDATION_URL_SUBJECT = 'Validation de votre demande de rendez-vous'
+VALIDATION_URL_REGEX = """https:\/\/rendezvousparis.hermes.com\/client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+"""
+
+
+class MailReader():
+ def __init__(self, login, password):
+ self.login = login
+ self.password = password
+
+ def read_emails(self, email_number=0) -> list:
+ # create an IMAP4 class with SSL
+ mail_list = []
+ imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER)
+ # authenticate
+ imap.login(self.login, password)
+ status, messages = imap.select("INBOX")
+ # total number of emails
+ messages = int(messages[0])
+ for i in range(messages, 0, -1):
+ # fetch the email message by ID
+ res, msg = imap.fetch(str(i), "(RFC822)")
+ # res, msg = imap.fetch(str(i))
+ body = ''
+ for response in msg:
+ if isinstance(response, tuple):
+ # parse a bytes email into a message object
+ msg = email.message_from_bytes(response[1])
+ # decode the email subject
+ subject, subject_encoded = decode_header(msg["Subject"])[0]
+ received_date = msg["Date"]
+ if isinstance(subject, bytes):
+ # if it's a bytes, decode to str
+ subject = subject.decode(subject_encoded)
+ # decode email sender
+ from_address, subject_encoded = decode_header(msg.get("From"))[0]
+ if isinstance(from_address, bytes):
+ from_address = from_address.decode(subject_encoded)
+ print("Subject:", subject)
+ print("From:", from_address)
+ # if the email message is multipart
+ if msg.is_multipart():
+ # iterate over email parts
+ for part in msg.walk():
+ try:
+ # get the email body
+ payloads = part.get_payload()
+ if isinstance(payloads, list):
+ for payload in payloads:
+ if isinstance(payload, Message):
+ body = body + payload.get_payload(decode=True).decode("iso-8859-1")
+ # print(body)
+ except Exception as Error:
+ print(Error)
+ else:
+ body = msg.get_payload(decode=True).decode()
+ print(body)
+ if VALIDATION_URL_SUBJECT in subject:
+ mail = MailPojo(subject=subject, body=body, from_address=from_address)
+ mail_list.append(mail)
+ # close the connection and logout
+ imap.close()
+ imap.logout()
+ return mail_list
+
+
+hermes_email = "no-reply@hermes.com"
+# account credentials
+# username = "appointment2022@aol.com"
+# password = "gyilpmvyyvlcaviq"
+
+
+username = "chenpeijun@aol.com"
+password = "ytifuwguknzifqyb"
+
+
+def clean(text):
+ # clean text for creating a folder
+ return "".join(c if c.isalnum() else "_" for c in text)
+
+
+def need_to_valid_url(url: str, successful_items) -> bool:
+ print("url is :" + url)
+ parts = url.split('/')
+ id = parts[5]
+ if len(id) == 6:
+ for item in successful_items:
+ if item.url_validated is not None:
+ print("id:{}, status:{} ".format(id, str(item.url_validated)))
+ if item.id == id:
+ if item.url_validated is not None:
+ return not item.url_validated
+ else:
+ # if url_validated is None
+ return True
+ # return True by default
+ return False
+ else:
+ print("id not valid:{}".format(id))
+ return False
+
+
+# check whether the url has already been clicked
+
+
+if __name__ == '__main__':
+ mail_reader = MailReader(username, password)
+ successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+ list = mail_reader.read_emails()
+ with ThreadPoolExecutor(max_workers=10) as executor:
+ for mail in list:
+ match = re.search(VALIDATION_URL_REGEX, mail.body)
+ if match:
+ url = match.group(0)
+ if need_to_valid_url(url, successful_items):
+ url_validator = LinkValidator(url)
+ print("need to validate url: " + url)
+ # .start_page(params.get_proxy(ProxyType.BRIGHT_DATA))
+ executor.submit(url_validator.start_page, params.get_proxy(ProxyType.BRIGHT_DATA))
+ else:
+ print("do not need to click url --> {}".format(mail))
+
+ # find link from mails
+ print(list)
diff --git a/src/pojo/MailPojo.py b/src/pojo/MailPojo.py
deleted file mode 100644
index 73364d1..0000000
--- a/src/pojo/MailPojo.py
+++ /dev/null
@@ -1,14 +0,0 @@
-class MailPojo:
- email: str
-
- def __init__(self, email: str):
- self.email = email
-
- @staticmethod
- def from_firestore_dict(source):
- email = source['email']
- result = MailPojo(email=email)
- return result
-
- def __repr__(self):
- return "email = " + self.email
diff --git a/src/pojo/ReserveResultPojo.py b/src/pojo/ReserveResultPojo.py
index 317ffe6..2e75dad 100644
--- a/src/pojo/ReserveResultPojo.py
+++ b/src/pojo/ReserveResultPojo.py
@@ -31,6 +31,7 @@ class ReserveResultPojo:
ccid: str = ""
source_from: str = config.LOG_SOURCE
store_type = 0
+ url_validated = None
@staticmethod
def from_firestore_dict(source):
@@ -68,6 +69,9 @@ class ReserveResultPojo:
if 'store_type' in source:
store_type = source['store_type']
result.store_type = store_type
+ if 'url_validated' in source:
+ url_validated = source['url_validated']
+ result.url_validated = bool(url_validated)
result.id = id
return result
@@ -88,6 +92,7 @@ class ReserveResultPojo:
u'source_from': self.source_from,
u'store_type': self.store_type,
u'accepted': self.accepted,
+ u'url_validated': self.url_validated,
}
return dest
diff --git a/src/pojo/mail/__init__.py b/src/pojo/mail/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/pojo/mail_pojo.py b/src/pojo/mail/mail_pojo.py
similarity index 58%
rename from src/pojo/mail_pojo.py
rename to src/pojo/mail/mail_pojo.py
index 7e12524..231025d 100644
--- a/src/pojo/mail_pojo.py
+++ b/src/pojo/mail/mail_pojo.py
@@ -1,4 +1,4 @@
-class Mail:
+class MailAddress:
def __init__(self, mail, password):
self.mail = mail
self.password = password
@@ -12,3 +12,14 @@ class Mail:
u'password': self.password
}
return dest
+
+
+class MailPojo:
+ from_address: str
+ body: str
+ subject: str
+
+ def __init__(self, from_address, body, subject):
+ self.body = body
+ self.subject = subject
+ self.from_address = from_address
diff --git a/src/utils/excel_reader.py b/src/utils/excel_reader.py
index edbe05a..3854935 100644
--- a/src/utils/excel_reader.py
+++ b/src/utils/excel_reader.py
@@ -8,7 +8,7 @@ import xlsxwriter
from src.config import CONTACT_LIST_FILE
from src.db.mongo_manager import MONGO_STORE_MANAGER
from src.pojo.contact_pojo import ContactPojo
-from src.pojo.mail_pojo import Mail
+from src.pojo.mail.mail_pojo import MailAddress
from src.utils.generate_random_passport_id import get_random_passport_id_number
phone_number_prefix = ['6']
@@ -55,7 +55,7 @@ class ExcelHelper:
if contact_dict['mail']:
mail = contact_dict['mail'].strip()
pwd = contact_dict['password']
- contact = Mail(mail, pwd)
+ contact = MailAddress(mail, pwd)
contact_list.append(contact)
return contact_list
@@ -124,7 +124,7 @@ def get_random_id_number() -> str:
return ran
-def write_new_contacts_to_excel(valid_contacts: list):
+def write_new_contacts_to_excel(valid_contacts: list, generate_passport = True):
row = 0
col = 0
# Create a workbook and add a worksheet.
@@ -151,9 +151,9 @@ def write_new_contacts_to_excel(valid_contacts: list):
if __name__ == '__main__':
excel_reader = ExcelHelper()
- # contacts = excel_reader.read_names("/Users/lpan/Downloads/real_contacts.xlsx")
- # print(contacts)
- # write_new_contacts_to_excel(valid_contacts=contacts)
- for mail in excel_reader.read_mails_and_pwd():
- MONGO_STORE_MANAGER.insert_email(mail)
+ contacts = excel_reader.read_names("/Users/lpan/Downloads/real_contacts_31.xls")
+ print(contacts)
+ write_new_contacts_to_excel(valid_contacts=contacts)
+ # for mail in excel_reader.read_mails_and_pwd():
+ # MONGO_STORE_MANAGER.insert_email(mail)
diff --git a/src/utils/real_contacts_52.xlsx b/src/utils/real_contacts_52.xlsx
new file mode 100644
index 0000000..0274f84
Binary files /dev/null and b/src/utils/real_contacts_52.xlsx differ
diff --git a/src/workers/commandor_page.py b/src/workers/commandor_page.py
index cdc45f5..e2790c9 100644
--- a/src/workers/commandor_page.py
+++ b/src/workers/commandor_page.py
@@ -8,8 +8,6 @@ import time
import traceback
from typing import Union
-from playwright.sync_api import sync_playwright
-
from src import params, definitions
from src.db.mongo_manager import MONGO_STORE_MANAGER
from src.pojo.ModeEnum import ModeEnum
@@ -31,6 +29,7 @@ MESSAGE_FIELD_CLASS = ".message"
BLANK_URL = "about:blank"
CONFIRMED_MESSAGE = "Your request for a Leather Goods appointment has been registered"
CONFIRMED_MESSAGE_FR = "Votre demande de rendez-vous Maroquinerie a bien été enregistrée et nous vous en remercions."
+MESSAGE_URL_VALIDATION_FR = "Nous avons envoyé un lien par e-mail."
DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today."
DOUBLE_REQUEST_ERROR_MESSAGE_FR = "Une demande avec les données saisies a déjà été validée aujourd’hui."
TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests"
@@ -162,7 +161,7 @@ class CommandorPage:
self.fill_fields()
try:
message = self.page.content()
- if CONFIRMED_MESSAGE in message or CONFIRMED_MESSAGE_FR in message:
+ if CONFIRMED_MESSAGE_FR in message or MESSAGE_URL_VALIDATION_FR in message:
# publish the successful message
self.publish_message_to_queue(self.contact, PublishType.SUCCESS, self.page.url)
self.get_errors()
diff --git a/src/workers/link_validator.py b/src/workers/link_validator.py
new file mode 100644
index 0000000..bad0a84
--- /dev/null
+++ b/src/workers/link_validator.py
@@ -0,0 +1,118 @@
+import logging
+import random
+import traceback
+from typing import Union
+
+import sys
+import time
+
+from src import params
+from src.db.mongo_manager import MONGO_STORE_MANAGER
+from src.pojo.ReserveResultPojo import PublishType
+from src.proxy.proxy_type import ProxyType
+from src.workers.TlsPlaywright import TlsPlaywright
+
+OTP_FIELD_ID = "#sms_code"
+TIME_OUT = 10 * 60 * 1000 # 10 mins
+PAGE_TIMEOUT = 40000
+CONFIRMED_MESSAGE_FR = "Votre demande de rendez-vous Maroquinerie a bien été enregistrée et nous vous en remercions."
+SORRY_SENTENCE_FR = "nous sommes sincèrement désolés de n'avoir pu vous satisfaire cette fois-ci"
+
+
+class LinkValidator:
+ tls = TlsPlaywright()
+
+ def __init__(self, link: str, proxy_type=ProxyType.BRIGHT_DATA, headless=False):
+ self.is_finished = False
+ self.link = link
+ self.proxy_type = proxy_type
+ self.is_event_sent = False
+ self.is_captcha_in_error = False
+ self.is_filling_fields = False
+ self.headless = headless
+ self.logger = logging.getLogger("LinkValidator")
+
+ def on_success(self):
+ self.logger.info("on_success called.")
+ self.is_finished = True
+ if not self.is_event_sent:
+ self.logger.info("will send successful event")
+ params.oracle_log_sender.send_url_validation_result()
+ self.is_event_sent = True
+
+ def timeout_occurred(self):
+ params.oracle_log_sender.send_timeout_log(self.link)
+ self.logger.info("will close timeout modem")
+ self.termine()
+
+ def _run(self, proxy):
+ self.logger.info("will start browser")
+ # reset otp_value to None
+ devices = random.choice(params.DEVICES)
+ first_page = None
+ while first_page is None:
+ first_page = self.start_browser(proxy, self.tls.playwright, devices)
+ proxy = params.get_proxy(self.proxy_type)
+
+ def start_browser(self, proxy, pwright, device) -> Union[str, None]:
+ try:
+ self.browser = pwright.webkit.launch(headless=self.headless, timeout=PAGE_TIMEOUT, proxy=proxy)
+ self.logger.info("模拟设备: " + device)
+ simulated_mobile = pwright.devices[device]
+ context = self.browser.new_context(**simulated_mobile, locale='fr-FR')
+ self.page = context.new_page()
+ # hide webdriver information
+ self.page.add_init_script("""() => {
+ Object.defineProperty(navigator,'webdriver',{get: () => undefined});
+ Object.defineProperty(navigator, 'platform', {
+ get: () => {
+ return "iPhone";
+ }});
+}
+ """)
+ self.page.on("load", self._on_page_loaded)
+ self.page.goto(self.link, timeout=PAGE_TIMEOUT)
+ return self.page.content()
+ except Exception as error:
+ params.oracle_log_sender.send_error(str(error))
+ traceback.print_exc(*sys.exc_info())
+ self.logger.exception(error)
+ self.logger.info("will close browser")
+ self.browser.close()
+ return None
+
+ def start_page(self, proxy):
+ self._run(proxy)
+
+ def _on_page_loaded(self):
+ self.logger.info("页面加载完毕")
+ self.logger.info("url is " + self.page.url)
+ try:
+ message = self.page.content()
+ if CONFIRMED_MESSAGE_FR in message:
+ # publish the successful message
+ self.publish_message_to_queue(PublishType.SUCCESS)
+ elif SORRY_SENTENCE_FR in message:
+ # publish the successful message
+ self.publish_message_to_queue(PublishType.SUCCESS)
+ except Exception as error:
+ self.logger.error(error)
+
+ def on_document_loaded(self):
+ self.logger.info("on_document_loaded called")
+
+ def _handle_errors(self, erro_content: str):
+ pass
+
+ def termine(self):
+ self.logger.info("will close browser")
+ time.sleep(1)
+ self.browser.close()
+
+ def publish_message_to_queue(self, status: PublishType):
+ # create the message
+ MONGO_STORE_MANAGER.link_validated_for_result(self.page.url)
+ if status is PublishType.SUCCESS:
+ self.on_success()
+ time.sleep(2)
+ self.browser.close()
diff --git a/validation_url_executor.py b/validation_url_executor.py
new file mode 100644
index 0000000..bdead3b
--- /dev/null
+++ b/validation_url_executor.py
@@ -0,0 +1,5 @@
+from src.mail.mail_reader import MailReader
+
+if __name__ == '__main__':
+ # read emails
+ mail_reader = MailReader()
\ No newline at end of file