From 0ad2a7bf4c23ae97c7785f4726249dd9c6dccc3d Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 23 Dec 2022 17:08:16 +0100 Subject: [PATCH 01/11] remove unused code --- src/mail/get_invoices.py | 46 ---------------------------------------- 1 file changed, 46 deletions(-) diff --git a/src/mail/get_invoices.py b/src/mail/get_invoices.py index 682e2ca..ae2f4fe 100644 --- a/src/mail/get_invoices.py +++ b/src/mail/get_invoices.py @@ -70,23 +70,6 @@ class InvoiceGetter(): mails_messages.extend(mail_list) return mail_list - # def downloadAllAttachmentsInInbox(self, imap): - # typ, items = imap.search(None, '(SUBJECT "{}" SINCE "{}")'.format(INVOICE_SUBJECT_fr, - # datetime.datetime.today().strftime( - # date_format))) - # items = items[0].split() - # self.downloaAttachmentsInEmail(imap, "./") - # - # def downloaAttachmentsInEmail(self, imap, outputdir): - # resp, data = imap.fetch(HERMES_INVOICE_EMAIL, "(BODY.PEEK[])") - # email_body = data[0][1] - # mail = email.message_from_string(email_body) - # if mail.get_content_maintype() != 'multipart': - # return - # for part in mail.walk(): - # if part.get_content_maintype() != 'multipart' and part.get('Content-Disposition') is not None: - # open(outputdir + '/' + part.get_filename(), 'wb').write(part.get_payload(decode=True)) - def _get_messages_from_folder(self, imap, subject, folder="INBOX") -> list: imap.select(folder) mail_messages = [] @@ -141,36 +124,7 @@ class InvoiceGetter(): return mail_messages -def clean(text): - # clean text for creating a folder - return "".join(c if c.isalnum() else "_" for c in text) - - -def need_to_valid_url(url: str, successful_items) -> bool: - if len(successful_items) == 0: - return False - print("url is :" + url) - parts = url.split('/') - id = parts[5] - if len(id) == 6: - for item in successful_items: - # if item.url_validated is not None: - # print("id:{}, status:{} ".format(id, str(item.url_validated))) - if item.id == id: - if item.url_validated is not None: - return not item.url_validated - else: - # if url_validated is None - return True - # return True by default - return False - else: - print("id not valid:{}".format(id)) - return False - - def get_invoices(): - # check time before start checking emails # get email address mail_list = MONGO_STORE_MANAGER.get_destination_emails() mail_list_to_check = [] From 22ef7707c490b0455edc7461ec179ba737d2426a Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Thu, 29 Dec 2022 18:05:39 +0100 Subject: [PATCH 02/11] correction on the logs --- src/mail/mail_reader.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/mail/mail_reader.py b/src/mail/mail_reader.py index 915d03f..510b53c 100644 --- a/src/mail/mail_reader.py +++ b/src/mail/mail_reader.py @@ -131,11 +131,6 @@ class MailReader(): return mail_messages -def clean(text): - # clean text for creating a folder - return "".join(c if c.isalnum() else "_" for c in text) - - def need_to_valid_url(url: str, successful_items) -> bool: if len(successful_items) == 0: return False @@ -167,7 +162,7 @@ def need_to_check_email(mail: str, successful_items) -> bool: for item in successful_items: if mail in item.email: if item.url_validated is not None: - print("url_validated for {} is {}".format(item.url_validated, mail)) + print("url_validated for {} is {}".format(mail, item.url_validated)) return not item.url_validated else: # if url-validated is none, need to check email @@ -205,10 +200,10 @@ def read_mails(): if match: url = match.group(0) if need_to_valid_url(url, successful_items): - # MONGO_STORE_MANAGER.save_links_to_validate(url) - url_validator = LinkValidator(url) + MONGO_STORE_MANAGER.save_links_to_validate(url) + # url_validator = LinkValidator(url) print("need to validate url: " + url) - executor.submit(url_validator.start_page, params.get_proxy(ProxyType.OXYLABS), True) + # executor.submit(url_validator.start_page, params.get_proxy(ProxyType.OXYLABS), False) else: print("do not need to click url --> {}".format(mail.mail_address)) From bfd54c1f1fb977f28121283f05a53cc19591402d Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Wed, 4 Jan 2023 10:53:27 +0100 Subject: [PATCH 03/11] remove unused code --- src/utils/send_email_mailjet.py | 21 +++++++++++++++------ src/workers/commandor_page.py | 3 +-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/utils/send_email_mailjet.py b/src/utils/send_email_mailjet.py index 7aaa5e0..d92ca57 100644 --- a/src/utils/send_email_mailjet.py +++ b/src/utils/send_email_mailjet.py @@ -9,14 +9,23 @@ api_key = "489274f04d5c155f81370fccc3904e20" api_secret = "edac41f0e1726ba49808dfb12204ecd6" mailjet = Client(auth=(api_key, api_secret), version='v3.1') from_email = "no-reply@lpaconsulting.fr" -# store = "Hermès Paris Faubourg" -store = "Hermès Paris Sèvres" +store = "Hermès Paris Faubourg" # store = "Hermès Paris George V" -# dest_email = "0429xiaohan@gmail.com" #小萝卜 -# dest_email = "1340219934@qq.com" #小于 +# store = "Hermès Paris Sèvres" +# dest_email = "928490803@qq.com" +# dest_email = "1757344572@qq.com" +# dest_email = "392417782@qq.com" # dest_email = "linghuili@hotmail.com" -dest_email = "landdu@outlook.com" -contact_name = "LI Jiaxi" +# dest_email = "fanfan19810617@icloud.com" +# dest_email = "fanchunying323@163.com" +# dest_email = "lytlaure@gmail.com" +# dest_email = "panleicim@gmail.com" +# dest_email = "sunrose75015@gmail.com" +dest_email = "1340219934@qq.com" +# dest_email = "lytlaure@gmail.com" +# dest_email = "m13805869912@163.com" +# dest_email = "tangliang0411@gmail.com" +contact_name = "DU Peijian" f = open(config.ROOT_DIR + "/templates/confirmed_rdv.html", "r") email_body = f.read() diff --git a/src/workers/commandor_page.py b/src/workers/commandor_page.py index fa4b490..99f2c12 100644 --- a/src/workers/commandor_page.py +++ b/src/workers/commandor_page.py @@ -97,7 +97,7 @@ class CommandorPage: self.otp_value = None # device = None device_key = random.choice(params.DEVICES) - device_key = random.sample(list(self.tls.playwright.devices), 1)[0] + # device_key = random.sample(list(self.tls.playwright.devices), 1)[0] device = self.tls.playwright.devices[device_key] # while device is None: # device_key = random.sample(list(self.tls.playwright.devices), 1)[0] @@ -417,7 +417,6 @@ class CommandorPage: def handle_captcha_error(self): MONGO_STORE_MANAGER.insert_captcha_error_contact(self.contact) - params.oracle_log_sender.send_captcha_error(self.contact) def check_is_blocked(self) -> bool: iframe = self.page.query_selector('iframe').content_frame() From 12202b1baf256095e388af1716dccddf7513a711 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Wed, 4 Jan 2023 23:29:07 +0100 Subject: [PATCH 04/11] add updated_at field to links_to_validate table --- src/db/mongo_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/db/mongo_manager.py b/src/db/mongo_manager.py index 64be987..1913a16 100644 --- a/src/db/mongo_manager.py +++ b/src/db/mongo_manager.py @@ -233,8 +233,10 @@ class MongoDbManager: def save_links_to_validate(self, link: str): collection_to_use = self.db[LINKS_TO_VALIDATE] + updated_at = time.strftime("%H:%M:%S", time.localtime()) collection_to_use.replace_one(filter={'_id': link, }, replacement={ - u'url': link + u'url': link, + "updated_at": updated_at }, upsert=True) From f00c4c2e2370a4c959d0c00e5768ba5695194eb2 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Wed, 4 Jan 2023 23:39:40 +0100 Subject: [PATCH 05/11] add auth to speetch to text server --- server.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/server.py b/server.py index e9b13b3..56357a8 100644 --- a/server.py +++ b/server.py @@ -5,15 +5,25 @@ from flask import Flask from flask import jsonify from flask import request from flask_cors import CORS, cross_origin +from flask_httpauth import HTTPTokenAuth from src.workers.SpeechToText import SpeechToText app = Flask(__name__) cors = CORS(app) app.config['CORS_HEADERS'] = 'Content-Type' +auth = HTTPTokenAuth(scheme='Bearer') + +secret_token = "97e36f7e-340e-4c02-b329-9415faee38c3" + + +@auth.verify_token +def verify_token(token): + return token == secret_token @app.route('/', methods=['POST']) +@auth.login_required @cross_origin() def post(): file_name = str(uuid.uuid4()) + ".wav" @@ -30,4 +40,4 @@ def post(): if __name__ == '__main__': - app.run(host='0.0.0.0', port=8000, debug=True) + app.run(host='0.0.0.0', port=8000, debug=False) From 3d9bb53d101d3705d530106c7a04930c75cbbcfa Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Wed, 4 Jan 2023 23:39:48 +0100 Subject: [PATCH 06/11] add auth to speetch to text server --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8ee4985..c238f9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,6 @@ openpyxl==3.0.9 google-cloud-firestore==2.4.0 PySimpleGUI==4.60.1 requests~=2.27.1 -Mako~=1.2.0 \ No newline at end of file +Mako~=1.2.0 +Flask~=2.2.2 +APScheduler~=3.9.1 \ No newline at end of file From 3f484ddba357debca620c8ecc9a2b25f3586334d Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Wed, 4 Jan 2023 23:48:50 +0100 Subject: [PATCH 07/11] add flask auth to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index c238f9e..d38f0e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,4 +17,5 @@ PySimpleGUI==4.60.1 requests~=2.27.1 Mako~=1.2.0 Flask~=2.2.2 +flask_httpauth~=4.7.0 APScheduler~=3.9.1 \ No newline at end of file From e669891cefa1fca9d72de1a5f8b8cefc7513d9be Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Thu, 5 Jan 2023 00:24:15 +0100 Subject: [PATCH 08/11] print logs to docker logs --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8f71484..d872c2e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,4 +18,4 @@ RUN pip3 install --upgrade pip RUN pip3 install -r requirements.txt COPY src src COPY server.py server.py -CMD [ "python3", "server.py", "--host=0.0.0.0"] \ No newline at end of file +CMD [ "python3","-u", "server.py", "--host=0.0.0.0"] \ No newline at end of file From 8ff06a65cbd28030744ace3f17c177820be5d980 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Thu, 5 Jan 2023 00:33:50 +0100 Subject: [PATCH 09/11] print logs to docker logs --- server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server.py b/server.py index 56357a8..235c55f 100644 --- a/server.py +++ b/server.py @@ -31,11 +31,11 @@ def post(): vid.write(request.data) speech_to_text = SpeechToText() result = speech_to_text.to_text(file_name) - print(result) + app.logger.info(result) try: os.remove(file_name) except OSError: - print(OSError) + app.logger.info(OSError) return jsonify(result) From 50a512f1e50fc0502efefd7f95e5c266be74ae0b Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Thu, 5 Jan 2023 00:41:47 +0100 Subject: [PATCH 10/11] readd the sphinx module --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d38f0e2..4766c47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ dataclasses~=0.6 SpeechRecognition==3.8.1 pymongo==4.1.1 wget==3.2.0 -#pocketsphinx==0.1.15 +pocketsphinx==0.1.15 oci~=2.54.1 XlsxWriter~=3.0.3 boto3~=1.21.13 From 11c935962bc2218970dd2a35dc218c650236512c Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Thu, 5 Jan 2023 13:29:51 +0100 Subject: [PATCH 11/11] add auth to speechToText container --- Dockerfile | 2 +- src/utils/send_email_mailjet.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index d872c2e..db15677 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,4 +18,4 @@ RUN pip3 install --upgrade pip RUN pip3 install -r requirements.txt COPY src src COPY server.py server.py -CMD [ "python3","-u", "server.py", "--host=0.0.0.0"] \ No newline at end of file +CMD [ "python3","-u", "server.py"] \ No newline at end of file diff --git a/src/utils/send_email_mailjet.py b/src/utils/send_email_mailjet.py index d92ca57..a30170c 100644 --- a/src/utils/send_email_mailjet.py +++ b/src/utils/send_email_mailjet.py @@ -9,8 +9,8 @@ api_key = "489274f04d5c155f81370fccc3904e20" api_secret = "edac41f0e1726ba49808dfb12204ecd6" mailjet = Client(auth=(api_key, api_secret), version='v3.1') from_email = "no-reply@lpaconsulting.fr" -store = "Hermès Paris Faubourg" -# store = "Hermès Paris George V" +# store = "Hermès Paris Faubourg" +store = "Hermès Paris George V" # store = "Hermès Paris Sèvres" # dest_email = "928490803@qq.com" # dest_email = "1757344572@qq.com" @@ -19,13 +19,13 @@ store = "Hermès Paris Faubourg" # dest_email = "fanfan19810617@icloud.com" # dest_email = "fanchunying323@163.com" # dest_email = "lytlaure@gmail.com" -# dest_email = "panleicim@gmail.com" +dest_email = "arianezhangsn@yahoo.com" # dest_email = "sunrose75015@gmail.com" -dest_email = "1340219934@qq.com" +# dest_email = "1340219934@qq.com" # dest_email = "lytlaure@gmail.com" # dest_email = "m13805869912@163.com" # dest_email = "tangliang0411@gmail.com" -contact_name = "DU Peijian" +contact_name = "LU yaping" f = open(config.ROOT_DIR + "/templates/confirmed_rdv.html", "r") email_body = f.read()