From 2a339be86e5cdf54472c5d6ae159b9a368d7a214 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Wed, 6 Jul 2022 18:08:00 +0200 Subject: [PATCH 1/2] check black list before send request --- main.py | 11 +++++++---- src/db/mongo_manager.py | 26 +++++++++++++++++++++++--- src/utils/black_list_checker.py | 16 ++++++++++++++++ src/workers/commandor_page.py | 1 + 4 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 src/utils/black_list_checker.py diff --git a/main.py b/main.py index 6919e42..03e204b 100644 --- a/main.py +++ b/main.py @@ -7,6 +7,7 @@ from src.logs.AppLogging import init_logger from src.pojo.ModeEnum import ModeEnum from src.pojo.contact_pojo import ContactPojo from src.proxy.proxy_type import ProxyType +from src.utils.black_list_checker import can_send_request from src.utils.excel_reader import ExcelHelper from src.workers.commandor_page import CommandorPage @@ -24,14 +25,16 @@ def start_book(start_number, end_number, store_choose_state=0, max_workers=10, p if len(all_contacts) <= end_number: end_number = len(all_contacts) contacts = all_contacts[start_number - 1: end_number] - logger.info(contacts) with ThreadPoolExecutor(max_workers=max_workers) as executor: for contact in contacts: proxy = get_proxy(proxy_type) # start the task in thread - executor.submit( - CommandorPage(contact, store_type=store_choose_state, proxy_type=proxy_type, mode=mode).start_page, - proxy) + if can_send_request(contact): + executor.submit( + CommandorPage(contact, store_type=store_choose_state, proxy_type=proxy_type, mode=mode).start_page, + proxy) + else: + logger.info("do not send request --> skip") def recheck_the_captcha_error_contacts(on_no_contact_found, store_type=0, mode: ModeEnum = ModeEnum.MANUAL, diff --git a/src/db/mongo_manager.py b/src/db/mongo_manager.py index 2724238..b651f7c 100644 --- a/src/db/mongo_manager.py +++ b/src/db/mongo_manager.py @@ -52,6 +52,17 @@ class MongoDbManager: except Exception as error: self.logger.info(error) + def get_blacklist_contacts(self) -> list: + collection_name = BLACK_LIST + black_list_contacts = [] + try: + collection_to_use = self.db[collection_name] + for document in collection_to_use.find(): + black_list_contacts.append(BlackContactPojo.from_firestore_dict(document)) + except Exception as error: + self.logger.info(error) + return black_list_contacts + def get_captcha_error_contacts_for_current_day(self) -> list: day = str(datetime.date.today()) collection_name = CAPTCHA_ERROR_COLLECTION_PREFIX + day @@ -71,9 +82,18 @@ class MongoDbManager: except Exception as error: self.logger.info(error) + def remove_contact_from_black_list(self, contact: ContactPojo): + collection_name = BLACK_LIST + collection = self.db[collection_name] + to_delete = {'_id': contact.mail} + try: + collection.delete_one(to_delete) + except Exception as error: + self.logger.info(error) + if __name__ == '__main__': db_manager = MongoDbManager() - contact = ContactPojo(phone_number='0755667750', passport_number='123456789', last_name='PAN', first_name='Lei', - mail='panleicim@gmail.com') - print(db_manager.insert_blacklist_contact(contact)) + black_list = db_manager.get_blacklist_contacts() + for contact in black_list: + print(contact) \ No newline at end of file diff --git a/src/utils/black_list_checker.py b/src/utils/black_list_checker.py new file mode 100644 index 0000000..6914028 --- /dev/null +++ b/src/utils/black_list_checker.py @@ -0,0 +1,16 @@ +import time + +from src import params +from src.pojo.contact_pojo import ContactPojo + +SEVEN_DAYS_IN_S = 7 * 24 * 3600 + + +def can_send_request(contact: ContactPojo) -> bool: + black_list = params.mongo_store_manager.get_blacklist_contacts() + for black_contact in black_list: + if contact.mail == black_contact.mail: + # check date + return black_contact.update_at_in_s + SEVEN_DAYS_IN_S < time.time() + + return True diff --git a/src/workers/commandor_page.py b/src/workers/commandor_page.py index 666c2e5..51af8be 100644 --- a/src/workers/commandor_page.py +++ b/src/workers/commandor_page.py @@ -332,6 +332,7 @@ class CommandorPage: params.firebase_store_manager.save(result) params.local_db_manager.handle_success(result) params.mongo_store_manager.delete_captcha_error_contact_for_current_day(self.contact) + params.mongo_store_manager.remove_contact_from_black_list(self.contact) if status is PublishType.SUCCESS: self.on_success(result) time.sleep(2) From 50cdafd8a277e58a9cab2569378513f048ae10d0 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Wed, 6 Jul 2022 18:10:47 +0200 Subject: [PATCH 2/2] remove some logs --- src/workers/SolveCaptch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/workers/SolveCaptch.py b/src/workers/SolveCaptch.py index d8f9023..e32f4fe 100644 --- a/src/workers/SolveCaptch.py +++ b/src/workers/SolveCaptch.py @@ -28,7 +28,7 @@ class SolveCaptcha: if len(data_sitekey) == 1: key_with_comma = data_sitekey[0].split("=")[-1] key = key_with_comma.replace("\"", '') - self.logger.info("key is : " + key) + # self.logger.info("key is : " + key) self.solve_captcha(key, handle_solution_received) def solve_captcha(self, google_key: str, handle_solution_received): @@ -50,5 +50,5 @@ class SolveCaptcha: solution_res = requests.get(url_response) time.sleep(5) solution = solution_res.text - print(solution) + self.logger.info(solution) handle_solution_received(solution.split("|")[-1])