From 83871f8e4bec8f259682f9d5d084cc8214e60bd1 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Thu, 8 Feb 2024 00:25:04 +0100 Subject: [PATCH] add result_pojo.py --- link_validator_executor.py | 2 +- models/result_pojo.py | 8 ++++++++ ...tsendor.py => appointmentrequestsender.py} | 8 +++++--- request_sender.py | 6 +++--- workers/link_validator.py | 19 ++++++++++--------- workers/sender.py | 9 +++++---- 6 files changed, 32 insertions(+), 20 deletions(-) create mode 100644 models/result_pojo.py rename queue_message/{appointmentrequestsendor.py => appointmentrequestsender.py} (95%) diff --git a/link_validator_executor.py b/link_validator_executor.py index c98b75e..c63953c 100644 --- a/link_validator_executor.py +++ b/link_validator_executor.py @@ -8,7 +8,7 @@ from db.mongo_manager import MONGO_STORE_MANAGER from excel_reader import read_contacts from models.contact_pojo import ContactPojo from queue_message.link_validator_receiver import LinkValidatorReceiver -from queue_message.appointmentrequestsendor import AppointmentRequestSendor, filter_contacts +from queue_message.appointmentrequestsender import AppointmentRequestSender, filter_contacts from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER from workers.sender import Sender diff --git a/models/result_pojo.py b/models/result_pojo.py new file mode 100644 index 0000000..0499039 --- /dev/null +++ b/models/result_pojo.py @@ -0,0 +1,8 @@ +from enum import Enum + + +class RequestResult(Enum): + BLOCKED = "BLOCKED" + PROXY_ERROR = "PROXY_ERROR" + SUCCESS = "SUCCESS" + UNKNOWN = "UNKNOWN" diff --git a/queue_message/appointmentrequestsendor.py b/queue_message/appointmentrequestsender.py similarity index 95% rename from queue_message/appointmentrequestsendor.py rename to queue_message/appointmentrequestsender.py index ea4e905..e629139 100644 --- a/queue_message/appointmentrequestsendor.py +++ b/queue_message/appointmentrequestsender.py @@ -8,10 +8,10 @@ import pika from db.mongo_manager import MONGO_STORE_MANAGER from models.contact_pojo import ContactPojo +from models.result_pojo import RequestResult from queue_message.CookiesPublisher import CookiesPublisher from utiles import is_time_between from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER -from workers.proxies_constants import PROXY_LIST_DE from workers.sender import Sender QUEUE_HOST = "appointment.lpaconsulting.fr" @@ -73,7 +73,7 @@ def is_open(): return is_time_between(datetime.time(10, 30), datetime.time(19, 00)) -class AppointmentRequestSendor(threading.Thread): +class AppointmentRequestSender(threading.Thread): def __init__(self, sub_contact_list: list, proxy_to_use_list, cookiesPublisher: CookiesPublisher, queue_name=REQUEST_DATA_QUEUE): super().__init__() @@ -117,9 +117,11 @@ class AppointmentRequestSendor(threading.Thread): self.valid_csrf = captchaResultGetter.get_csrf(proxy_to_use=random.choice(self.proxy_to_use_list), cookie=body.decode("UTF-8")) can_continue = sender.send_request(HERMES_REGISTER, con, csrf=self.valid_csrf) - if not can_continue: + if can_continue == RequestResult.BLOCKED: print("cannot continue, valid_csrf is " + str(self.valid_csrf)) break + elif can_continue == RequestResult.PROXY_ERROR: + print("PROXY_ERROR, will not reset valid_csrf") else: print("can continue, will reset valid_csrf") self.valid_csrf = None diff --git a/request_sender.py b/request_sender.py index 9ff5463..bd92ab2 100644 --- a/request_sender.py +++ b/request_sender.py @@ -7,7 +7,7 @@ from db.mongo_manager import MONGO_STORE_MANAGER from excel_reader import read_contacts from models.contact_pojo import ContactPojo from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_DE -from queue_message.appointmentrequestsendor import AppointmentRequestSendor, REQUEST_DATA_QUEUE +from queue_message.appointmentrequestsender import AppointmentRequestSender, REQUEST_DATA_QUEUE from utiles import is_time_between from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER from workers.proxies_constants import PROXY_LIST_DE, PROXY_LIST_FR @@ -75,11 +75,11 @@ def send_appointment_request(message_queue_name, contacts_file_path): _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name) _cookiesPublisher.set_up_connection() contact_list = read_contacts(contacts_file_path) - receiver = AppointmentRequestSendor(sub_contact_list=contact_list, proxy_to_use_list=PROXY_LIST_FR, + receiver = AppointmentRequestSender(sub_contact_list=contact_list, proxy_to_use_list=PROXY_LIST_FR, queue_name=message_queue_name, cookiesPublisher=_cookiesPublisher) receiver.run() if __name__ == '__main__': - send_appointment_request(REQUEST_DATA_QUEUE, '/Users/rdv/Desktop/yahoo_aol_valid_26-1.xlsx') + send_appointment_request(REQUEST_DATA_QUEUE, '~/Desktop/yahoo_aol_valid_26-1.xlsx') diff --git a/workers/link_validator.py b/workers/link_validator.py index 1a9d947..1229692 100644 --- a/workers/link_validator.py +++ b/workers/link_validator.py @@ -8,8 +8,9 @@ import requests from db.mongo_manager import MONGO_STORE_MANAGER from models.LinkPojo import LinkPojo +from models.result_pojo import RequestResult from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST -from queue_message.appointmentrequestsendor import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials +from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials from workers.proxies_constants import PROXY_LIST_FR DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ces coordonnées" @@ -41,7 +42,7 @@ class LinkValidator(threading.Thread): self.channel.basic_consume(queue=self.queue_to_listen, auto_ack=False, on_message_callback=callback) self.channel.start_consuming() - def send_request(self, linkPojo: LinkPojo) -> bool: + def send_request(self, linkPojo: LinkPojo) -> RequestResult: self.cookie.load(self.cookie_str) headers = { 'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36', @@ -78,7 +79,7 @@ class LinkValidator(threading.Thread): # upload the cookie to queue self.cookiesPublisher.publish_body(new_coolies_str) self.cookie_str = new_coolies_str - return True + return RequestResult.SUCCESS elif DOUBLE_MESSAGE in _content: print(response.url) MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True) @@ -93,14 +94,14 @@ class LinkValidator(threading.Thread): # upload the cookie to queue self.cookiesPublisher.publish_body(new_coolies_str) self.cookie_str = new_coolies_str - return True + return RequestResult.SUCCESS else: - return False + return RequestResult.UNKNOWN else: - return False + return RequestResult.BLOCKED except Exception as error: print(error) - return False + return RequestResult.PROXY_ERROR def on_message(self, ch, method, properties, body): print(f" [x] Received {body}") @@ -116,8 +117,8 @@ class LinkValidator(threading.Thread): # if not is_already_sent(con): print(con.email) can_continue = self.send_request(con) - if not can_continue: - print("cannot continue, skip") + if can_continue == RequestResult.BLOCKED: + print("cannot continue, blocked, then skip") break else: time.sleep(random.randint(2, 5)) diff --git a/workers/sender.py b/workers/sender.py index 7e6b9d4..6a2f320 100644 --- a/workers/sender.py +++ b/workers/sender.py @@ -7,6 +7,7 @@ import requests from db.mongo_manager import MONGO_STORE_MANAGER from models.ReserveResultPojo import ReserveResultPojo, PublishType +from models.result_pojo import RequestResult from queue_message.CookiesPublisher import CookiesPublisher from workers.proxies_constants import PROXY_LIST_FR @@ -43,7 +44,7 @@ class Sender: def get_csrf(self): pass - def send_request(self, url, contact: ContactPojo, csrf: str = None): + def send_request(self, url, contact: ContactPojo, csrf: str = None) -> RequestResult: if csrf is None: if self._csrf is None: _csrf = '8Bs2dBwb-nHONOzo9Tei2CcMZglEfsRqUz8E' @@ -86,9 +87,9 @@ class Sender: # upload the cookie to queue self.cookiesPublisher.publish_body(new_coolies_str) self.cookie_str = new_coolies_str - return True + return RequestResult.SUCCESS else: - return False + return RequestResult.BLOCKED except Exception as Error: print(Error) - return False + return RequestResult.PROXY_ERROR