add result_pojo.py

link_validator_executor.py

@@ -8,7 +8,7 @@ from db.mongo_manager import MONGO_STORE_MANAGER
 from excel_reader import read_contacts
 from models.contact_pojo import ContactPojo
 from queue_message.link_validator_receiver import LinkValidatorReceiver
-from queue_message.appointmentrequestsendor import AppointmentRequestSendor, filter_contacts
+from queue_message.appointmentrequestsender import AppointmentRequestSender, filter_contacts
 from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
 from workers.sender import Sender
 

models/result_pojo.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class RequestResult(Enum):
+    BLOCKED = "BLOCKED"
+    PROXY_ERROR = "PROXY_ERROR"
+    SUCCESS = "SUCCESS"
+    UNKNOWN = "UNKNOWN"

queue_message/appointmentrequestsender.py

@@ -8,10 +8,10 @@ import pika
 
 from db.mongo_manager import MONGO_STORE_MANAGER
 from models.contact_pojo import ContactPojo
+from models.result_pojo import RequestResult
 from queue_message.CookiesPublisher import CookiesPublisher
 from utiles import is_time_between
 from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
-from workers.proxies_constants import PROXY_LIST_DE
 from workers.sender import Sender
 
 QUEUE_HOST = "appointment.lpaconsulting.fr"
@@ -73,7 +73,7 @@ def is_open():
     return is_time_between(datetime.time(10, 30), datetime.time(19, 00))
 
 
-class AppointmentRequestSendor(threading.Thread):
+class AppointmentRequestSender(threading.Thread):
     def __init__(self, sub_contact_list: list, proxy_to_use_list, cookiesPublisher: CookiesPublisher,
                  queue_name=REQUEST_DATA_QUEUE):
         super().__init__()
@@ -117,9 +117,11 @@ class AppointmentRequestSendor(threading.Thread):
                     self.valid_csrf = captchaResultGetter.get_csrf(proxy_to_use=random.choice(self.proxy_to_use_list),
                                                                    cookie=body.decode("UTF-8"))
                 can_continue = sender.send_request(HERMES_REGISTER, con, csrf=self.valid_csrf)
-                if not can_continue:
+                if can_continue == RequestResult.BLOCKED:
                     print("cannot continue, valid_csrf is " + str(self.valid_csrf))
                     break
+                elif can_continue == RequestResult.PROXY_ERROR:
+                    print("PROXY_ERROR, will not reset valid_csrf")
                 else:
                     print("can continue, will reset valid_csrf")
                     self.valid_csrf = None

request_sender.py

@@ -7,7 +7,7 @@ from db.mongo_manager import MONGO_STORE_MANAGER
 from excel_reader import read_contacts
 from models.contact_pojo import ContactPojo
 from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_DE
-from queue_message.appointmentrequestsendor import AppointmentRequestSendor, REQUEST_DATA_QUEUE
+from queue_message.appointmentrequestsender import AppointmentRequestSender, REQUEST_DATA_QUEUE
 from utiles import is_time_between
 from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
 from workers.proxies_constants import PROXY_LIST_DE, PROXY_LIST_FR
@@ -75,11 +75,11 @@ def send_appointment_request(message_queue_name, contacts_file_path):
     _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name)
     _cookiesPublisher.set_up_connection()
     contact_list = read_contacts(contacts_file_path)
-    receiver = AppointmentRequestSendor(sub_contact_list=contact_list, proxy_to_use_list=PROXY_LIST_FR,
+    receiver = AppointmentRequestSender(sub_contact_list=contact_list, proxy_to_use_list=PROXY_LIST_FR,
                                         queue_name=message_queue_name,
                                         cookiesPublisher=_cookiesPublisher)
     receiver.run()
 
 
 if __name__ == '__main__':
-    send_appointment_request(REQUEST_DATA_QUEUE, '/Users/rdv/Desktop/yahoo_aol_valid_26-1.xlsx')
+    send_appointment_request(REQUEST_DATA_QUEUE, '~/Desktop/yahoo_aol_valid_26-1.xlsx')

workers/link_validator.py

@@ -8,8 +8,9 @@ import requests
 
 from db.mongo_manager import MONGO_STORE_MANAGER
 from models.LinkPojo import LinkPojo
+from models.result_pojo import RequestResult
 from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST
-from queue_message.appointmentrequestsendor import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
+from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
 from workers.proxies_constants import PROXY_LIST_FR
 
 DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ces coordonnées"
@@ -41,7 +42,7 @@ class LinkValidator(threading.Thread):
         self.channel.basic_consume(queue=self.queue_to_listen, auto_ack=False, on_message_callback=callback)
         self.channel.start_consuming()
 
-    def send_request(self, linkPojo: LinkPojo) -> bool:
+    def send_request(self, linkPojo: LinkPojo) -> RequestResult:
         self.cookie.load(self.cookie_str)
         headers = {
             'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
@@ -78,7 +79,7 @@ class LinkValidator(threading.Thread):
                     # upload the cookie to queue
                     self.cookiesPublisher.publish_body(new_coolies_str)
                     self.cookie_str = new_coolies_str
-                    return True
+                    return RequestResult.SUCCESS
                 elif DOUBLE_MESSAGE in _content:
                     print(response.url)
                     MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True)
@@ -93,14 +94,14 @@ class LinkValidator(threading.Thread):
                     # upload the cookie to queue
                     self.cookiesPublisher.publish_body(new_coolies_str)
                     self.cookie_str = new_coolies_str
-                    return True
+                    return RequestResult.SUCCESS
                 else:
-                    return False
+                    return RequestResult.UNKNOWN
             else:
-                return False
+                return RequestResult.BLOCKED
         except Exception as error:
             print(error)
-            return False
+            return RequestResult.PROXY_ERROR
 
     def on_message(self, ch, method, properties, body):
         print(f" [x] Received {body}")
@@ -116,8 +117,8 @@ class LinkValidator(threading.Thread):
                 # if not is_already_sent(con):
                 print(con.email)
                 can_continue = self.send_request(con)
-                if not can_continue:
+                if can_continue == RequestResult.BLOCKED:
-                    print("cannot continue, skip")
+                    print("cannot continue, blocked, then skip")
                     break
                 else:
                     time.sleep(random.randint(2, 5))

workers/sender.py

@@ -7,6 +7,7 @@ import requests
 
 from db.mongo_manager import MONGO_STORE_MANAGER
 from models.ReserveResultPojo import ReserveResultPojo, PublishType
+from models.result_pojo import RequestResult
 from queue_message.CookiesPublisher import CookiesPublisher
 from workers.proxies_constants import PROXY_LIST_FR
 
@@ -43,7 +44,7 @@ class Sender:
     def get_csrf(self):
         pass
 
-    def send_request(self, url, contact: ContactPojo, csrf: str = None):
+    def send_request(self, url, contact: ContactPojo, csrf: str = None) -> RequestResult:
         if csrf is None:
             if self._csrf is None:
                 _csrf = '8Bs2dBwb-nHONOzo9Tei2CcMZglEfsRqUz8E'
@@ -86,9 +87,9 @@ class Sender:
                 # upload the cookie to queue
                 self.cookiesPublisher.publish_body(new_coolies_str)
                 self.cookie_str = new_coolies_str
-                return True
+                return RequestResult.SUCCESS
             else:
-                return False
+                return RequestResult.BLOCKED
         except Exception as Error:
             print(Error)
-            return False
+            return RequestResult.PROXY_ERROR