support ip_country field

link_validator_executor.py

@@ -8,7 +8,7 @@ from db.mongo_manager import MONGO_STORE_MANAGER
 from excel_reader import read_contacts
 from models.contact_pojo import ContactPojo
 from queue_message.link_validator_receiver import LinkValidatorReceiver
-from queue_message.receiver import Receiver, filter_contacts
+from queue_message.appointmentrequestsendor import AppointmentRequestSendor, filter_contacts
 from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
 from workers.sender import Sender
 

models/LinkPojo.py

@@ -1,13 +1,17 @@
 class LinkPojo():
-    def __init__(self, url, email, updated_at):
+    def __init__(self, url, email, updated_at, ip_country):
         self.url = url
         self.email = email
         self.updated_at = updated_at
+        self.ip_country = ip_country
 
     @staticmethod
     def from_firestore_dict(source):
         updated_at = source['updated_at']
         email = source['email']
         url = source['url']
-        result = LinkPojo(email=email, url=url, updated_at=updated_at)
+        ip_country = "FR"
+        if source.get('ip_country'):
+            ip_country = source['ip_country']
+        result = LinkPojo(email=email, url=url, updated_at=updated_at, ip_country=ip_country)
         return result

queue_message/CookiesPublisher.py

@@ -2,7 +2,9 @@ import pika
 
 QUEUE_HOST = "appointment.lpaconsulting.fr"
 REQUEST_DATA_QUEUE = 'REQUEST_DATA'
+# REQUEST_DATA_QUEUE_TEST = 'REQUEST_DATA_TEST'
 REQUEST_DATA_QUEUE_TEST = 'REQUEST_DATA_TEST'
+REQUEST_DATA_QUEUE_DE = 'REQUEST_DATA_DE'
 credentials = pika.PlainCredentials('appointment', 'ZyuhJZ2xEYWhElhpJjy7YEpZGZwNYJz2fHIu')
 
 
@@ -21,6 +23,7 @@ class CookiesPublisher:
         self.queue_method = self.channel.queue_declare(queue=self.to_queue, durable=True)
 
     def publish_body(self, body: str):
+        print("will push to queue {}".format(self.to_queue))
         self.channel.basic_publish(exchange='', routing_key=self.to_queue, body=body)
 
     def message_count(self):

queue_message/appointmentrequestsendor.py

@@ -11,6 +11,7 @@ from models.contact_pojo import ContactPojo
 from queue_message.CookiesPublisher import CookiesPublisher
 from utiles import is_time_between
 from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
+from workers.proxies_constants import PROXY_LIST_DE
 from workers.sender import Sender
 
 QUEUE_HOST = "appointment.lpaconsulting.fr"
@@ -72,14 +73,15 @@ def is_open():
     return is_time_between(datetime.time(10, 30), datetime.time(19, 00))
 
 
-class Receiver(threading.Thread):
+class AppointmentRequestSendor(threading.Thread):
-    def __init__(self, sub_contact_list: list, cookiesPublisher: CookiesPublisher):
+    def __init__(self, sub_contact_list: list, cookiesPublisher: CookiesPublisher, queue_name=REQUEST_DATA_QUEUE):
         super().__init__()
         self.connection = None
         self.cookiesPublisher = cookiesPublisher
         self.channel = None
         self.valid_csrf = None
         self.contact_list = sub_contact_list
+        self.queue_name = queue_name
 
     def set_up_connection(self):
         self.connection = pika.BlockingConnection(
@@ -87,14 +89,16 @@ class Receiver(threading.Thread):
         self.channel = self.connection.channel()
 
     def listen_to_queue(self, callback):
+        print("listen to queue {}".format(self.queue_name))
         self.channel.basic_qos(prefetch_count=1)
-        self.channel.basic_consume(queue=REQUEST_DATA_QUEUE, auto_ack=False, on_message_callback=callback)
+        self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback)
         self.channel.start_consuming()
 
     def on_message(self, ch, method, properties, body):
         print(f" [x] Received {body}")
         print("message count in queue is {}".format(self.cookiesPublisher.message_count()))
-        sender = Sender(body.decode("UTF-8"), cookiesPublisher=self.cookiesPublisher)
+        sender = Sender(body.decode("UTF-8"), cookiesPublisher=self.cookiesPublisher,
+                        proxy_to_use=random.choice(PROXY_LIST_DE))
         self.contact_list = filter_contacts(self.contact_list)
         # remove already booked contacts
         random.shuffle(self.contact_list)

request_sender.py

@@ -7,7 +7,7 @@ from db.mongo_manager import MONGO_STORE_MANAGER
 from excel_reader import read_contacts
 from models.contact_pojo import ContactPojo
 from queue_message.CookiesPublisher import CookiesPublisher
-from queue_message.receiver import Receiver
+from queue_message.appointmentrequestsendor import AppointmentRequestSendor
 from utiles import is_time_between
 from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
 from workers.sender import Sender
@@ -71,8 +71,10 @@ def is_open():
 
 
 if __name__ == '__main__':
-    cookiesPublisher = CookiesPublisher()
+    REQUEST_DATA_QUEUE_DE = 'REQUEST_DATA_DE'
+    cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_DE)
     cookiesPublisher.set_up_connection()
-    contact_list = read_contacts('/Users/panlei/Desktop/real_name_contacts_77_14_01_2024.xlsx')
+    contact_list = read_contacts('/Users/lpan/Desktop/08_01_24_valid_de.xlsx')
-    receiver = Receiver(sub_contact_list=contact_list, cookiesPublisher=cookiesPublisher)
+    receiver = AppointmentRequestSendor(sub_contact_list=contact_list, queue_name=REQUEST_DATA_QUEUE_DE,
+                                        cookiesPublisher=cookiesPublisher)
     receiver.run()

workers/captcha_result_getter.py

@@ -5,7 +5,7 @@ from time import time
 from typing import Union
 import requests
 
-from workers.proxies_constants import PROXY_LIST
+from workers.proxies_constants import PROXY_LIST_FR
 
 API_KEY = "d66aaf490d8aa424a5175e1fbd1aadea"
 
@@ -21,7 +21,7 @@ class CaptchaResultGetter:
         self.cookie_str = 'datadome=5Nq~NEP_qQSHC0g_lZNnZmEv36J8gVV~rpZ329xmCkTq2~H3meIoXr4h_b988qB2XW5Te7iEGsvq8BzA5KeFupyrZFh4kgrDyl8hT2UymSByKHzAcDaNIBPDsRu2g_KG; Max-Age=31536000; Domain=.hermes.com; Path=/; Secure; SameSite=None'
         pass
 
-    def get_csrf(self, cookie: str = None) -> Union[str, None]:
+    def get_csrf(self, proxy_to_use, cookie: str = None) -> Union[str, None]:
         if cookie is not None:
             headers = {'Content-Type': 'application/x-www-form-urlencoded',
                        'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
@@ -39,8 +39,8 @@ class CaptchaResultGetter:
                        'Sec-Fetch-Mode': 'navigate',
                        'Sec-Fetch-Dest': 'document',
                        'Accept-Language': 'fr-FR,fr;q=0.6'}
-        proxy_to_use = random.choice(PROXY_LIST)
         print("received cookie is " + str(cookie))
+        print(proxy_to_use)
         try:
             response = requests.get(url=HERMES_REGISTER, headers=headers, verify=False, proxies=proxy_to_use,
                                     timeout=15)
@@ -62,7 +62,7 @@ class CaptchaResultGetter:
             return result_list[-1]
         return None
 
-    def get_valid_cookie(self, old_valid_cookie: str, dvm=4, hc=6) -> Union[str, None]:
+    def get_valid_cookie(self, proxy_to_use, old_valid_cookie: str, dvm=4, hc=6) -> Union[str, None]:
         headers = {'content-Type': 'application/x-www-form-urlencoded',
                    'user-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
                    'accept': '*/*',
@@ -75,8 +75,8 @@ class CaptchaResultGetter:
                    'sec-ch-ua-platform': 'Android',
                    'accept-encoding': 'gzip, deflate, br',
                    'accept-language': 'fr-FR,fr;q=0.6'}
-        proxy_to_use = random.choice(PROXY_LIST)
         print("send request to get new cookie")
+        print(proxy_to_use)
         print(headers)
         try:
             # tag_pu = 10 * Math.random()
@@ -97,7 +97,7 @@ class CaptchaResultGetter:
             print(error)
             return None
 
-    def get_valid_ch_cookie(self, old_valid_cookie: str = None, dvm=4, hc=6) -> Union[str, None]:
+    def get_valid_ch_cookie(self, proxy_to_use, old_valid_cookie: str = None, dvm=4, hc=6) -> Union[str, None]:
         headers = {'content-Type': 'application/x-www-form-urlencoded',
                    'user-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
                    'accept': '*/*',
@@ -109,7 +109,7 @@ class CaptchaResultGetter:
                    'sec-fetch-dest': 'empty',
                    'accept-encoding': 'gzip, deflate, br',
                    'accept-language': 'fr-FR,fr;q=0.6'}
-        proxy_to_use = random.choice(PROXY_LIST)
+        print(proxy_to_use)
         print("send request to get new cookie")
         print(headers)
         try:

workers/cookie_generator.py

@@ -13,12 +13,13 @@ DVM_LIST = [2, 3, 4, 6]
 
 
 class CookiesGenerator(threading.Thread):
-    def __init__(self, cookiesPublisher: CookiesPublisher):
+    def __init__(self, proxy_to_use_list: list, cookiesPublisher: CookiesPublisher):
         super().__init__()
         self.connection = None
         self.cookiesPublisher = cookiesPublisher
         self.channel = None
         self.valid_csrf = None
+        self.proxy_to_use_list = proxy_to_use_list
 
     def set_up_connection(self):
         self.connection = pika.BlockingConnection(
@@ -40,22 +41,25 @@ class CookiesGenerator(threading.Thread):
 
         #     generate 10 cookies to new_queue
         captchaResultGetter = CaptchaResultGetter()
-        _cookies_count = random.randint(2, 10)
+        _cookies_count = random.randint(8, 10)
         _received_cookies = body.decode("UTF-8")
         dvm = random.choice(DVM_LIST)
         hc = random.choice(DVM_LIST)
         if _message_count < 10:
+            _proxy_to_use = random.choice(self.proxy_to_use_list)
             for i in range(1, _cookies_count):
-                new_cookie = captchaResultGetter.get_valid_ch_cookie(dvm=dvm, hc=hc)
+                # new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, dvm=dvm, hc=hc)
                 _app_sig = get_app_and_app_sig(_received_cookies)
-                if new_cookie is not None and _app_sig is not None:
+                # if new_cookie is not None and _app_sig is not None:
-                    new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
+                #     new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
-                    new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
+                #     new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
-                        "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "")
+                #         "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "")
-                    print("new_cookie is " + new_cookie)
+                #     print("new_cookie is " + new_cookie)
-                    _received_cookies = new_cookie
+                #     _received_cookies = new_cookie
-                    self.cookiesPublisher.publish_body(new_cookie)
+                #     self.cookiesPublisher.publish_body(new_cookie)
-                new_cookie = captchaResultGetter.get_valid_cookie(new_cookie, dvm=dvm, hc=hc)
+                new_cookie = captchaResultGetter.get_valid_cookie(proxy_to_use=_proxy_to_use,
+                                                                  old_valid_cookie=_received_cookies, dvm=dvm,
+                                                                  hc=hc)
                 if new_cookie is not None and _app_sig is not None:
                     new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
                     new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
@@ -86,9 +90,6 @@ def get_app_and_app_sig(cookies: str):
 
 
 if __name__ == '__main__':
-    # _list = get_app_and_app_sig(
-    #     "app.sig=axk5aJ2c83dgV56DswNhw98y4SY;datadome=2dTtQDIHCadAbzh8JWv7MFKGH8~fkOoGlA1O3J7llv_sdYOSZYbaKruHPBbARchV1nnrstsaMX5E_XwbJmGiXzt5qZXOL0UCAV9TTHqBRh10JvO~GCGJv2JVO~6RnrcU;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiZEE2bEdnN04yd2t0eWVmZmVjVkxMY1dTIiwiYXBwb2ludG1lbnRfY29kZSI6Ik5DTlVaQiIsImJsb2NrX3JlZ2lzdHJhdGlvbiI6ZmFsc2V9;policy=accepted;lang=fr;")
-    # print(_list)
     cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_TEST)
     cookiesPublisher.set_up_connection()
     cookieGenerator = CookiesGenerator(cookiesPublisher)

workers/de_cookie_generator.py

@@ -0,0 +1,9 @@
+from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher
+from workers.cookie_generator import CookiesGenerator
+from workers.proxies_constants import PROXY_LIST_DE
+
+if __name__ == '__main__':
+    cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_DE)
+    cookiesPublisher.set_up_connection()
+    cookieGenerator = CookiesGenerator(cookiesPublisher=cookiesPublisher, proxy_to_use_list=PROXY_LIST_DE)
+    cookieGenerator.run()

workers/link_validator.py

@@ -9,19 +9,22 @@ import requests
 from db.mongo_manager import MONGO_STORE_MANAGER
 from models.LinkPojo import LinkPojo
 from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST
-from queue_message.receiver import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
+from queue_message.appointmentrequestsendor import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
-from workers.proxies_constants import PROXY_LIST
+from workers.proxies_constants import PROXY_LIST_FR
 
 
 class LinkValidator(threading.Thread):
 
     def __init__(self, link_to_validate_list: list,
-                 cookiesPublisher: CookiesPublisher, queue_to_listen=REQUEST_DATA_QUEUE):
+                 cookiesPublisher: CookiesPublisher, proxy_to_use, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR"):
         super().__init__()
         self.cookie = SimpleCookie()
         self.cookiesPublisher = cookiesPublisher
         self.link_to_validate_list = link_to_validate_list
         self.queue_to_listen = queue_to_listen
+        self.ip_country = ip_country
+        self.filter_with_ip_country()
+        self.proxy_to_use = proxy_to_use
         # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
 
     def set_up_connection(self):
@@ -48,15 +51,16 @@ class LinkValidator(threading.Thread):
             'Sec-Fetch-Site': 'same-origin',
             'Sec-Fetch-Dest': 'document',
             'Accept-Language': 'fr-FR,fr;q=0.6'}
-        proxy_to_use = random.choice(PROXY_LIST)
+        print(self.proxy_to_use)
-        print(proxy_to_use)
         print("received cookie is " + str(self.cookie_str))
         try:
-            response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=proxy_to_use,
+            response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
                                     timeout=15)
             print(response.status_code)
             if response.status_code == 200:
+                _content = response.text
                 print(response.text)
+                if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content:
                     print(response.url)
                     MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo)
                     # set new cookies
@@ -73,6 +77,8 @@ class LinkValidator(threading.Thread):
                     return True
                 else:
                     return False
+            else:
+                return False
         except Exception as error:
             print(error)
             return False
@@ -80,6 +86,7 @@ class LinkValidator(threading.Thread):
     def on_message(self, ch, method, properties, body):
         print(f" [x] Received {body}")
         self.link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
+        self.filter_with_ip_country()
         self.cookie_str = body.decode("UTF-8")
         random.shuffle(self.link_to_validate_list)
         if len(self.link_to_validate_list) > 0:
@@ -101,12 +108,34 @@ class LinkValidator(threading.Thread):
             time.sleep(60)
             ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
 
+    def filter_with_ip_country(self):
+        _link_list_to_click = []
+        for _link in self.link_to_validate_list:
+            if _link.ip_country == self.ip_country:
+                _link_list_to_click.append(_link)
+        self.link_to_validate_list = _link_list_to_click
 
-if __name__ == '__main__':
+
+def validate_with_FR_ip():
+    _queue_name = REQUEST_DATA_QUEUE_TEST
     link_list = MONGO_STORE_MANAGER.get_links_to_validate()
-    cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_TEST)
+    cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
     cookiesPublisher.set_up_connection()
-    receiver = LinkValidator(link_to_validate_list=link_list, cookiesPublisher=cookiesPublisher,
+    print("filter links with ip_country")
-                             queue_to_listen=REQUEST_DATA_QUEUE_TEST)
+    _link_list_to_click = []
+    for _link in link_list:
+        if _link.ip_country == "FR":
+            _link_list_to_click.append(_link)
+    for _l in _link_list_to_click:
+        print(_l.ip_country)
+    _fr_proxy_to_use = random.choice(PROXY_LIST_FR)
+    receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
+                             proxy_to_use=_fr_proxy_to_use,
+                             queue_to_listen=_queue_name, ip_country="FR")
     receiver.set_up_connection()
     receiver.listen_to_queue(receiver.on_message)
+    pass
+
+
+if __name__ == '__main__':
+    validate_with_FR_ip()

workers/proxies_constants.py

@@ -31,4 +31,5 @@ DE_PROXY_RES = {
 # FR_PROXY_MOBILE
 # FR_PROXY_RES
 # PROXY_LIST = [FR_PROXY_MOBILE, FR_PROXY_RES, DE_PROXY_RES, DE_PROXY_MOBILE, ES_PROXY_MOBILE, IT_PROXY_MOBILE]
-PROXY_LIST = [FR_PROXY_MOBILE, FR_PROXY_RES]
+PROXY_LIST_FR = [FR_PROXY_MOBILE, FR_PROXY_RES]
+PROXY_LIST_DE = [DE_PROXY_RES, DE_PROXY_MOBILE]

workers/sender.py

@@ -8,18 +8,19 @@ import requests
 from db.mongo_manager import MONGO_STORE_MANAGER
 from models.ReserveResultPojo import ReserveResultPojo, PublishType
 from queue_message.CookiesPublisher import CookiesPublisher
-from workers.proxies_constants import PROXY_LIST
+from workers.proxies_constants import PROXY_LIST_FR
 
 
 class Sender:
 
-    def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher):
+    def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher, proxy_to_use):
         self.store_type = "random"
         self.cookie = SimpleCookie()
         self.cookiesPublisher = cookiesPublisher
         # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
         self.cookie_str = cookie_str
         self._csrf = None
+        self.proxy_to_use = proxy_to_use
         self.cookie.load(self.cookie_str)
 
     def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str):
@@ -64,7 +65,7 @@ class Sender:
                 'passport_id': contact.passport, 'processing': 'on', 'cgu': 'on'}
         print(data)
         try:
-            proxy_to_use = random.choice(PROXY_LIST)
+            proxy_to_use = self.proxy_to_use
             # proxy_to_use = PROXY_LIST[0]
             print(proxy_to_use)
             response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data,

workers/validate_links_DE.py

@@ -0,0 +1,33 @@
+import random
+
+from db.mongo_manager import MONGO_STORE_MANAGER
+from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher
+from workers.link_validator import LinkValidator
+from workers.proxies_constants import PROXY_LIST_DE
+
+
+def validate_with_DE_ip():
+    _queue_name = REQUEST_DATA_QUEUE_DE
+    link_list = MONGO_STORE_MANAGER.get_links_to_validate()
+    cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
+    cookiesPublisher.set_up_connection()
+    print("filter links with ip_country")
+    _link_list_to_click = []
+    for _link in link_list:
+        if _link.ip_country == "DE":
+            _link_list_to_click.append(_link)
+    for _l in _link_list_to_click:
+        print(_l.ip_country)
+        # else:
+        #     print(_link.ip_country)
+    _de_proxy_to_use = random.choice(PROXY_LIST_DE)
+    receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
+                             proxy_to_use=_de_proxy_to_use,
+                             queue_to_listen=_queue_name, ip_country="DE")
+    receiver.set_up_connection()
+    receiver.listen_to_queue(receiver.on_message)
+    pass
+
+
+if __name__ == '__main__':
+    validate_with_DE_ip()