Merge branch 'feature/tls_ja3' of bitbucket.org:panleicim/appointment_request into feature/tls_ja3

migrate_to_data_cache_2.py

@@ -1,6 +1,8 @@
-
+from queue_message.CookiesPublisher import MORNING_DATA_CACHE_2, MORNING_DATA_CACHE, MORNING_DATA_CACHE_BAK, \
-from queue_message.CookiesPublisher import MORNING_DATA_CACHE_2, MORNING_DATA_CACHE, MORNING_DATA_CACHE_BAK
+    REQUEST_DATA_QUEUE_DE
-from workers.MessagerTransporter import migrate_message_to_queue
+from workers.MessagerTransporter import migrate_message_to_queue, empty_message_just_to
 
 if __name__ == '__main__':
-    migrate_message_to_queue(from_queue=MORNING_DATA_CACHE,to_queue=MORNING_DATA_CACHE_2)
+    migrate_message_to_queue(from_queue=MORNING_DATA_CACHE_BAK, to_queue=MORNING_DATA_CACHE)
+    # empty_message_just_to(2000, queue_name=MORNING_DATA_CACHE_BAK)
+    # migrate_message_to_queue(from_queue=MORNING_DATA_CACHE_BAK,to_queue=REQUEST_DATA_QUEUE_DE)

queue_message/CookiesPublisher.py

@@ -31,7 +31,7 @@ class CookiesPublisher:
         self.queue_method = self.channel.queue_declare(queue=self.to_queue, durable=True)
 
     def publish_body(self, body: str):
-        print("will push to queue {}".format(self.to_queue))
+        print("will push to queue {}".format(body))
         self.channel.basic_publish(exchange='', routing_key=self.to_queue, body=body, properties=pika.BasicProperties(
             delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE
         ))

workers/MessagerTransporter.py

@@ -39,6 +39,43 @@ class MessageTransporter(threading.Thread):
             exit(0)
 
 
+class MessageRemover(threading.Thread):
+
+    def __init__(self, limit: int,
+                 queue_to_listen):
+        self.queue_to_listen = queue_to_listen
+        self.limit = limit
+
+    def set_up_connection(self):
+        self.connection = pika.BlockingConnection(
+            pika.ConnectionParameters(host=QUEUE_HOST, port=5672, credentials=credentials))
+        self.channel = self.connection.channel()
+
+    def listen_to_queue(self):
+        self.channel.basic_qos(prefetch_count=1)
+        self.channel.basic_consume(queue=self.queue_to_listen, auto_ack=False, on_message_callback=self.on_message)
+        self.channel.start_consuming()
+
+    def message_count(self):
+        return self.channel.queue_declare(queue=self.queue_to_listen, durable=True).method.message_count
+
+    def on_message(self, ch, method, properties, body):
+        print(f" [x] Received {body}")
+        _received_object = body.decode("UTF-8")
+        # _message_in_queue_count = self.message_count
+        # print("message count in queue is {}".format(_message_in_queue_count))
+        ch.basic_ack(delivery_tag=method.delivery_tag)
+        if self.message_count() <= self.limit:
+            print("all messages are processed")
+            exit(0)
+
+
+def empty_message_just_to(left_message_number, queue_name):
+    _remover = MessageRemover(left_message_number, queue_name)
+    _remover.set_up_connection()
+    _remover.listen_to_queue()
+
+
 def migrate_message_to_queue(from_queue, to_queue="MORNING_DATA_CACHE_BAK"):
     cookiesPublisher = CookiesPublisher(queue_name=to_queue)
     cookiesPublisher.set_up_connection()

workers/captcha_result_getter.py

@@ -112,17 +112,16 @@ class CaptchaResultGetter:
         return _raw_data
 
     def get_valid_ch_cookie(self, proxy_to_use, js_data: JsDataPojo, old_valid_cookie: str = None,
-                            referrer: str = None) -> Union[str, None]:
+                            referrer: str = "https://rendezvousparis.hermes.com/",
+                            origin="https://rendezvousparis.hermes.com/") -> Union[str, None]:
         # extract datadome cookie
         # get new cookie
         _cookies_to_use = get_datadome_cookies(old_valid_cookie)
         _app_cookies = get_app_cookies(old_valid_cookie)
         _lang_cookies = get_lang_cookies(old_valid_cookie)
-        _origin = "https://rendezvousparis.hermes.com/"
+        _origin = origin
         # _origin = "https://www.hermes.com"
-        _referer = "https://rendezvousparis.hermes.com/"
+        _referer = referrer
-        if referrer is not None:
-            _referer = referrer
         # _referer = "https://www.hermes.com"
 
         headers = {'content-Type': 'application/x-www-form-urlencoded',

workers/cookie_generator.py

@@ -1,3 +1,5 @@
+import json
+import logging
 import random
 import threading
 import time
@@ -5,8 +7,11 @@ from http.cookies import SimpleCookie
 
 import pika
 
+from models.jsdata_le_pojo import JsDataLeTypePojo
+from models.jsdata_pojo import JsDataPojo
+from proxy_manager.proxy_manager import ProxyManager
 from queue_message.CookiesPublisher import CookiesPublisher, QUEUE_HOST, credentials, REQUEST_DATA_QUEUE, \
-    REQUEST_DATA_QUEUE_TEST
+    REQUEST_DATA_QUEUE_TEST, MORNING_DATA_CACHE_BAK
 from workers.captcha_result_getter import CaptchaResultGetter
 from workers.proxies_constants import PROXY_LIST_FR
 
@@ -14,13 +19,16 @@ DVM_LIST = [2, 3, 4, 6]
 
 
 class CookiesGenerator(threading.Thread):
-    def __init__(self, proxy_to_use_list: list, cookiesPublisher: CookiesPublisher):
+    def __init__(self, proxy_manager: ProxyManager, cookiesPublisher: CookiesPublisher,
+                 logger=logging.getLogger(__name__)):
         super().__init__()
         self.connection = None
         self.cookiesPublisher = cookiesPublisher
         self.channel = None
         self.valid_csrf = None
-        self.proxy_to_use_list = proxy_to_use_list
+        self.proxy_manager = proxy_manager
+        self.queue_name = MORNING_DATA_CACHE_BAK
+        self.logger = logger
 
     def set_up_connection(self):
         self.connection = pika.BlockingConnection(
@@ -29,50 +37,90 @@ class CookiesGenerator(threading.Thread):
 
     def listen_to_queue(self, callback):
         self.channel.basic_qos(prefetch_count=1)
-        self.channel.basic_consume(queue=REQUEST_DATA_QUEUE, auto_ack=False, on_message_callback=callback)
+        self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback)
         self.channel.start_consuming()
 
     def message_count(self):
-        return self.channel.queue_declare(queue=REQUEST_DATA_QUEUE, durable=True).method.message_count
+        return self.channel.queue_declare(queue=self.queue_name, durable=True).method.message_count
 
     def on_message(self, ch, method, properties, body):
         print(f" [x] Received {body}")
-        _cached_message_count = self.cookiesPublisher.message_count()
+        _received_object = body.decode("UTF-8")
-        _valid_message_count = self.message_count()
+        _received_dict = json.loads(_received_object)
-        print("message count in cache is {}".format(_cached_message_count))
+        _received_cookies = _received_dict["cookiesStr"]
-        print("message count in current queue is {}".format(_valid_message_count))
-
         #     generate 10 cookies to new_queue
         captchaResultGetter = CaptchaResultGetter()
         _cookies_count = random.randint(8, 10)
         _received_cookies = body.decode("UTF-8")
         dvm = random.choice(DVM_LIST)
         hc = random.choice(DVM_LIST)
-        if _cached_message_count < 10 and _valid_message_count > 40:
+        _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
-            _proxy_to_use = random.choice(self.proxy_to_use_list)
+        # new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, dvm=dvm, hc=hc)
-            for i in range(1, _cookies_count):
+        _app_sig = get_app_and_app_sig(_received_cookies)
-                # new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, dvm=dvm, hc=hc)
+        # if new_cookie is not None and _app_sig is not None:
-                _app_sig = get_app_and_app_sig(_received_cookies)
+        #     new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
-                # if new_cookie is not None and _app_sig is not None:
+        #     new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
-                #     new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
+        #         "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "")
-                #     new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
+        #     print("new_cookie is " + new_cookie)
-                #         "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "")
+        #     _received_cookies = new_cookie
-                #     print("new_cookie is " + new_cookie)
+        #     self.cookiesPublisher.publish_body(new_cookie)
-                #     _received_cookies = new_cookie
+        js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'],
-                #     self.cookiesPublisher.publish_body(new_cookie)
+                             ua=_received_dict['ua'], br_oh=_received_dict['br_oh'], br_ow=_received_dict['br_ow'],
-                new_cookie = captchaResultGetter.get_valid_cookie(proxy_to_use=_proxy_to_use,
+                             ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'], pr=_received_dict['pr'],
-                                                                  old_valid_cookie=_received_cookies, dvm=dvm,
+                             plg=_received_dict['plg'], br_h=_received_dict['br_h'], br_w=_received_dict['br_w'],
-                                                                  hc=hc)
+                             plu=_received_dict['plu'], vnd=_received_dict['vnd'], dvm=_received_dict['dvm'],
-                if new_cookie is not None and _app_sig is not None:
+                             ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'],
-                    new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
+                             rs_h=_received_dict['rs_h'],
-                    new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
+                             rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'], emd=_received_dict['emd'])
-                        "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "")
+
-                    _received_cookies = new_cookie
+        new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, js_data=js_data,
-                    self.cookiesPublisher.publish_body(new_cookie)
+                                                             old_valid_cookie=_received_cookies,
-            ch.basic_ack(delivery_tag=method.delivery_tag)
+                                                             origin="https://www.hermes.com",
-        else:
+                                                             referrer="https://www.hermes.com")
-            time.sleep(30)
+        if new_cookie is not None:
-            ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
+            new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
+            new_cookie = new_cookie.replace("Path=/;", "").replace(
+                "Secure; SameSite=None", "").replace(" ", "")
+            _received_cookies = new_cookie
+            # _received_dict["cookiesStr"] = new_cookie
+            # self.cookiesPublisher.publish_body(json.dumps(_received_dict))
+            if _received_cookies is not None:
+                self.logger.info("new cookie is " + _received_cookies)
+                time.sleep(random.randint(1, 3))
+                # m_s_c = f.scroll
+                m_s_c = random.randint(0, 3)
+                m_c_c = random.randint(3, 5)  # click count
+                m_m_c = random.randint(3, 5)  # move count
+                m_cm_r = m_c_c / m_m_c
+                m_ms_r = -1  # move scroll ratio
+                if m_s_c == 0:
+                    m_ms_r = -1
+                else:
+                    m_ms_r = m_m_c / m_s_c
+
+                js_le_data = JsDataLeTypePojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'],
+                                              hc=_received_dict['hc'],
+                                              ua=_received_dict['ua'], br_oh=_received_dict['br_oh'],
+                                              br_ow=_received_dict['br_ow'],
+                                              ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'],
+                                              pr=_received_dict['pr'],
+                                              plg=_received_dict['plg'], br_h=_received_dict['br_h'],
+                                              br_w=_received_dict['br_w'],
+                                              plu=_received_dict['plu'], vnd=_received_dict['vnd'],
+                                              dvm=_received_dict['dvm'],
+                                              ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'],
+                                              rs_h=_received_dict['rs_h'],
+                                              rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'],
+                                              m_s_c=m_s_c, m_m_c=m_m_c, m_c_c=m_c_c,
+                                              m_cm_r=m_cm_r, m_ms_r=m_ms_r, emd=_received_dict['emd'])
+                time.sleep(random.randint(1, 4))
+                _new_le_cookies = captchaResultGetter.get_le_valid_cookie(proxy_to_use=_proxy_to_use,
+                                                                          js_le_type_data=js_le_data,
+                                                                          old_valid_cookie=_received_cookies)
+                _received_dict["cookiesStr"] = new_cookie
+                self.cookiesPublisher.publish_body(json.dumps(_received_dict))
+            # self.cookiesPublisher.publish_body(new_cookie)
+        ch.basic_ack(delivery_tag=method.delivery_tag)
 
     def run(self):
         print(threading.currentThread().name + " starts")

workers/de_cookie_generator.py

@@ -1,9 +1,17 @@
-from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher
+import logging
+import sys
+
+from proxy_manager.proxy_manager import ProxyManager
+from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher, MORNING_DATA_CACHE
+from utils.AppLogging import init_logger
 from workers.cookie_generator import CookiesGenerator
-from workers.proxies_constants import PROXY_LIST_DE
 
 if __name__ == '__main__':
-    cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_DE)
+    init_logger()
+    logger = logging.getLogger()
+    logger.addHandler(logging.StreamHandler(stream=sys.stdout))
+    cookiesPublisher = CookiesPublisher(queue_name=MORNING_DATA_CACHE)
     cookiesPublisher.set_up_connection()
-    cookieGenerator = CookiesGenerator(cookiesPublisher=cookiesPublisher, proxy_to_use_list=PROXY_LIST_DE)
+    cookieGenerator = CookiesGenerator(proxy_manager=ProxyManager(logger=logger), cookiesPublisher=cookiesPublisher,
+                                       logger=logger)
     cookieGenerator.run()

workers/link_validator_with_provided_list.py

@@ -155,12 +155,8 @@ class LinkValidatorWithProvidedList(threading.Thread):
         _message_in_queue_count = self.cookiesPublisher.message_count()
         print("message count in queue is {}".format(_message_in_queue_count))
         _received_object = body.decode("UTF-8")
-        _received_dict = None
+        _received_dict = json.loads(_received_object)
-        if "glrd" in _received_object:
+        _received_cookies = _received_dict["cookiesStr"]
-            _received_dict = json.loads(_received_object)
-            _received_cookies = _received_dict["cookiesStr"]
-        else:
-            _received_cookies = _received_object
         self.cookie_str = _received_cookies
         random.shuffle(self.link_to_validate_list)
         if len(self.link_to_validate_list) > 0 and _message_in_queue_count >= self.limit: