diff --git a/queue_message/CookiesPublisher.py b/queue_message/CookiesPublisher.py index 46ffbf8..1de344f 100644 --- a/queue_message/CookiesPublisher.py +++ b/queue_message/CookiesPublisher.py @@ -31,7 +31,7 @@ class CookiesPublisher: self.queue_method = self.channel.queue_declare(queue=self.to_queue, durable=True) def publish_body(self, body: str): - print("will push to queue {}".format(self.to_queue)) + print("will push to queue {}".format(body)) self.channel.basic_publish(exchange='', routing_key=self.to_queue, body=body, properties=pika.BasicProperties( delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE )) diff --git a/workers/captcha_result_getter.py b/workers/captcha_result_getter.py index ab684da..b76eb2f 100644 --- a/workers/captcha_result_getter.py +++ b/workers/captcha_result_getter.py @@ -112,17 +112,16 @@ class CaptchaResultGetter: return _raw_data def get_valid_ch_cookie(self, proxy_to_use, js_data: JsDataPojo, old_valid_cookie: str = None, - referrer: str = None) -> Union[str, None]: + referrer: str = "https://rendezvousparis.hermes.com/", + origin="https://rendezvousparis.hermes.com/") -> Union[str, None]: # extract datadome cookie # get new cookie _cookies_to_use = get_datadome_cookies(old_valid_cookie) _app_cookies = get_app_cookies(old_valid_cookie) _lang_cookies = get_lang_cookies(old_valid_cookie) - _origin = "https://rendezvousparis.hermes.com/" + _origin = origin # _origin = "https://www.hermes.com" - _referer = "https://rendezvousparis.hermes.com/" - if referrer is not None: - _referer = referrer + _referer = referrer # _referer = "https://www.hermes.com" headers = {'content-Type': 'application/x-www-form-urlencoded', diff --git a/workers/cookie_generator.py b/workers/cookie_generator.py index 65c3e71..daf5520 100644 --- a/workers/cookie_generator.py +++ b/workers/cookie_generator.py @@ -1,3 +1,5 @@ +import json +import logging import random import threading import time @@ -5,8 +7,11 @@ from http.cookies import SimpleCookie import pika +from models.jsdata_le_pojo import JsDataLeTypePojo +from models.jsdata_pojo import JsDataPojo +from proxy_manager.proxy_manager import ProxyManager from queue_message.CookiesPublisher import CookiesPublisher, QUEUE_HOST, credentials, REQUEST_DATA_QUEUE, \ - REQUEST_DATA_QUEUE_TEST + REQUEST_DATA_QUEUE_TEST, MORNING_DATA_CACHE_BAK from workers.captcha_result_getter import CaptchaResultGetter from workers.proxies_constants import PROXY_LIST_FR @@ -14,13 +19,16 @@ DVM_LIST = [2, 3, 4, 6] class CookiesGenerator(threading.Thread): - def __init__(self, proxy_to_use_list: list, cookiesPublisher: CookiesPublisher): + def __init__(self, proxy_manager: ProxyManager, cookiesPublisher: CookiesPublisher, + logger=logging.getLogger(__name__)): super().__init__() self.connection = None self.cookiesPublisher = cookiesPublisher self.channel = None self.valid_csrf = None - self.proxy_to_use_list = proxy_to_use_list + self.proxy_manager = proxy_manager + self.queue_name = MORNING_DATA_CACHE_BAK + self.logger = logger def set_up_connection(self): self.connection = pika.BlockingConnection( @@ -29,50 +37,90 @@ class CookiesGenerator(threading.Thread): def listen_to_queue(self, callback): self.channel.basic_qos(prefetch_count=1) - self.channel.basic_consume(queue=REQUEST_DATA_QUEUE, auto_ack=False, on_message_callback=callback) + self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback) self.channel.start_consuming() def message_count(self): - return self.channel.queue_declare(queue=REQUEST_DATA_QUEUE, durable=True).method.message_count + return self.channel.queue_declare(queue=self.queue_name, durable=True).method.message_count def on_message(self, ch, method, properties, body): print(f" [x] Received {body}") - _cached_message_count = self.cookiesPublisher.message_count() - _valid_message_count = self.message_count() - print("message count in cache is {}".format(_cached_message_count)) - print("message count in current queue is {}".format(_valid_message_count)) - + _received_object = body.decode("UTF-8") + _received_dict = json.loads(_received_object) + _received_cookies = _received_dict["cookiesStr"] # generate 10 cookies to new_queue captchaResultGetter = CaptchaResultGetter() _cookies_count = random.randint(8, 10) _received_cookies = body.decode("UTF-8") dvm = random.choice(DVM_LIST) hc = random.choice(DVM_LIST) - if _cached_message_count < 10 and _valid_message_count > 40: - _proxy_to_use = random.choice(self.proxy_to_use_list) - for i in range(1, _cookies_count): - # new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, dvm=dvm, hc=hc) - _app_sig = get_app_and_app_sig(_received_cookies) - # if new_cookie is not None and _app_sig is not None: - # new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie - # new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace( - # "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "") - # print("new_cookie is " + new_cookie) - # _received_cookies = new_cookie - # self.cookiesPublisher.publish_body(new_cookie) - new_cookie = captchaResultGetter.get_valid_cookie(proxy_to_use=_proxy_to_use, - old_valid_cookie=_received_cookies, dvm=dvm, - hc=hc) - if new_cookie is not None and _app_sig is not None: - new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie - new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace( - "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "") - _received_cookies = new_cookie - self.cookiesPublisher.publish_body(new_cookie) - ch.basic_ack(delivery_tag=method.delivery_tag) - else: - time.sleep(30) - ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True) + _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() + # new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, dvm=dvm, hc=hc) + _app_sig = get_app_and_app_sig(_received_cookies) + # if new_cookie is not None and _app_sig is not None: + # new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie + # new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace( + # "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "") + # print("new_cookie is " + new_cookie) + # _received_cookies = new_cookie + # self.cookiesPublisher.publish_body(new_cookie) + js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'], + ua=_received_dict['ua'], br_oh=_received_dict['br_oh'], br_ow=_received_dict['br_ow'], + ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'], pr=_received_dict['pr'], + plg=_received_dict['plg'], br_h=_received_dict['br_h'], br_w=_received_dict['br_w'], + plu=_received_dict['plu'], vnd=_received_dict['vnd'], dvm=_received_dict['dvm'], + ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'], + rs_h=_received_dict['rs_h'], + rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'], emd=_received_dict['emd']) + + new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, js_data=js_data, + old_valid_cookie=_received_cookies, + origin="https://www.hermes.com", + referrer="https://www.hermes.com") + if new_cookie is not None: + new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie + new_cookie = new_cookie.replace("Path=/;", "").replace( + "Secure; SameSite=None", "").replace(" ", "") + _received_cookies = new_cookie + # _received_dict["cookiesStr"] = new_cookie + # self.cookiesPublisher.publish_body(json.dumps(_received_dict)) + if _received_cookies is not None: + self.logger.info("new cookie is " + _received_cookies) + time.sleep(random.randint(1, 3)) + # m_s_c = f.scroll + m_s_c = random.randint(0, 3) + m_c_c = random.randint(3, 5) # click count + m_m_c = random.randint(3, 5) # move count + m_cm_r = m_c_c / m_m_c + m_ms_r = -1 # move scroll ratio + if m_s_c == 0: + m_ms_r = -1 + else: + m_ms_r = m_m_c / m_s_c + + js_le_data = JsDataLeTypePojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], + hc=_received_dict['hc'], + ua=_received_dict['ua'], br_oh=_received_dict['br_oh'], + br_ow=_received_dict['br_ow'], + ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'], + pr=_received_dict['pr'], + plg=_received_dict['plg'], br_h=_received_dict['br_h'], + br_w=_received_dict['br_w'], + plu=_received_dict['plu'], vnd=_received_dict['vnd'], + dvm=_received_dict['dvm'], + ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'], + rs_h=_received_dict['rs_h'], + rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'], + m_s_c=m_s_c, m_m_c=m_m_c, m_c_c=m_c_c, + m_cm_r=m_cm_r, m_ms_r=m_ms_r, emd=_received_dict['emd']) + time.sleep(random.randint(1, 4)) + _new_le_cookies = captchaResultGetter.get_le_valid_cookie(proxy_to_use=_proxy_to_use, + js_le_type_data=js_le_data, + old_valid_cookie=_received_cookies) + _received_dict["cookiesStr"] = new_cookie + self.cookiesPublisher.publish_body(json.dumps(_received_dict)) + # self.cookiesPublisher.publish_body(new_cookie) + ch.basic_ack(delivery_tag=method.delivery_tag) def run(self): print(threading.currentThread().name + " starts") diff --git a/workers/de_cookie_generator.py b/workers/de_cookie_generator.py index 9acdaa0..5a8e563 100644 --- a/workers/de_cookie_generator.py +++ b/workers/de_cookie_generator.py @@ -1,9 +1,17 @@ -from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher +import logging +import sys + +from proxy_manager.proxy_manager import ProxyManager +from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher, MORNING_DATA_CACHE +from utils.AppLogging import init_logger from workers.cookie_generator import CookiesGenerator -from workers.proxies_constants import PROXY_LIST_DE if __name__ == '__main__': - cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_DE) + init_logger() + logger = logging.getLogger() + logger.addHandler(logging.StreamHandler(stream=sys.stdout)) + cookiesPublisher = CookiesPublisher(queue_name=MORNING_DATA_CACHE) cookiesPublisher.set_up_connection() - cookieGenerator = CookiesGenerator(cookiesPublisher=cookiesPublisher, proxy_to_use_list=PROXY_LIST_DE) + cookieGenerator = CookiesGenerator(proxy_manager=ProxyManager(logger=logger), cookiesPublisher=cookiesPublisher, + logger=logger) cookieGenerator.run()