import json import logging import random import threading import time from http.cookies import SimpleCookie import pika from models.jsdata_le_pojo import JsDataLeTypePojo from models.jsdata_pojo import JsDataPojo from proxy_manager.proxy_manager import ProxyManager from queue_message.CookiesPublisher import CookiesPublisher, QUEUE_HOST, credentials, REQUEST_DATA_QUEUE, \ REQUEST_DATA_QUEUE_TEST, MORNING_DATA_CACHE_BAK from workers.captcha_result_getter import CaptchaResultGetter from workers.proxies_constants import PROXY_LIST_FR DVM_LIST = [2, 3, 4, 6] class CookiesGenerator(threading.Thread): def __init__(self, proxy_manager: ProxyManager, cookiesPublisher: CookiesPublisher, logger=logging.getLogger(__name__)): super().__init__() self.connection = None self.cookiesPublisher = cookiesPublisher self.channel = None self.valid_csrf = None self.proxy_manager = proxy_manager self.queue_name = MORNING_DATA_CACHE_BAK self.logger = logger def set_up_connection(self): self.connection = pika.BlockingConnection( pika.ConnectionParameters(host=QUEUE_HOST, port=5672, credentials=credentials)) self.channel = self.connection.channel() def listen_to_queue(self, callback): self.channel.basic_qos(prefetch_count=1) self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback) self.channel.start_consuming() def message_count(self): return self.channel.queue_declare(queue=self.queue_name, durable=True).method.message_count def on_message(self, ch, method, properties, body): print(f" [x] Received {body}") _received_object = body.decode("UTF-8") _received_dict = json.loads(_received_object) _received_cookies = _received_dict["cookiesStr"] # generate 10 cookies to new_queue captchaResultGetter = CaptchaResultGetter() _cookies_count = random.randint(8, 10) _received_cookies = body.decode("UTF-8") dvm = random.choice(DVM_LIST) hc = random.choice(DVM_LIST) _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() # new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, dvm=dvm, hc=hc) _app_sig = get_app_and_app_sig(_received_cookies) # if new_cookie is not None and _app_sig is not None: # new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie # new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace( # "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "") # print("new_cookie is " + new_cookie) # _received_cookies = new_cookie # self.cookiesPublisher.publish_body(new_cookie) js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'], ua=_received_dict['ua'], br_oh=_received_dict['br_oh'], br_ow=_received_dict['br_ow'], ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'], pr=_received_dict['pr'], plg=_received_dict['plg'], br_h=_received_dict['br_h'], br_w=_received_dict['br_w'], plu=_received_dict['plu'], vnd=_received_dict['vnd'], dvm=_received_dict['dvm'], ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'], rs_h=_received_dict['rs_h'], rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'], emd=_received_dict['emd']) new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, js_data=js_data, old_valid_cookie=_received_cookies, origin="https://www.hermes.com", referrer="https://www.hermes.com") if new_cookie is not None: new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie new_cookie = new_cookie.replace("Path=/;", "").replace( "Secure; SameSite=None", "").replace(" ", "") _received_cookies = new_cookie # _received_dict["cookiesStr"] = new_cookie # self.cookiesPublisher.publish_body(json.dumps(_received_dict)) if _received_cookies is not None: self.logger.info("new cookie is " + _received_cookies) time.sleep(random.randint(1, 3)) # m_s_c = f.scroll m_s_c = random.randint(0, 3) m_c_c = random.randint(3, 5) # click count m_m_c = random.randint(3, 5) # move count m_cm_r = m_c_c / m_m_c m_ms_r = -1 # move scroll ratio if m_s_c == 0: m_ms_r = -1 else: m_ms_r = m_m_c / m_s_c js_le_data = JsDataLeTypePojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'], ua=_received_dict['ua'], br_oh=_received_dict['br_oh'], br_ow=_received_dict['br_ow'], ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'], pr=_received_dict['pr'], plg=_received_dict['plg'], br_h=_received_dict['br_h'], br_w=_received_dict['br_w'], plu=_received_dict['plu'], vnd=_received_dict['vnd'], dvm=_received_dict['dvm'], ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'], rs_h=_received_dict['rs_h'], rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'], m_s_c=m_s_c, m_m_c=m_m_c, m_c_c=m_c_c, m_cm_r=m_cm_r, m_ms_r=m_ms_r, emd=_received_dict['emd']) time.sleep(random.randint(1, 4)) _new_le_cookies = captchaResultGetter.get_le_valid_cookie(proxy_to_use=_proxy_to_use, js_le_type_data=js_le_data, old_valid_cookie=_received_cookies) _received_dict["cookiesStr"] = new_cookie self.cookiesPublisher.publish_body(json.dumps(_received_dict)) # self.cookiesPublisher.publish_body(new_cookie) ch.basic_ack(delivery_tag=method.delivery_tag) def run(self): print(threading.currentThread().name + " starts") self.set_up_connection() self.listen_to_queue(self.on_message) self.channel.start_consuming() def get_app_and_app_sig(cookies: str): _local_cookies = SimpleCookie() _local_cookies.load(cookies) new_coolies_str = "" new_cookies = {k: v.value for k, v in _local_cookies.items()} for key in _local_cookies: if key == "app" or key == "app.sig": new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";" return new_coolies_str if __name__ == '__main__': cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_TEST) cookiesPublisher.set_up_connection() cookieGenerator = CookiesGenerator(proxy_to_use_list=PROXY_LIST_FR, cookiesPublisher=cookiesPublisher) cookieGenerator.run()