import datetime import json import random import threading import time from http.cookies import SimpleCookie import pika from db.mongo_manager import MONGO_STORE_MANAGER from models.contact_pojo import ContactPojo from models.jsdata_pojo import JsDataPojo from models.result_pojo import RequestResult from queue_message.CookiesPublisher import CookiesPublisher from utiles import is_time_between from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER from workers.sender import Sender QUEUE_HOST = "appointment.lpaconsulting.fr" REQUEST_DATA_QUEUE = 'REQUEST_DATA' REQUEST_DATA_DE = 'REQUEST_DATA_DE' credentials = pika.PlainCredentials('appointment', 'ZyuhJZ2xEYWhElhpJjy7YEpZGZwNYJz2fHIu') def is_already_sent(contact: ContactPojo) -> bool: already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day() for required_contact in already_sent_contacts: if contact.mail == required_contact.email: return True return False def filter_contacts(_contact_list: list) -> list: already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day() _contact_list_to_book = [] for contact in _contact_list: _to_add = True for booked in already_sent_contacts: if contact.mail == booked.email: _to_add = False if _to_add: _contact_list_to_book.append(contact) return _contact_list_to_book def get_valid_csrf() -> str: captchaResultGetter = CaptchaResultGetter() _valid_cookie = captchaResultGetter.get_valid_cookie() # while _valid_cookie is None: # _valid_cookie = captchaResultGetter.get_valid_cookie() new_csrf = None while new_csrf is None: valid_cookie = None if _valid_cookie is not None: simple_cookie = SimpleCookie() simple_cookie.load(_valid_cookie) new_cookies = {k: v.value for k, v in simple_cookie.items()} new_coolies_str = "" for key in new_cookies: print(key) new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";" print(new_coolies_str) valid_cookie = new_coolies_str + "app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;policy=accepted;lang=fr;" print(valid_cookie) new_csrf = captchaResultGetter.get_csrf(valid_cookie) if new_csrf is None: _valid_cookie = None while _valid_cookie is None: _valid_cookie = captchaResultGetter.get_valid_cookie() time.sleep(2) return new_csrf def is_open(): return is_time_between(datetime.time(10, 30), datetime.time(19, 00)) class AppointmentRequestSender(threading.Thread): def __init__(self, sub_contact_list: list, proxy_to_use_list, cookiesPublisher: CookiesPublisher, queue_name=REQUEST_DATA_QUEUE): super().__init__() self.connection = None self.cookiesPublisher = cookiesPublisher self.channel = None self.valid_csrf = None self.contact_list = sub_contact_list self.queue_name = queue_name self.proxy_to_use_list = proxy_to_use_list def set_up_connection(self): self.connection = pika.BlockingConnection( pika.ConnectionParameters(host=QUEUE_HOST, port=5672, credentials=credentials)) self.channel = self.connection.channel() def listen_to_queue(self, callback): print("listen to queue {}".format(self.queue_name)) self.channel.basic_qos(prefetch_count=1) self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback) self.channel.start_consuming() def on_message(self, ch, method, properties, body): _message_count = self.cookiesPublisher.message_count() print("message count in queue is {}".format(_message_count)) _received_object = body.decode("UTF-8") print(f" [x] Received {_received_object}") if "glrd" in _received_object: _received_dict = json.loads(_received_object) js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'], ua=_received_dict['ua'], br_oh=_received_dict['br_oh'], br_ow=_received_dict['br_ow'], ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'], pr=_received_dict['pr'], plg=_received_dict['plg'], br_h=_received_dict['br_h'], br_w=_received_dict['br_w'], plu=_received_dict['plu'], vnd=_received_dict['vnd'], dvm=_received_dict['dvm'], ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'], rs_h=_received_dict['rs_h'], rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd']) _received_cookies = _received_dict["cookiesStr"] sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict, proxy_to_use=random.choice(self.proxy_to_use_list)) self.contact_list = filter_contacts(self.contact_list) # remove already booked contacts random.shuffle(self.contact_list) if len(self.contact_list) > 0 and is_open(): captchaResultGetter = CaptchaResultGetter() print("contact number is {}".format(len(self.contact_list))) self.contact_list = filter_contacts(self.contact_list) for con in self.contact_list: # if not is_already_sent(con): print(con.mail) # time.sleep(random.randint(1, 5)) if self.valid_csrf is None: self.valid_csrf = captchaResultGetter.get_csrf( proxy_to_use=random.choice(self.proxy_to_use_list), cookie=body.decode("UTF-8")) _new_cookies = captchaResultGetter.get_valid_ch_cookie(sender.proxy_to_use, js_data, old_valid_cookie=_received_cookies) if _received_cookies is not None: print("new cookie is " + _received_cookies) sender.cookie_str = _received_cookies time.sleep(random.randint(1, 5)) can_continue = sender.send_request(HERMES_REGISTER, con, csrf=self.valid_csrf) else: can_continue = RequestResult.COOKIES_ERROR if can_continue == RequestResult.BLOCKED: print("cannot continue, valid_csrf is " + str(self.valid_csrf)) break elif can_continue == RequestResult.PROXY_ERROR: print("PROXY_ERROR, will not reset valid_csrf") elif can_continue == RequestResult.COOKIES_ERROR: print("COOKIES_ERROR, will not reset valid_csrf") else: print("can continue, will reset valid_csrf") self.valid_csrf = None time.sleep(random.randint(1, 2)) print("will ack method.delivery_tag: " + str(method.delivery_tag)) ch.basic_ack(delivery_tag=method.delivery_tag) else: print("empty list") time.sleep(120) print("will basic_reject method.delivery_tag: " + str(method.delivery_tag)) ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True) else: print("not a valid object") ch.basic_ack(delivery_tag=method.delivery_tag) def run(self): print(threading.currentThread().name + " starts") self.set_up_connection() self.listen_to_queue(self.on_message) self.channel.start_consuming()