diff --git a/queue_message/appointmentrequestsender.py b/queue_message/appointmentrequestsender.py index 7d2ed7d..7cdc2cc 100644 --- a/queue_message/appointmentrequestsender.py +++ b/queue_message/appointmentrequestsender.py @@ -48,10 +48,11 @@ def is_open(): class AppointmentRequestSender(threading.Thread): - def __init__(self, sub_contact_list: list, proxy_to_use_list, cookiesPublisher: CookiesPublisher, + def __init__(self, sub_contact_list: list, proxy_to_use_list, logger, cookiesPublisher: CookiesPublisher, queue_name=REQUEST_DATA_QUEUE): super().__init__() self.connection = None + self.logger = logger self.cookiesPublisher = cookiesPublisher self.channel = None self.valid_csrf = None @@ -65,22 +66,22 @@ class AppointmentRequestSender(threading.Thread): self.channel = self.connection.channel() def listen_to_queue(self, callback): - print("listen to queue {}".format(self.queue_name)) + self.logger.info("listen to queue {}".format(self.queue_name)) self.channel.basic_qos(prefetch_count=1) self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback) self.channel.start_consuming() def on_message(self, ch, method, properties, body): _message_count = self.cookiesPublisher.message_count() - print("message count in queue is {}".format(_message_count)) + self.logger.info("message count in queue is {}".format(_message_count)) _received_object = body.decode("UTF-8") - print(f" [x] Received {_received_object}") + self.logger.info(f" [x] Received {_received_object}") _port = random.randint(40001, 49999) - print("generated port is {}".format(_port)) - _choosed_proxy = random.choice(self.proxy_to_use_list) + self.logger.info("generated port is {}".format(_port)) + _chosen_proxy = random.choice(self.proxy_to_use_list) _proxy_to_use = {} - _proxy_to_use["http"] = _choosed_proxy["http"].format(_port) - _proxy_to_use["https"] = _choosed_proxy["https"].format(_port) + _proxy_to_use["http"] = _chosen_proxy["http"].format(_port) + _proxy_to_use["https"] = _chosen_proxy["https"].format(_port) if "glrd" in _received_object: _received_dict = json.loads(_received_object) js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'], @@ -93,16 +94,16 @@ class AppointmentRequestSender(threading.Thread): rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd']) _received_cookies = _received_dict["cookiesStr"] sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict, - proxy_to_use=_proxy_to_use) + proxy_to_use=_proxy_to_use, logger=self.logger) self.contact_list = filter_contacts(self.contact_list) # remove already booked contacts random.shuffle(self.contact_list) if len(self.contact_list) > 0 and is_open(): captchaResultGetter = CaptchaResultGetter() - print("contact number is {}".format(len(self.contact_list))) + self.logger.info("contact number is {}".format(len(self.contact_list))) self.contact_list = filter_contacts(self.contact_list) for con in self.contact_list: - print(con.mail) + self.logger.info(con.mail) if self.valid_csrf is None: self.valid_csrf = captchaResultGetter.get_csrf( proxy_to_use=_proxy_to_use, js_data=js_data, @@ -110,8 +111,8 @@ class AppointmentRequestSender(threading.Thread): _new_cookies = captchaResultGetter.get_valid_ch_cookie(sender.proxy_to_use, js_data, old_valid_cookie=_received_cookies) if _new_cookies is not None: - print("new cookie is " + _new_cookies) - time.sleep(random.randint(1, 5)) + # self.logger.info("new cookie is " + _new_cookies) + time.sleep(random.randint(1, 3)) # m_s_c = f.scroll m_s_c = random.randint(0, 3) m_c_c = random.randint(3, 5) # click count @@ -138,38 +139,38 @@ class AppointmentRequestSender(threading.Thread): js_le_type_data=js_le_data, old_valid_cookie=_new_cookies) if _new_le_cookies is not None: - print("new le type cookie is " + _new_le_cookies) + # self.logger.info("new le type cookie is " + _new_le_cookies) sender.cookie_str = _new_le_cookies - time.sleep(random.randint(1, 5)) + time.sleep(random.randint(1, 3)) can_continue = sender.send_request(HERMES_REGISTER, js_data, con, csrf=self.valid_csrf) else: can_continue = RequestResult.COOKIES_ERROR else: can_continue = RequestResult.COOKIES_ERROR if can_continue == RequestResult.BLOCKED: - print("cannot continue, valid_csrf is " + str(self.valid_csrf)) + self.logger.info("cannot continue, valid_csrf is " + str(self.valid_csrf)) break elif can_continue == RequestResult.PROXY_ERROR: - print("PROXY_ERROR, will not reset valid_csrf") + self.logger.info("PROXY_ERROR, will not reset valid_csrf") elif can_continue == RequestResult.COOKIES_ERROR: - print("COOKIES_ERROR, will not reset valid_csrf") + self.logger.info("COOKIES_ERROR, will not reset valid_csrf") else: - print("can continue, will reset valid_csrf") + self.logger.info("can continue, will reset valid_csrf") self.valid_csrf = None time.sleep(random.randint(1, 2)) - print("will ack method.delivery_tag: " + str(method.delivery_tag)) + self.logger.info("will ack method.delivery_tag: " + str(method.delivery_tag)) ch.basic_ack(delivery_tag=method.delivery_tag) else: - print("empty list") + self.logger.info("empty list") time.sleep(120) - print("will basic_reject method.delivery_tag: " + str(method.delivery_tag)) + self.logger.info("will basic_reject method.delivery_tag: " + str(method.delivery_tag)) ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True) else: - print("not a valid object") + self.logger.info("not a valid object") ch.basic_ack(delivery_tag=method.delivery_tag) def run(self): - print(threading.currentThread().name + " starts") + self.logger.info(threading.currentThread().name + " starts") self.set_up_connection() self.listen_to_queue(self.on_message) self.channel.start_consuming() diff --git a/request_sender.py b/request_sender.py index 497b151..03a1875 100644 --- a/request_sender.py +++ b/request_sender.py @@ -1,19 +1,16 @@ import datetime -import random -import time -from http.cookies import SimpleCookie +import logging +import sys from threading import Thread from db.mongo_manager import MONGO_STORE_MANAGER from excel_reader import read_contacts from models.contact_pojo import ContactPojo -from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_DE, REQUEST_DATA_QUEUE_TEST, \ - REQUEST_DATA_OBJECT, TEST_QUEUE, REGISTER_QUEUE -from queue_message.appointmentrequestsender import AppointmentRequestSender, REQUEST_DATA_QUEUE +from queue_message.CookiesPublisher import CookiesPublisher, TEST_QUEUE +from queue_message.appointmentrequestsender import AppointmentRequestSender from utiles import is_time_between -from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER -from workers.proxies_constants import PROXY_LIST_DE, PROXY_LIST_FR, MOBILE_PROXY_LIST_FR, FR_PROXY_MOB_OXY_STICKY -from workers.sender import Sender +from utils.AppLogging import init_logger +from workers.proxies_constants import MOBILE_PROXY_LIST_FR IPFIY = 'http://api.ipify.org' NGROK_TEST = "https://bcc6-193-164-156-53.ngrok-free.app" @@ -41,64 +38,40 @@ def filter_contacts(_contact_list: list) -> list: return _contact_list_to_book -def get_valid_csrf() -> str: - captchaResultGetter = CaptchaResultGetter() - _valid_cookie = captchaResultGetter.get_valid_cookie() - # while _valid_cookie is None: - # _valid_cookie = captchaResultGetter.get_valid_cookie() - new_csrf = None - while new_csrf is None and is_open(): - valid_cookie = None - if _valid_cookie is not None: - simple_cookie = SimpleCookie() - simple_cookie.load(_valid_cookie) - new_cookies = {k: v.value for k, v in simple_cookie.items()} - new_coolies_str = "" - for key in new_cookies: - print(key) - new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";" - print(new_coolies_str) - valid_cookie = new_coolies_str + "app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;policy=accepted;lang=fr;" - print(valid_cookie) - new_csrf = captchaResultGetter.get_csrf(valid_cookie) - if new_csrf is None: - _valid_cookie = None - while _valid_cookie is None: - _valid_cookie = captchaResultGetter.get_valid_cookie() - time.sleep(2) - return new_csrf - - def is_open(): return is_time_between(datetime.time(10, 30), datetime.time(19, 00)) count = 0 +init_logger() +logger = logging.getLogger() + +logger.addHandler(logging.StreamHandler(stream=sys.stdout)) def send_appointment_request(message_queue_name, _contact_list): global count count = count + 1 + for _contact in _contact_list: + logger.info(_contact) _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name) _cookiesPublisher.set_up_connection() receiver = AppointmentRequestSender(sub_contact_list=_contact_list, proxy_to_use_list=MOBILE_PROXY_LIST_FR, queue_name=message_queue_name, - cookiesPublisher=_cookiesPublisher) + cookiesPublisher=_cookiesPublisher, logger=logger) print("count is " + str(count)) receiver.run() if __name__ == '__main__': - contacts_file_path = '~/Desktop/contact_list_2024-03-15.xlsx' + contacts_file_path = '~/Desktop/contact_list_2024-03-16.xlsx' _contact_list = read_contacts(contacts_file_path) _contact_list_to_book = filter_contacts(_contact_list) - _segment_number = 5 - print("{} contacts to book".format(len(_contact_list_to_book))) + _segment_number = 2 + logger.info("{} contacts to book".format(len(_contact_list_to_book))) for i in range(0, _segment_number): - print(i) + logger.info("segment is {}".format(i)) _step = int(len(_contact_list_to_book) / _segment_number) - _sublist = _contact_list_to_book[i:_step * (i + 1)] - print(_sublist[0]) - print(_sublist[-1]) + _sublist = _contact_list_to_book[i * _step:_step * (i + 1)] _thread1 = Thread(target=send_appointment_request, args=(TEST_QUEUE, _sublist)) _thread1.start() diff --git a/utils/AppLogging.py b/utils/AppLogging.py new file mode 100644 index 0000000..ca8b8b8 --- /dev/null +++ b/utils/AppLogging.py @@ -0,0 +1,13 @@ +import datetime +import logging +from pathlib import Path + +LOGS_DIR = str(Path.home()) + + +def init_logger(): + logging.basicConfig(filename=LOGS_DIR + "/request_{}.log".format(str(datetime.date.today())), + filemode='a', + format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + level=logging.INFO) diff --git a/workers/sender.py b/workers/sender.py index e189a3f..8b861cb 100644 --- a/workers/sender.py +++ b/workers/sender.py @@ -16,12 +16,13 @@ from workers.proxies_constants import PROXY_LIST_FR class Sender: - def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher, received_dict, proxy_to_use): + def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher, received_dict, proxy_to_use, logger): self.store_type = "random" self.cookie = SimpleCookie() self.cookiesPublisher = cookiesPublisher # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;' self.cookie_str = cookie_str + self.logger = logger self._csrf = None self.received_dict = received_dict self.proxy_to_use = proxy_to_use @@ -74,11 +75,11 @@ class Sender: print(proxy_to_use) response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data, timeout=15) - print(response.status_code) + self.logger.info(response.status_code) if response.status_code == 200: # add to mongodb - print(response.text) - print(response.url) + self.logger.info(response.text) + self.logger.info("{}:{}".format(contact.mail, response.url)) self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=response.url) cookies_to_set = response.headers['set-cookie'] self.cookie.load(cookies_to_set) @@ -86,10 +87,10 @@ class Sender: new_coolies_str = "" for key in new_cookies: new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";" - print("will publish to queue {}".format(new_coolies_str)) + self.logger.info("will publish to queue {}".format(new_coolies_str)) # upload the cookie to queue self.received_dict['cookiesStr'] = new_coolies_str - print("body in json:{}".format(json.dumps(self.received_dict))) + self.logger.info("body in json:{}".format(json.dumps(self.received_dict))) self.cookiesPublisher.publish_body(json.dumps(self.received_dict)) # self.cookie_str = new_coolies_str return RequestResult.SUCCESS