log to file

queue_message/appointmentrequestsender.py

@@ -48,10 +48,11 @@ def is_open():
 
 
 class AppointmentRequestSender(threading.Thread):
-    def __init__(self, sub_contact_list: list, proxy_to_use_list, cookiesPublisher: CookiesPublisher,
+    def __init__(self, sub_contact_list: list, proxy_to_use_list, logger, cookiesPublisher: CookiesPublisher,
                  queue_name=REQUEST_DATA_QUEUE):
         super().__init__()
         self.connection = None
+        self.logger = logger
         self.cookiesPublisher = cookiesPublisher
         self.channel = None
         self.valid_csrf = None
@@ -65,22 +66,22 @@ class AppointmentRequestSender(threading.Thread):
         self.channel = self.connection.channel()
 
     def listen_to_queue(self, callback):
-        print("listen to queue {}".format(self.queue_name))
+        self.logger.info("listen to queue {}".format(self.queue_name))
         self.channel.basic_qos(prefetch_count=1)
         self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback)
         self.channel.start_consuming()
 
     def on_message(self, ch, method, properties, body):
         _message_count = self.cookiesPublisher.message_count()
-        print("message count in queue is {}".format(_message_count))
+        self.logger.info("message count in queue is {}".format(_message_count))
         _received_object = body.decode("UTF-8")
-        print(f" [x] Received {_received_object}")
+        self.logger.info(f" [x] Received {_received_object}")
         _port = random.randint(40001, 49999)
-        print("generated port is {}".format(_port))
+        self.logger.info("generated port is {}".format(_port))
-        _choosed_proxy = random.choice(self.proxy_to_use_list)
+        _chosen_proxy = random.choice(self.proxy_to_use_list)
         _proxy_to_use = {}
-        _proxy_to_use["http"] = _choosed_proxy["http"].format(_port)
+        _proxy_to_use["http"] = _chosen_proxy["http"].format(_port)
-        _proxy_to_use["https"] = _choosed_proxy["https"].format(_port)
+        _proxy_to_use["https"] = _chosen_proxy["https"].format(_port)
         if "glrd" in _received_object:
             _received_dict = json.loads(_received_object)
             js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'],
@@ -93,16 +94,16 @@ class AppointmentRequestSender(threading.Thread):
                                  rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'])
             _received_cookies = _received_dict["cookiesStr"]
             sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict,
-                            proxy_to_use=_proxy_to_use)
+                            proxy_to_use=_proxy_to_use, logger=self.logger)
             self.contact_list = filter_contacts(self.contact_list)
             # remove already booked contacts
             random.shuffle(self.contact_list)
             if len(self.contact_list) > 0 and is_open():
                 captchaResultGetter = CaptchaResultGetter()
-                print("contact number is {}".format(len(self.contact_list)))
+                self.logger.info("contact number is {}".format(len(self.contact_list)))
                 self.contact_list = filter_contacts(self.contact_list)
                 for con in self.contact_list:
-                    print(con.mail)
+                    self.logger.info(con.mail)
                     if self.valid_csrf is None:
                         self.valid_csrf = captchaResultGetter.get_csrf(
                             proxy_to_use=_proxy_to_use, js_data=js_data,
@@ -110,8 +111,8 @@ class AppointmentRequestSender(threading.Thread):
                     _new_cookies = captchaResultGetter.get_valid_ch_cookie(sender.proxy_to_use, js_data,
                                                                            old_valid_cookie=_received_cookies)
                     if _new_cookies is not None:
-                        print("new cookie is " + _new_cookies)
+                        # self.logger.info("new cookie is " + _new_cookies)
-                        time.sleep(random.randint(1, 5))
+                        time.sleep(random.randint(1, 3))
                         # m_s_c = f.scroll
                         m_s_c = random.randint(0, 3)
                         m_c_c = random.randint(3, 5)  # click count
@@ -138,38 +139,38 @@ class AppointmentRequestSender(threading.Thread):
                                                                                   js_le_type_data=js_le_data,
                                                                                   old_valid_cookie=_new_cookies)
                         if _new_le_cookies is not None:
-                            print("new le type cookie is " + _new_le_cookies)
+                            # self.logger.info("new le type cookie is " + _new_le_cookies)
                             sender.cookie_str = _new_le_cookies
-                            time.sleep(random.randint(1, 5))
+                            time.sleep(random.randint(1, 3))
                             can_continue = sender.send_request(HERMES_REGISTER, js_data, con, csrf=self.valid_csrf)
                         else:
                             can_continue = RequestResult.COOKIES_ERROR
                     else:
                         can_continue = RequestResult.COOKIES_ERROR
                     if can_continue == RequestResult.BLOCKED:
-                        print("cannot continue, valid_csrf is " + str(self.valid_csrf))
+                        self.logger.info("cannot continue, valid_csrf is " + str(self.valid_csrf))
                         break
                     elif can_continue == RequestResult.PROXY_ERROR:
-                        print("PROXY_ERROR, will not reset valid_csrf")
+                        self.logger.info("PROXY_ERROR, will not reset valid_csrf")
                     elif can_continue == RequestResult.COOKIES_ERROR:
-                        print("COOKIES_ERROR, will not reset valid_csrf")
+                        self.logger.info("COOKIES_ERROR, will not reset valid_csrf")
                     else:
-                        print("can continue, will reset valid_csrf")
+                        self.logger.info("can continue, will reset valid_csrf")
                         self.valid_csrf = None
                 time.sleep(random.randint(1, 2))
-                print("will ack method.delivery_tag: " + str(method.delivery_tag))
+                self.logger.info("will ack method.delivery_tag: " + str(method.delivery_tag))
                 ch.basic_ack(delivery_tag=method.delivery_tag)
             else:
-                print("empty list")
+                self.logger.info("empty list")
                 time.sleep(120)
-                print("will basic_reject method.delivery_tag: " + str(method.delivery_tag))
+                self.logger.info("will basic_reject method.delivery_tag: " + str(method.delivery_tag))
                 ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
         else:
-            print("not a valid object")
+            self.logger.info("not a valid object")
             ch.basic_ack(delivery_tag=method.delivery_tag)
 
     def run(self):
-        print(threading.currentThread().name + " starts")
+        self.logger.info(threading.currentThread().name + " starts")
         self.set_up_connection()
         self.listen_to_queue(self.on_message)
         self.channel.start_consuming()

request_sender.py

@@ -1,19 +1,16 @@
 import datetime
-import random
+import logging
-import time
+import sys
-from http.cookies import SimpleCookie
 from threading import Thread
 
 from db.mongo_manager import MONGO_STORE_MANAGER
 from excel_reader import read_contacts
 from models.contact_pojo import ContactPojo
-from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_DE, REQUEST_DATA_QUEUE_TEST, \
+from queue_message.CookiesPublisher import CookiesPublisher, TEST_QUEUE
-    REQUEST_DATA_OBJECT, TEST_QUEUE, REGISTER_QUEUE
+from queue_message.appointmentrequestsender import AppointmentRequestSender
-from queue_message.appointmentrequestsender import AppointmentRequestSender, REQUEST_DATA_QUEUE
 from utiles import is_time_between
-from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
+from utils.AppLogging import init_logger
-from workers.proxies_constants import PROXY_LIST_DE, PROXY_LIST_FR, MOBILE_PROXY_LIST_FR, FR_PROXY_MOB_OXY_STICKY
+from workers.proxies_constants import MOBILE_PROXY_LIST_FR
-from workers.sender import Sender
 
 IPFIY = 'http://api.ipify.org'
 NGROK_TEST = "https://bcc6-193-164-156-53.ngrok-free.app"
@@ -41,64 +38,40 @@ def filter_contacts(_contact_list: list) -> list:
     return _contact_list_to_book
 
 
-def get_valid_csrf() -> str:
-    captchaResultGetter = CaptchaResultGetter()
-    _valid_cookie = captchaResultGetter.get_valid_cookie()
-    # while _valid_cookie is None:
-    #     _valid_cookie = captchaResultGetter.get_valid_cookie()
-    new_csrf = None
-    while new_csrf is None and is_open():
-        valid_cookie = None
-        if _valid_cookie is not None:
-            simple_cookie = SimpleCookie()
-            simple_cookie.load(_valid_cookie)
-            new_cookies = {k: v.value for k, v in simple_cookie.items()}
-            new_coolies_str = ""
-            for key in new_cookies:
-                print(key)
-                new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
-            print(new_coolies_str)
-            valid_cookie = new_coolies_str + "app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;policy=accepted;lang=fr;"
-        print(valid_cookie)
-        new_csrf = captchaResultGetter.get_csrf(valid_cookie)
-        if new_csrf is None:
-            _valid_cookie = None
-            while _valid_cookie is None:
-                _valid_cookie = captchaResultGetter.get_valid_cookie()
-        time.sleep(2)
-    return new_csrf
-
-
 def is_open():
     return is_time_between(datetime.time(10, 30), datetime.time(19, 00))
 
 
 count = 0
+init_logger()
+logger = logging.getLogger()
+
+logger.addHandler(logging.StreamHandler(stream=sys.stdout))
 
 
 def send_appointment_request(message_queue_name, _contact_list):
     global count
     count = count + 1
+    for _contact in _contact_list:
+        logger.info(_contact)
     _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name)
     _cookiesPublisher.set_up_connection()
     receiver = AppointmentRequestSender(sub_contact_list=_contact_list, proxy_to_use_list=MOBILE_PROXY_LIST_FR,
                                         queue_name=message_queue_name,
-                                        cookiesPublisher=_cookiesPublisher)
+                                        cookiesPublisher=_cookiesPublisher, logger=logger)
     print("count is " + str(count))
     receiver.run()
 
 
 if __name__ == '__main__':
-    contacts_file_path = '~/Desktop/contact_list_2024-03-15.xlsx'
+    contacts_file_path = '~/Desktop/contact_list_2024-03-16.xlsx'
     _contact_list = read_contacts(contacts_file_path)
     _contact_list_to_book = filter_contacts(_contact_list)
-    _segment_number = 5
+    _segment_number = 2
-    print("{} contacts to book".format(len(_contact_list_to_book)))
+    logger.info("{} contacts to book".format(len(_contact_list_to_book)))
     for i in range(0, _segment_number):
-        print(i)
+        logger.info("segment is {}".format(i))
         _step = int(len(_contact_list_to_book) / _segment_number)
-        _sublist = _contact_list_to_book[i:_step * (i + 1)]
+        _sublist = _contact_list_to_book[i * _step:_step * (i + 1)]
-        print(_sublist[0])
-        print(_sublist[-1])
         _thread1 = Thread(target=send_appointment_request, args=(TEST_QUEUE, _sublist))
         _thread1.start()

utils/AppLogging.py

@@ -0,0 +1,13 @@
+import datetime
+import logging
+from pathlib import Path
+
+LOGS_DIR = str(Path.home())
+
+
+def init_logger():
+    logging.basicConfig(filename=LOGS_DIR + "/request_{}.log".format(str(datetime.date.today())),
+                        filemode='a',
+                        format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
+                        datefmt='%Y-%m-%d %H:%M:%S',
+                        level=logging.INFO)

workers/sender.py

@@ -16,12 +16,13 @@ from workers.proxies_constants import PROXY_LIST_FR
 
 class Sender:
 
-    def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher, received_dict, proxy_to_use):
+    def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher, received_dict, proxy_to_use, logger):
         self.store_type = "random"
         self.cookie = SimpleCookie()
         self.cookiesPublisher = cookiesPublisher
         # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
         self.cookie_str = cookie_str
+        self.logger = logger
         self._csrf = None
         self.received_dict = received_dict
         self.proxy_to_use = proxy_to_use
@@ -74,11 +75,11 @@ class Sender:
             print(proxy_to_use)
             response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data,
                                      timeout=15)
-            print(response.status_code)
+            self.logger.info(response.status_code)
             if response.status_code == 200:
                 # add to mongodb
-                print(response.text)
+                self.logger.info(response.text)
-                print(response.url)
+                self.logger.info("{}:{}".format(contact.mail, response.url))
                 self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=response.url)
                 cookies_to_set = response.headers['set-cookie']
                 self.cookie.load(cookies_to_set)
@@ -86,10 +87,10 @@ class Sender:
                 new_coolies_str = ""
                 for key in new_cookies:
                     new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
-                print("will publish to queue {}".format(new_coolies_str))
+                self.logger.info("will publish to queue {}".format(new_coolies_str))
                 # upload the cookie to queue
                 self.received_dict['cookiesStr'] = new_coolies_str
-                print("body in json:{}".format(json.dumps(self.received_dict)))
+                self.logger.info("body in json:{}".format(json.dumps(self.received_dict)))
                 self.cookiesPublisher.publish_body(json.dumps(self.received_dict))
                 # self.cookie_str = new_coolies_str
                 return RequestResult.SUCCESS