panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

log to file

Browse Source
This commit is contained in:
Lei PAN
2024-03-20 14:18:38 +01:00
parent 53ea721cb6
commit 40c7c7cf96
4 changed files with 63 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
queue_message/appointmentrequestsender.py
+25 -24
View File
@@ -48,10 +48,11 @@ def is_open():
class AppointmentRequestSender(threading.Thread):
def __init__(self, sub_contact_list: list, proxy_to_use_list, cookiesPublisher: CookiesPublisher,
def __init__(self, sub_contact_list: list, proxy_to_use_list, logger, cookiesPublisher: CookiesPublisher,
queue_name=REQUEST_DATA_QUEUE):
super().__init__()
self.connection = None
self.logger = logger
self.cookiesPublisher = cookiesPublisher
self.channel = None
self.valid_csrf = None
@@ -65,22 +66,22 @@ class AppointmentRequestSender(threading.Thread):
self.channel = self.connection.channel()
def listen_to_queue(self, callback):
print("listen to queue {}".format(self.queue_name))
self.logger.info("listen to queue {}".format(self.queue_name))
self.channel.basic_qos(prefetch_count=1)
self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback)
self.channel.start_consuming()
def on_message(self, ch, method, properties, body):
_message_count = self.cookiesPublisher.message_count()
print("message count in queue is {}".format(_message_count))
self.logger.info("message count in queue is {}".format(_message_count))
_received_object = body.decode("UTF-8")
print(f" [x] Received {_received_object}")
self.logger.info(f" [x] Received {_received_object}")
_port = random.randint(40001, 49999)
print("generated port is {}".format(_port))
_choosed_proxy = random.choice(self.proxy_to_use_list)
self.logger.info("generated port is {}".format(_port))
_chosen_proxy = random.choice(self.proxy_to_use_list)
_proxy_to_use = {}
_proxy_to_use["http"] = _choosed_proxy["http"].format(_port)
_proxy_to_use["https"] = _choosed_proxy["https"].format(_port)
_proxy_to_use["http"] = _chosen_proxy["http"].format(_port)
_proxy_to_use["https"] = _chosen_proxy["https"].format(_port)
if "glrd" in _received_object:
_received_dict = json.loads(_received_object)
js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'],
@@ -93,16 +94,16 @@ class AppointmentRequestSender(threading.Thread):
rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'])
_received_cookies = _received_dict["cookiesStr"]
sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict,
proxy_to_use=_proxy_to_use)
proxy_to_use=_proxy_to_use, logger=self.logger)
self.contact_list = filter_contacts(self.contact_list)
# remove already booked contacts
random.shuffle(self.contact_list)
if len(self.contact_list) > 0 and is_open():
captchaResultGetter = CaptchaResultGetter()
print("contact number is {}".format(len(self.contact_list)))
self.logger.info("contact number is {}".format(len(self.contact_list)))
self.contact_list = filter_contacts(self.contact_list)
for con in self.contact_list:
print(con.mail)
self.logger.info(con.mail)
if self.valid_csrf is None:
self.valid_csrf = captchaResultGetter.get_csrf(
proxy_to_use=_proxy_to_use, js_data=js_data,
@@ -110,8 +111,8 @@ class AppointmentRequestSender(threading.Thread):
_new_cookies = captchaResultGetter.get_valid_ch_cookie(sender.proxy_to_use, js_data,
old_valid_cookie=_received_cookies)
if _new_cookies is not None:
print("new cookie is " + _new_cookies)
time.sleep(random.randint(1, 5))
# self.logger.info("new cookie is " + _new_cookies)
time.sleep(random.randint(1, 3))
# m_s_c = f.scroll
m_s_c = random.randint(0, 3)
m_c_c = random.randint(3, 5) # click count
@@ -138,38 +139,38 @@ class AppointmentRequestSender(threading.Thread):
js_le_type_data=js_le_data,
old_valid_cookie=_new_cookies)
if _new_le_cookies is not None:
print("new le type cookie is " + _new_le_cookies)
# self.logger.info("new le type cookie is " + _new_le_cookies)
sender.cookie_str = _new_le_cookies
time.sleep(random.randint(1, 5))
time.sleep(random.randint(1, 3))
can_continue = sender.send_request(HERMES_REGISTER, js_data, con, csrf=self.valid_csrf)
else:
can_continue = RequestResult.COOKIES_ERROR
else:
can_continue = RequestResult.COOKIES_ERROR
if can_continue == RequestResult.BLOCKED:
print("cannot continue, valid_csrf is " + str(self.valid_csrf))
self.logger.info("cannot continue, valid_csrf is " + str(self.valid_csrf))
break
elif can_continue == RequestResult.PROXY_ERROR:
print("PROXY_ERROR, will not reset valid_csrf")
self.logger.info("PROXY_ERROR, will not reset valid_csrf")
elif can_continue == RequestResult.COOKIES_ERROR:
print("COOKIES_ERROR, will not reset valid_csrf")
self.logger.info("COOKIES_ERROR, will not reset valid_csrf")
else:
print("can continue, will reset valid_csrf")
self.logger.info("can continue, will reset valid_csrf")
self.valid_csrf = None
time.sleep(random.randint(1, 2))
print("will ack method.delivery_tag: " + str(method.delivery_tag))
self.logger.info("will ack method.delivery_tag: " + str(method.delivery_tag))
ch.basic_ack(delivery_tag=method.delivery_tag)
else:
print("empty list")
self.logger.info("empty list")
time.sleep(120)
print("will basic_reject method.delivery_tag: " + str(method.delivery_tag))
self.logger.info("will basic_reject method.delivery_tag: " + str(method.delivery_tag))
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
else:
print("not a valid object")
self.logger.info("not a valid object")
ch.basic_ack(delivery_tag=method.delivery_tag)
def run(self):
print(threading.currentThread().name + " starts")
self.logger.info(threading.currentThread().name + " starts")
self.set_up_connection()
self.listen_to_queue(self.on_message)
self.channel.start_consuming()
request_sender.py
+18 -45
View File
@@ -1,19 +1,16 @@
import datetime
import random
import time
from http.cookies import SimpleCookie
import logging
import sys
from threading import Thread
from db.mongo_manager import MONGO_STORE_MANAGER
from excel_reader import read_contacts
from models.contact_pojo import ContactPojo
from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_DE, REQUEST_DATA_QUEUE_TEST, \
REQUEST_DATA_OBJECT, TEST_QUEUE, REGISTER_QUEUE
from queue_message.appointmentrequestsender import AppointmentRequestSender, REQUEST_DATA_QUEUE
from queue_message.CookiesPublisher import CookiesPublisher, TEST_QUEUE
from queue_message.appointmentrequestsender import AppointmentRequestSender
from utiles import is_time_between
from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
from workers.proxies_constants import PROXY_LIST_DE, PROXY_LIST_FR, MOBILE_PROXY_LIST_FR, FR_PROXY_MOB_OXY_STICKY
from workers.sender import Sender
from utils.AppLogging import init_logger
from workers.proxies_constants import MOBILE_PROXY_LIST_FR
IPFIY = 'http://api.ipify.org'
NGROK_TEST = "https://bcc6-193-164-156-53.ngrok-free.app"
@@ -41,64 +38,40 @@ def filter_contacts(_contact_list: list) -> list:
return _contact_list_to_book
def get_valid_csrf() -> str:
captchaResultGetter = CaptchaResultGetter()
_valid_cookie = captchaResultGetter.get_valid_cookie()
# while _valid_cookie is None:
# _valid_cookie = captchaResultGetter.get_valid_cookie()
new_csrf = None
while new_csrf is None and is_open():
valid_cookie = None
if _valid_cookie is not None:
simple_cookie = SimpleCookie()
simple_cookie.load(_valid_cookie)
new_cookies = {k: v.value for k, v in simple_cookie.items()}
new_coolies_str = ""
for key in new_cookies:
print(key)
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
print(new_coolies_str)
valid_cookie = new_coolies_str + "app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;policy=accepted;lang=fr;"
print(valid_cookie)
new_csrf = captchaResultGetter.get_csrf(valid_cookie)
if new_csrf is None:
_valid_cookie = None
while _valid_cookie is None:
_valid_cookie = captchaResultGetter.get_valid_cookie()
time.sleep(2)
return new_csrf
def is_open():
return is_time_between(datetime.time(10, 30), datetime.time(19, 00))
count = 0
init_logger()
logger = logging.getLogger()
logger.addHandler(logging.StreamHandler(stream=sys.stdout))
def send_appointment_request(message_queue_name, _contact_list):
global count
count = count + 1
for _contact in _contact_list:
logger.info(_contact)
_cookiesPublisher = CookiesPublisher(queue_name=message_queue_name)
_cookiesPublisher.set_up_connection()
receiver = AppointmentRequestSender(sub_contact_list=_contact_list, proxy_to_use_list=MOBILE_PROXY_LIST_FR,
queue_name=message_queue_name,
cookiesPublisher=_cookiesPublisher)
cookiesPublisher=_cookiesPublisher, logger=logger)
print("count is " + str(count))
receiver.run()
if __name__ == '__main__':
contacts_file_path = '~/Desktop/contact_list_2024-03-15.xlsx'
contacts_file_path = '~/Desktop/contact_list_2024-03-16.xlsx'
_contact_list = read_contacts(contacts_file_path)
_contact_list_to_book = filter_contacts(_contact_list)
_segment_number = 5
print("{} contacts to book".format(len(_contact_list_to_book)))
_segment_number = 2
logger.info("{} contacts to book".format(len(_contact_list_to_book)))
for i in range(0, _segment_number):
print(i)
logger.info("segment is {}".format(i))
_step = int(len(_contact_list_to_book) / _segment_number)
_sublist = _contact_list_to_book[i:_step * (i + 1)]
print(_sublist[0])
print(_sublist[-1])
_sublist = _contact_list_to_book[i * _step:_step * (i + 1)]
_thread1 = Thread(target=send_appointment_request, args=(TEST_QUEUE, _sublist))
_thread1.start()
utils/AppLogging.py
+13
View File
@@ -0,0 +1,13 @@
import datetime
import logging
from pathlib import Path
LOGS_DIR = str(Path.home())
def init_logger():
logging.basicConfig(filename=LOGS_DIR + "/request_{}.log".format(str(datetime.date.today())),
filemode='a',
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO)
workers/sender.py
+7 -6
View File
@@ -16,12 +16,13 @@ from workers.proxies_constants import PROXY_LIST_FR
class Sender:
def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher, received_dict, proxy_to_use):
def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher, received_dict, proxy_to_use, logger):
self.store_type = "random"
self.cookie = SimpleCookie()
self.cookiesPublisher = cookiesPublisher
# self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
self.cookie_str = cookie_str
self.logger = logger
self._csrf = None
self.received_dict = received_dict
self.proxy_to_use = proxy_to_use
@@ -74,11 +75,11 @@ class Sender:
print(proxy_to_use)
response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data,
timeout=15)
print(response.status_code)
self.logger.info(response.status_code)
if response.status_code == 200:
# add to mongodb
print(response.text)
print(response.url)
self.logger.info(response.text)
self.logger.info("{}:{}".format(contact.mail, response.url))
self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=response.url)
cookies_to_set = response.headers['set-cookie']
self.cookie.load(cookies_to_set)
@@ -86,10 +87,10 @@ class Sender:
new_coolies_str = ""
for key in new_cookies:
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
print("will publish to queue {}".format(new_coolies_str))
self.logger.info("will publish to queue {}".format(new_coolies_str))
# upload the cookie to queue
self.received_dict['cookiesStr'] = new_coolies_str
print("body in json:{}".format(json.dumps(self.received_dict)))
self.logger.info("body in json:{}".format(json.dumps(self.received_dict)))
self.cookiesPublisher.publish_body(json.dumps(self.received_dict))
# self.cookie_str = new_coolies_str
return RequestResult.SUCCESS