panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/tls_ja3' of bitbucket.org:panleicim/appointment_request into feature/tls_ja3

Browse Source
This commit is contained in:
PAN Lei
2024-12-03 18:49:02 +01:00
parent 629a18aaf9 8da4b5d530
commit adbabca2dc
7 changed files with 145 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/MessagerTransporter.py
+37
View File
@@ -39,6 +39,43 @@ class MessageTransporter(threading.Thread):
exit(0)
class MessageRemover(threading.Thread):
def __init__(self, limit: int,
queue_to_listen):
self.queue_to_listen = queue_to_listen
self.limit = limit
def set_up_connection(self):
self.connection = pika.BlockingConnection(
pika.ConnectionParameters(host=QUEUE_HOST, port=5672, credentials=credentials))
self.channel = self.connection.channel()
def listen_to_queue(self):
self.channel.basic_qos(prefetch_count=1)
self.channel.basic_consume(queue=self.queue_to_listen, auto_ack=False, on_message_callback=self.on_message)
self.channel.start_consuming()
def message_count(self):
return self.channel.queue_declare(queue=self.queue_to_listen, durable=True).method.message_count
def on_message(self, ch, method, properties, body):
print(f" [x] Received {body}")
_received_object = body.decode("UTF-8")
# _message_in_queue_count = self.message_count
# print("message count in queue is {}".format(_message_in_queue_count))
ch.basic_ack(delivery_tag=method.delivery_tag)
if self.message_count() <= self.limit:
print("all messages are processed")
exit(0)
def empty_message_just_to(left_message_number, queue_name):
_remover = MessageRemover(left_message_number, queue_name)
_remover.set_up_connection()
_remover.listen_to_queue()
def migrate_message_to_queue(from_queue, to_queue="MORNING_DATA_CACHE_BAK"):
cookiesPublisher = CookiesPublisher(queue_name=to_queue)
cookiesPublisher.set_up_connection()
workers/captcha_result_getter.py
+4 -5
View File
@@ -112,17 +112,16 @@ class CaptchaResultGetter:
return _raw_data
def get_valid_ch_cookie(self, proxy_to_use, js_data: JsDataPojo, old_valid_cookie: str = None,
referrer: str = None) -> Union[str, None]:
referrer: str = "https://rendezvousparis.hermes.com/",
origin="https://rendezvousparis.hermes.com/") -> Union[str, None]:
# extract datadome cookie
# get new cookie
_cookies_to_use = get_datadome_cookies(old_valid_cookie)
_app_cookies = get_app_cookies(old_valid_cookie)
_lang_cookies = get_lang_cookies(old_valid_cookie)
_origin = "https://rendezvousparis.hermes.com/"
_origin = origin
# _origin = "https://www.hermes.com"
_referer = "https://rendezvousparis.hermes.com/"
if referrer is not None:
_referer = referrer
_referer = referrer
# _referer = "https://www.hermes.com"
headers = {'content-Type': 'application/x-www-form-urlencoded',
workers/cookie_generator.py
+83 -35
View File
@@ -1,3 +1,5 @@
import json
import logging
import random
import threading
import time
@@ -5,8 +7,11 @@ from http.cookies import SimpleCookie
import pika
from models.jsdata_le_pojo import JsDataLeTypePojo
from models.jsdata_pojo import JsDataPojo
from proxy_manager.proxy_manager import ProxyManager
from queue_message.CookiesPublisher import CookiesPublisher, QUEUE_HOST, credentials, REQUEST_DATA_QUEUE, \
REQUEST_DATA_QUEUE_TEST
REQUEST_DATA_QUEUE_TEST, MORNING_DATA_CACHE_BAK
from workers.captcha_result_getter import CaptchaResultGetter
from workers.proxies_constants import PROXY_LIST_FR
@@ -14,13 +19,16 @@ DVM_LIST = [2, 3, 4, 6]
class CookiesGenerator(threading.Thread):
def __init__(self, proxy_to_use_list: list, cookiesPublisher: CookiesPublisher):
def __init__(self, proxy_manager: ProxyManager, cookiesPublisher: CookiesPublisher,
logger=logging.getLogger(__name__)):
super().__init__()
self.connection = None
self.cookiesPublisher = cookiesPublisher
self.channel = None
self.valid_csrf = None
self.proxy_to_use_list = proxy_to_use_list
self.proxy_manager = proxy_manager
self.queue_name = MORNING_DATA_CACHE_BAK
self.logger = logger
def set_up_connection(self):
self.connection = pika.BlockingConnection(
@@ -29,50 +37,90 @@ class CookiesGenerator(threading.Thread):
def listen_to_queue(self, callback):
self.channel.basic_qos(prefetch_count=1)
self.channel.basic_consume(queue=REQUEST_DATA_QUEUE, auto_ack=False, on_message_callback=callback)
self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback)
self.channel.start_consuming()
def message_count(self):
return self.channel.queue_declare(queue=REQUEST_DATA_QUEUE, durable=True).method.message_count
return self.channel.queue_declare(queue=self.queue_name, durable=True).method.message_count
def on_message(self, ch, method, properties, body):
print(f" [x] Received {body}")
_cached_message_count = self.cookiesPublisher.message_count()
_valid_message_count = self.message_count()
print("message count in cache is {}".format(_cached_message_count))
print("message count in current queue is {}".format(_valid_message_count))
_received_object = body.decode("UTF-8")
_received_dict = json.loads(_received_object)
_received_cookies = _received_dict["cookiesStr"]
# generate 10 cookies to new_queue
captchaResultGetter = CaptchaResultGetter()
_cookies_count = random.randint(8, 10)
_received_cookies = body.decode("UTF-8")
dvm = random.choice(DVM_LIST)
hc = random.choice(DVM_LIST)
if _cached_message_count < 10 and _valid_message_count > 40:
_proxy_to_use = random.choice(self.proxy_to_use_list)
for i in range(1, _cookies_count):
# new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, dvm=dvm, hc=hc)
_app_sig = get_app_and_app_sig(_received_cookies)
# if new_cookie is not None and _app_sig is not None:
# new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
# new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
# "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "")
# print("new_cookie is " + new_cookie)
# _received_cookies = new_cookie
# self.cookiesPublisher.publish_body(new_cookie)
new_cookie = captchaResultGetter.get_valid_cookie(proxy_to_use=_proxy_to_use,
old_valid_cookie=_received_cookies, dvm=dvm,
hc=hc)
if new_cookie is not None and _app_sig is not None:
new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
"Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "")
_received_cookies = new_cookie
self.cookiesPublisher.publish_body(new_cookie)
ch.basic_ack(delivery_tag=method.delivery_tag)
else:
time.sleep(30)
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
_proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
# new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, dvm=dvm, hc=hc)
_app_sig = get_app_and_app_sig(_received_cookies)
# if new_cookie is not None and _app_sig is not None:
# new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
# new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
# "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "")
# print("new_cookie is " + new_cookie)
# _received_cookies = new_cookie
# self.cookiesPublisher.publish_body(new_cookie)
js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'],
ua=_received_dict['ua'], br_oh=_received_dict['br_oh'], br_ow=_received_dict['br_ow'],
ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'], pr=_received_dict['pr'],
plg=_received_dict['plg'], br_h=_received_dict['br_h'], br_w=_received_dict['br_w'],
plu=_received_dict['plu'], vnd=_received_dict['vnd'], dvm=_received_dict['dvm'],
ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'],
rs_h=_received_dict['rs_h'],
rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'], emd=_received_dict['emd'])
new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, js_data=js_data,
old_valid_cookie=_received_cookies,
origin="https://www.hermes.com",
referrer="https://www.hermes.com")
if new_cookie is not None:
new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
new_cookie = new_cookie.replace("Path=/;", "").replace(
"Secure; SameSite=None", "").replace(" ", "")
_received_cookies = new_cookie
# _received_dict["cookiesStr"] = new_cookie
# self.cookiesPublisher.publish_body(json.dumps(_received_dict))
if _received_cookies is not None:
self.logger.info("new cookie is " + _received_cookies)
time.sleep(random.randint(1, 3))
# m_s_c = f.scroll
m_s_c = random.randint(0, 3)
m_c_c = random.randint(3, 5) # click count
m_m_c = random.randint(3, 5) # move count
m_cm_r = m_c_c / m_m_c
m_ms_r = -1 # move scroll ratio
if m_s_c == 0:
m_ms_r = -1
else:
m_ms_r = m_m_c / m_s_c
js_le_data = JsDataLeTypePojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'],
hc=_received_dict['hc'],
ua=_received_dict['ua'], br_oh=_received_dict['br_oh'],
br_ow=_received_dict['br_ow'],
ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'],
pr=_received_dict['pr'],
plg=_received_dict['plg'], br_h=_received_dict['br_h'],
br_w=_received_dict['br_w'],
plu=_received_dict['plu'], vnd=_received_dict['vnd'],
dvm=_received_dict['dvm'],
ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'],
rs_h=_received_dict['rs_h'],
rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'],
m_s_c=m_s_c, m_m_c=m_m_c, m_c_c=m_c_c,
m_cm_r=m_cm_r, m_ms_r=m_ms_r, emd=_received_dict['emd'])
time.sleep(random.randint(1, 4))
_new_le_cookies = captchaResultGetter.get_le_valid_cookie(proxy_to_use=_proxy_to_use,
js_le_type_data=js_le_data,
old_valid_cookie=_received_cookies)
_received_dict["cookiesStr"] = new_cookie
self.cookiesPublisher.publish_body(json.dumps(_received_dict))
# self.cookiesPublisher.publish_body(new_cookie)
ch.basic_ack(delivery_tag=method.delivery_tag)
def run(self):
print(threading.currentThread().name + " starts")
workers/de_cookie_generator.py
+12 -4
View File
@@ -1,9 +1,17 @@
from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher
import logging
import sys
from proxy_manager.proxy_manager import ProxyManager
from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher, MORNING_DATA_CACHE
from utils.AppLogging import init_logger
from workers.cookie_generator import CookiesGenerator
from workers.proxies_constants import PROXY_LIST_DE
if __name__ == '__main__':
cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_DE)
init_logger()
logger = logging.getLogger()
logger.addHandler(logging.StreamHandler(stream=sys.stdout))
cookiesPublisher = CookiesPublisher(queue_name=MORNING_DATA_CACHE)
cookiesPublisher.set_up_connection()
cookieGenerator = CookiesGenerator(cookiesPublisher=cookiesPublisher, proxy_to_use_list=PROXY_LIST_DE)
cookieGenerator = CookiesGenerator(proxy_manager=ProxyManager(logger=logger), cookiesPublisher=cookiesPublisher,
logger=logger)
cookieGenerator.run()
workers/link_validator_with_provided_list.py
+2 -6
View File
@@ -155,12 +155,8 @@ class LinkValidatorWithProvidedList(threading.Thread):
_message_in_queue_count = self.cookiesPublisher.message_count()
print("message count in queue is {}".format(_message_in_queue_count))
_received_object = body.decode("UTF-8")
_received_dict = None
if "glrd" in _received_object:
_received_dict = json.loads(_received_object)
_received_cookies = _received_dict["cookiesStr"]
else:
_received_cookies = _received_object
_received_dict = json.loads(_received_object)
_received_cookies = _received_dict["cookiesStr"]
self.cookie_str = _received_cookies
random.shuffle(self.link_to_validate_list)
if len(self.link_to_validate_list) > 0 and _message_in_queue_count >= self.limit: