panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

republish valid cookie

Browse Source
This commit is contained in:
Lei PAN
2024-01-15 16:03:12 +01:00
parent d1ffd28b6a
commit 455997a71c
8 changed files with 244 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/link_validator.py
+69 -13
View File
@@ -1,27 +1,45 @@
import random
import threading
import time
from http.cookies import SimpleCookie
import pika
import requests
from db.mongo_manager import MONGO_STORE_MANAGER
from models.LinkPojo import LinkPojo
from queue_message.CookiesPublisher import CookiesPublisher
from queue_message.receiver import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
from workers.proxies_constants import PROXY_LIST
class LinkValidator:
class LinkValidator(threading.Thread):
def __init__(self, cookie_str, link_to_validate: LinkPojo):
def __init__(self, link_to_validate_list: list, cookiesPublisher: CookiesPublisher):
super().__init__()
self.cookie = SimpleCookie()
self.link = link_to_validate.url
self.cookiesPublisher = cookiesPublisher
self.link_to_validate_list = link_to_validate_list
# self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
self.cookie_str = cookie_str
self.cookie.load(self.cookie_str)
def send_request(self):
def set_up_connection(self):
self.connection = pika.BlockingConnection(
pika.ConnectionParameters(host=QUEUE_HOST, port=5672, credentials=credentials))
self.channel = self.connection.channel()
def listen_to_queue(self, callback):
self.channel.basic_qos(prefetch_count=1)
self.channel.basic_consume(queue=REQUEST_DATA_QUEUE, auto_ack=False, on_message_callback=callback)
self.channel.start_consuming()
def send_request(self, linkPojo: LinkPojo) -> bool:
self.cookie.load(self.cookie_str)
headers = {
'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'max-age=0',
'Referer': self.link,
'Referer': linkPojo.url,
'Cookie': self.cookie_str,
'Sec-Fetch-Mode': 'navigate',
'Host': 'rendezvousparis.hermes.com',
@@ -32,20 +50,58 @@ class LinkValidator:
print(proxy_to_use)
print("received cookie is " + str(self.cookie_str))
try:
response = requests.get(url=self.link, headers=headers, verify=False, proxies=proxy_to_use,
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=proxy_to_use,
timeout=15)
print(response.status_code)
if response.status_code == 200:
print(response.text)
print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo)
# set new cookies
_cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set)
new_cookies = {k: v.value for k, v in self.cookie.items()}
new_coolies_str = ""
for key in new_cookies:
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
print("will publish to queue {}".format(new_coolies_str))
# upload the cookie to queue
self.cookiesPublisher.publish_body(new_coolies_str)
self.cookie_str = new_coolies_str
return True
else:
return None
return False
except Exception as error:
print(error)
return False
def on_message(self, ch, method, properties, body):
print(f" [x] Received {body}")
self.link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
self.cookie_str = body.decode("UTF-8")
random.shuffle(self.link_to_validate_list)
if len(self.link_to_validate_list) > 0:
print("links number is {}".format(len(self.link_to_validate_list)))
for con in self.link_to_validate_list:
# if not is_already_sent(con):
print(con.email)
can_continue = self.send_request(con)
if not can_continue:
print("cannot continue, skip")
break
else:
time.sleep(random.randint(2, 5))
print("can continue, continue")
print("will ack")
ch.basic_ack(delivery_tag=method.delivery_tag)
else:
print("empty list")
if __name__ == '__main__':
link_validator = LinkValidator(
cookie_str="W3BqZ5LSU5~Yl1LAd68HLGCuSELSQzuyPC1xuM0MpMLwX13p0HxwYL4mxM6BgmtLe0TwMyJKRnOdZgMW2CzGW0h53oQDNioqXAt0GbiDbiY5R_AaUajk_O7MtDqPwFCB;policy=accepted;app.sig=C3MpoPl8yFlKfm6UIrgnxYqs8Hs;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiV3Z6QWJGX09jZXpQREFnZW9PcnM4eXNIIn0=;lang=fr;",
link_to_validate="https://rendezvousparis.hermes.com/client/register/55TCMY/validate?code=011367")
link_validator.send_request()
link_list = MONGO_STORE_MANAGER.get_links_to_validate()
cookiesPublisher = CookiesPublisher()
cookiesPublisher.set_up_connection()
receiver = LinkValidator(link_to_validate_list=link_list, cookiesPublisher=cookiesPublisher)
receiver.set_up_connection()
receiver.listen_to_queue(receiver.on_message)
workers/sender.py
+12 -3
View File
@@ -7,16 +7,19 @@ import requests
from db.mongo_manager import MONGO_STORE_MANAGER
from models.ReserveResultPojo import ReserveResultPojo, PublishType
from queue_message.CookiesPublisher import CookiesPublisher
from workers.proxies_constants import PROXY_LIST
class Sender:
def __init__(self, cookie_str):
def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher):
self.store_type = "random"
self.cookie = SimpleCookie()
self.cookiesPublisher = cookiesPublisher
# self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
self.cookie_str = cookie_str
self._csrf = None
self.cookie.load(self.cookie_str)
def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str):
@@ -41,8 +44,12 @@ class Sender:
def send_request(self, url, contact: ContactPojo, csrf: str = None):
if csrf is None:
_csrf = '8Bs2dBwb-nHONOzo9Tei2CcMZglEfsRqUz8E'
if self._csrf is None:
_csrf = '8Bs2dBwb-nHONOzo9Tei2CcMZglEfsRqUz8E'
else:
_csrf = self._csrf
else:
self._csrf = csrf
_csrf = csrf
headers = {'Content-Type': 'application/x-www-form-urlencoded',
'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
@@ -74,7 +81,9 @@ class Sender:
new_coolies_str = ""
for key in new_cookies:
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
print(new_coolies_str)
print("will publish to queue {}".format(new_coolies_str))
# upload the cookie to queue
self.cookiesPublisher.publish_body(new_coolies_str)
self.cookie_str = new_coolies_str
return True
else: