108 lines
4.7 KiB
Python
108 lines
4.7 KiB
Python
import random
|
|
import threading
|
|
import time
|
|
from http.cookies import SimpleCookie
|
|
|
|
import pika
|
|
import requests
|
|
|
|
from db.mongo_manager import MONGO_STORE_MANAGER
|
|
from models.LinkPojo import LinkPojo
|
|
from queue_message.CookiesPublisher import CookiesPublisher
|
|
from queue_message.receiver import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
|
|
from workers.proxies_constants import PROXY_LIST
|
|
|
|
|
|
class LinkValidator(threading.Thread):
|
|
|
|
def __init__(self, link_to_validate_list: list, cookiesPublisher: CookiesPublisher):
|
|
super().__init__()
|
|
self.cookie = SimpleCookie()
|
|
self.cookiesPublisher = cookiesPublisher
|
|
self.link_to_validate_list = link_to_validate_list
|
|
# self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
|
|
|
|
def set_up_connection(self):
|
|
self.connection = pika.BlockingConnection(
|
|
pika.ConnectionParameters(host=QUEUE_HOST, port=5672, credentials=credentials))
|
|
self.channel = self.connection.channel()
|
|
|
|
def listen_to_queue(self, callback):
|
|
self.channel.basic_qos(prefetch_count=1)
|
|
self.channel.basic_consume(queue=REQUEST_DATA_QUEUE, auto_ack=False, on_message_callback=callback)
|
|
self.channel.start_consuming()
|
|
|
|
def send_request(self, linkPojo: LinkPojo) -> bool:
|
|
self.cookie.load(self.cookie_str)
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
|
|
'Accept': '*/*',
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
'Cache-Control': 'max-age=0',
|
|
'Referer': linkPojo.url,
|
|
'Cookie': self.cookie_str,
|
|
'Sec-Fetch-Mode': 'navigate',
|
|
'Host': 'rendezvousparis.hermes.com',
|
|
'Sec-Fetch-Site': 'same-origin',
|
|
'Sec-Fetch-Dest': 'document',
|
|
'Accept-Language': 'fr-FR,fr;q=0.6'}
|
|
proxy_to_use = random.choice(PROXY_LIST)
|
|
print(proxy_to_use)
|
|
print("received cookie is " + str(self.cookie_str))
|
|
try:
|
|
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=proxy_to_use,
|
|
timeout=15)
|
|
print(response.status_code)
|
|
if response.status_code == 200:
|
|
print(response.text)
|
|
print(response.url)
|
|
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo)
|
|
# set new cookies
|
|
_cookies_to_set = response.headers['set-cookie']
|
|
self.cookie.load(_cookies_to_set)
|
|
new_cookies = {k: v.value for k, v in self.cookie.items()}
|
|
new_coolies_str = ""
|
|
for key in new_cookies:
|
|
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
|
|
print("will publish to queue {}".format(new_coolies_str))
|
|
# upload the cookie to queue
|
|
self.cookiesPublisher.publish_body(new_coolies_str)
|
|
self.cookie_str = new_coolies_str
|
|
return True
|
|
else:
|
|
return False
|
|
except Exception as error:
|
|
print(error)
|
|
return False
|
|
|
|
def on_message(self, ch, method, properties, body):
|
|
print(f" [x] Received {body}")
|
|
self.link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
|
|
self.cookie_str = body.decode("UTF-8")
|
|
random.shuffle(self.link_to_validate_list)
|
|
if len(self.link_to_validate_list) > 0:
|
|
print("links number is {}".format(len(self.link_to_validate_list)))
|
|
for con in self.link_to_validate_list:
|
|
# if not is_already_sent(con):
|
|
print(con.email)
|
|
can_continue = self.send_request(con)
|
|
if not can_continue:
|
|
print("cannot continue, skip")
|
|
break
|
|
else:
|
|
time.sleep(random.randint(2, 5))
|
|
print("can continue, continue")
|
|
print("will ack")
|
|
ch.basic_ack(delivery_tag=method.delivery_tag)
|
|
else:
|
|
print("empty list")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
link_list = MONGO_STORE_MANAGER.get_links_to_validate()
|
|
cookiesPublisher = CookiesPublisher()
|
|
cookiesPublisher.set_up_connection()
|
|
receiver = LinkValidator(link_to_validate_list=link_list, cookiesPublisher=cookiesPublisher)
|
|
receiver.set_up_connection()
|
|
receiver.listen_to_queue(receiver.on_message)
|