panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rename the method

Browse Source
This commit is contained in:
Lei PAN
2024-03-30 16:12:09 +01:00
parent c1cb4b4d2f
commit 75091c0a6b
2 changed files with 38 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/link_validator.py
+29 -12
View File
@@ -1,3 +1,4 @@
import json
import random
import threading
import time
@@ -9,7 +10,7 @@ import requests
from db.mongo_manager import MONGO_STORE_MANAGER
from models.LinkPojo import LinkPojo
from models.result_pojo import RequestResult
from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST
from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST, TEST_QUEUE, SHARED_OBJECT
from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
from workers.proxies_constants import PROXY_LIST_FR
@@ -30,7 +31,6 @@ class LinkValidator(threading.Thread):
self.filter_with_ip_country()
self.proxy_to_use = proxy_to_use
self.limit = limit
# self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
def set_up_connection(self):
self.connection = pika.BlockingConnection(
@@ -42,10 +42,13 @@ class LinkValidator(threading.Thread):
self.channel.basic_consume(queue=self.queue_to_listen, auto_ack=False, on_message_callback=callback)
self.channel.start_consuming()
def send_request(self, linkPojo: LinkPojo) -> RequestResult:
def send_request(self, linkPojo: LinkPojo, _received_dict = None) -> RequestResult:
_ua = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36'
if _received_dict is not None:
_ua = _received_dict['ua']
self.cookie.load(self.cookie_str)
headers = {
'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
'User-Agent': _ua,
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'max-age=0',
@@ -59,6 +62,8 @@ class LinkValidator(threading.Thread):
print(self.proxy_to_use)
print("received cookie is " + str(self.cookie_str))
try:
print("will send request with ua {}".format(_ua))
print("will send request with cookie {}".format(self.cookie_str))
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
timeout=15)
print(response.status_code)
@@ -77,7 +82,12 @@ class LinkValidator(threading.Thread):
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
print("will publish to queue {}".format(new_coolies_str))
# upload the cookie to queue
self.cookiesPublisher.publish_body(new_coolies_str)
if _received_dict is not None:
_received_dict['cookiesStr'] = new_coolies_str
print("body in json:{}".format(json.dumps(_received_dict)))
self.cookiesPublisher.publish_body(json.dumps(_received_dict))
else:
self.cookiesPublisher.publish_body(new_coolies_str)
self.cookie_str = new_coolies_str
return RequestResult.SUCCESS
elif DOUBLE_MESSAGE in _content:
@@ -109,15 +119,23 @@ class LinkValidator(threading.Thread):
print("message count in queue is {}".format(_message_in_queue_count))
self.link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
self.filter_with_ip_country()
self.cookie_str = body.decode("UTF-8")
_received_object = body.decode("UTF-8")
js_data = None
_received_dict = None
if "glrd" in _received_object:
_received_dict = json.loads(_received_object)
_received_cookies = _received_dict["cookiesStr"]
else:
_received_cookies = _received_object
self.cookie_str = _received_cookies
random.shuffle(self.link_to_validate_list)
if len(self.link_to_validate_list) > 0 and _message_in_queue_count > self.limit:
if len(self.link_to_validate_list) > 0 and _message_in_queue_count >= self.limit:
print("links number is {}".format(len(self.link_to_validate_list)))
for con in self.link_to_validate_list:
# if not is_already_sent(con):
print(con.email)
self.proxy_to_use = random.choice(PROXY_LIST_FR)
can_continue = self.send_request(con)
can_continue = self.send_request(con, _received_dict)
if can_continue == RequestResult.BLOCKED:
print("cannot continue, blocked, then skip")
break
@@ -140,9 +158,9 @@ class LinkValidator(threading.Thread):
def validate_with_FR_ip():
_queue_name = REQUEST_DATA_QUEUE
_queue_name = TEST_QUEUE
link_list = MONGO_STORE_MANAGER.get_links_to_validate()
cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
cookiesPublisher = CookiesPublisher(queue_name=SHARED_OBJECT)
cookiesPublisher.set_up_connection()
print("filter links with ip_country")
_link_list_to_click = []
@@ -152,10 +170,9 @@ def validate_with_FR_ip():
for _l in _link_list_to_click:
print(_l.ip_country)
_fr_proxy_to_use = random.choice(PROXY_LIST_FR)
# random.shuffle(_link_list_to_click)
receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
proxy_to_use=_fr_proxy_to_use,
queue_to_listen=_queue_name, ip_country="FR", limit=50)
queue_to_listen=_queue_name, ip_country="FR", limit=0)
print("will connect to queue")
receiver.set_up_connection()
receiver.listen_to_queue(receiver.on_message)