panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature/proxy_manager

Browse Source
This commit is contained in:
PAN Lei
2024-04-17 15:50:50 +02:00
parent 4450bb1afb 32c9d002b3
commit f6f1b44011
26 changed files with 1493 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/link_validator.py
+31 -14
View File
@@ -1,3 +1,4 @@
import json
import random
import threading
import time
@@ -10,8 +11,8 @@ from db.mongo_manager import MONGO_STORE_MANAGER
from models.LinkPojo import LinkPojo
from models.result_pojo import RequestResult
from proxy_manager.proxy_manager import ProxyManager
from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST
from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials, REQUEST_DATA_DE
from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST, TEST_QUEUE, SHARED_OBJECT
from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
from workers.proxies_constants import PROXY_LIST_FR
DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ces coordonnées"
@@ -35,7 +36,6 @@ class LinkValidator(threading.Thread):
self.filter_with_ip_country()
self.proxy_manager = proxy_manager
self.limit = limit
# self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
def set_up_connection(self):
self.connection = pika.BlockingConnection(
@@ -47,10 +47,13 @@ class LinkValidator(threading.Thread):
self.channel.basic_consume(queue=self.queue_to_listen, auto_ack=False, on_message_callback=callback)
self.channel.start_consuming()
def send_request(self, linkPojo: LinkPojo) -> RequestResult:
def send_request(self, linkPojo: LinkPojo, _received_dict=None) -> RequestResult:
_ua = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36'
if _received_dict is not None:
_ua = _received_dict['ua']
self.cookie.load(self.cookie_str)
headers = {
'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
'User-Agent': _ua,
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'max-age=0',
@@ -66,7 +69,9 @@ class LinkValidator(threading.Thread):
print("received cookie is " + str(self.cookie_str))
print("send request for link: " + linkPojo.url)
try:
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use,
print("will send request with ua {}".format(_ua))
print("will send request with cookie {}".format(self.cookie_str))
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
timeout=60)
print(response.status_code)
if response.status_code == 200:
@@ -85,7 +90,12 @@ class LinkValidator(threading.Thread):
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
print("will publish to queue {}".format(new_coolies_str))
# upload the cookie to queue
self.cookiesPublisher.publish_body(new_coolies_str)
if _received_dict is not None:
_received_dict['cookiesStr'] = new_coolies_str
print("body in json:{}".format(json.dumps(_received_dict)))
self.cookiesPublisher.publish_body(json.dumps(_received_dict))
else:
self.cookiesPublisher.publish_body(new_coolies_str)
self.cookie_str = new_coolies_str
return RequestResult.SUCCESS
elif INVALID in _content:
@@ -151,16 +161,24 @@ class LinkValidator(threading.Thread):
print("message count in queue is {}".format(_message_in_queue_count))
self.update_validate_list()
self.filter_with_ip_country()
self.cookie_str = body.decode("UTF-8")
_received_object = body.decode("UTF-8")
js_data = None
_received_dict = None
if "glrd" in _received_object:
_received_dict = json.loads(_received_object)
_received_cookies = _received_dict["cookiesStr"]
else:
_received_cookies = _received_object
self.cookie_str = _received_cookies
random.shuffle(self.link_to_validate_list)
if len(self.link_to_validate_list) > 0 and _message_in_queue_count > self.limit:
if len(self.link_to_validate_list) > 0 and _message_in_queue_count >= self.limit:
print("links number is {}".format(len(self.link_to_validate_list)))
can_continue = None
for con in self.link_to_validate_list:
# if not is_already_sent(con):
print(con.email)
self.proxy_to_use = random.choice(PROXY_LIST_FR)
can_continue = self.send_request(con)
can_continue = self.send_request(con, _received_dict)
if can_continue == RequestResult.BLOCKED:
print("cannot continue, blocked, then skip")
break
@@ -190,13 +208,12 @@ class LinkValidator(threading.Thread):
def validate_with_FR_ip(segment_position=1):
_queue_name = REQUEST_DATA_QUEUE
cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
_queue_name = TEST_QUEUE
cookiesPublisher = CookiesPublisher(queue_name=TEST_QUEUE)
cookiesPublisher.set_up_connection()
print("filter links with ip_country")
_proxy_manager = ProxyManager()
receiver = LinkValidator(cookiesPublisher=cookiesPublisher,
proxy_manager=_proxy_manager,
receiver = LinkValidator(cookiesPublisher=cookiesPublisher, proxy_manager=_proxy_manager,
queue_to_listen=_queue_name, ip_country="FR", segment_position=segment_position, limit=0)
print("will connect to queue")
receiver.set_up_connection()