panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support invalid url

Browse Source
This commit is contained in:
PAN Lei
2024-04-06 13:50:58 +02:00
parent 88ff6d539b
commit 5d5d7afed5
4 changed files with 33 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/link_validator.py
+22 -3
View File
@@ -11,10 +11,11 @@ from models.LinkPojo import LinkPojo
from models.result_pojo import RequestResult
from proxy_manager.proxy_manager import ProxyManager
from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST
from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials, REQUEST_DATA_DE
from workers.proxies_constants import PROXY_LIST_FR
DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ces coordonnées"
INVALID = "Depuis plus de 130 ans,"
class LinkValidator(threading.Thread):
@@ -63,6 +64,7 @@ class LinkValidator(threading.Thread):
_proxy_to_use = random.choice(self.proxy_manager.get_link_validate_proxy(self.link_to_validate_list))
print(_proxy_to_use)
print("received cookie is " + str(self.cookie_str))
print("send request for link: " + linkPojo.url)
try:
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use,
timeout=30)
@@ -85,6 +87,22 @@ class LinkValidator(threading.Thread):
self.cookiesPublisher.publish_body(new_coolies_str)
self.cookie_str = new_coolies_str
return RequestResult.SUCCESS
elif INVALID in _content:
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True)
# set new cookies
_cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set)
new_cookies = {k: v.value for k, v in self.cookie.items()}
new_coolies_str = ""
for key in new_cookies:
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
print("will publish to queue {}".format(new_coolies_str))
# upload the cookie to queue
self.cookiesPublisher.publish_body(new_coolies_str)
self.cookie_str = new_coolies_str
return RequestResult.SUCCESS
elif DOUBLE_MESSAGE in _content:
print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True)
@@ -171,14 +189,15 @@ class LinkValidator(threading.Thread):
def validate_with_FR_ip(segment_position=1):
_queue_name = REQUEST_DATA_QUEUE
# _queue_name = REQUEST_DATA_QUEUE
_queue_name = REQUEST_DATA_DE
cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
cookiesPublisher.set_up_connection()
print("filter links with ip_country")
_proxy_manager = ProxyManager()
receiver = LinkValidator(cookiesPublisher=cookiesPublisher,
proxy_manager=_proxy_manager,
queue_to_listen=_queue_name, ip_country="FR", segment_position=segment_position, limit=50)
queue_to_listen=_queue_name, ip_country="FR", segment_position=segment_position, limit=0)
print("will connect to queue")
receiver.set_up_connection()
receiver.listen_to_queue(receiver.on_message)