panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support ip_country field

Browse Source
This commit is contained in:
Lei PAN
2024-01-23 23:33:55 +01:00
parent 97c2b9ce8c
commit aa26e17e99
12 changed files with 147 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/link_validator.py
+53 -24
View File
@@ -9,19 +9,22 @@ import requests
from db.mongo_manager import MONGO_STORE_MANAGER
from models.LinkPojo import LinkPojo
from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST
from queue_message.receiver import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
from workers.proxies_constants import PROXY_LIST
from queue_message.appointmentrequestsendor import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
from workers.proxies_constants import PROXY_LIST_FR
class LinkValidator(threading.Thread):
def __init__(self, link_to_validate_list: list,
cookiesPublisher: CookiesPublisher, queue_to_listen=REQUEST_DATA_QUEUE):
cookiesPublisher: CookiesPublisher, proxy_to_use, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR"):
super().__init__()
self.cookie = SimpleCookie()
self.cookiesPublisher = cookiesPublisher
self.link_to_validate_list = link_to_validate_list
self.queue_to_listen = queue_to_listen
self.ip_country = ip_country
self.filter_with_ip_country()
self.proxy_to_use = proxy_to_use
# self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
def set_up_connection(self):
@@ -48,29 +51,32 @@ class LinkValidator(threading.Thread):
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Dest': 'document',
'Accept-Language': 'fr-FR,fr;q=0.6'}
proxy_to_use = random.choice(PROXY_LIST)
print(proxy_to_use)
print(self.proxy_to_use)
print("received cookie is " + str(self.cookie_str))
try:
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=proxy_to_use,
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
timeout=15)
print(response.status_code)
if response.status_code == 200:
_content = response.text
print(response.text)
print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo)
# set new cookies
_cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set)
new_cookies = {k: v.value for k, v in self.cookie.items()}
new_coolies_str = ""
for key in new_cookies:
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
print("will publish to queue {}".format(new_coolies_str))
# upload the cookie to queue
self.cookiesPublisher.publish_body(new_coolies_str)
self.cookie_str = new_coolies_str
return True
if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content:
print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo)
# set new cookies
_cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set)
new_cookies = {k: v.value for k, v in self.cookie.items()}
new_coolies_str = ""
for key in new_cookies:
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
print("will publish to queue {}".format(new_coolies_str))
# upload the cookie to queue
self.cookiesPublisher.publish_body(new_coolies_str)
self.cookie_str = new_coolies_str
return True
else:
return False
else:
return False
except Exception as error:
@@ -80,6 +86,7 @@ class LinkValidator(threading.Thread):
def on_message(self, ch, method, properties, body):
print(f" [x] Received {body}")
self.link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
self.filter_with_ip_country()
self.cookie_str = body.decode("UTF-8")
random.shuffle(self.link_to_validate_list)
if len(self.link_to_validate_list) > 0:
@@ -101,12 +108,34 @@ class LinkValidator(threading.Thread):
time.sleep(60)
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
def filter_with_ip_country(self):
_link_list_to_click = []
for _link in self.link_to_validate_list:
if _link.ip_country == self.ip_country:
_link_list_to_click.append(_link)
self.link_to_validate_list = _link_list_to_click
if __name__ == '__main__':
def validate_with_FR_ip():
_queue_name = REQUEST_DATA_QUEUE_TEST
link_list = MONGO_STORE_MANAGER.get_links_to_validate()
cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_TEST)
cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
cookiesPublisher.set_up_connection()
receiver = LinkValidator(link_to_validate_list=link_list, cookiesPublisher=cookiesPublisher,
queue_to_listen=REQUEST_DATA_QUEUE_TEST)
print("filter links with ip_country")
_link_list_to_click = []
for _link in link_list:
if _link.ip_country == "FR":
_link_list_to_click.append(_link)
for _l in _link_list_to_click:
print(_l.ip_country)
_fr_proxy_to_use = random.choice(PROXY_LIST_FR)
receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
proxy_to_use=_fr_proxy_to_use,
queue_to_listen=_queue_name, ip_country="FR")
receiver.set_up_connection()
receiver.listen_to_queue(receiver.on_message)
pass
if __name__ == '__main__':
validate_with_FR_ip()