From aa26e17e9987644332cce32336651c25d645a0f0 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Tue, 23 Jan 2024 23:33:55 +0100 Subject: [PATCH] support ip_country field --- link_validator_executor.py | 2 +- models/LinkPojo.py | 8 +- queue_message/CookiesPublisher.py | 3 + ...eceiver.py => appointmentrequestsendor.py} | 12 ++- request_sender.py | 10 ++- workers/captcha_result_getter.py | 14 ++-- workers/cookie_generator.py | 29 +++---- workers/de_cookie_generator.py | 9 +++ workers/link_validator.py | 77 +++++++++++++------ workers/proxies_constants.py | 3 +- workers/sender.py | 7 +- workers/validate_links_DE.py | 33 ++++++++ 12 files changed, 147 insertions(+), 60 deletions(-) rename queue_message/{receiver.py => appointmentrequestsendor.py} (91%) create mode 100644 workers/de_cookie_generator.py create mode 100644 workers/validate_links_DE.py diff --git a/link_validator_executor.py b/link_validator_executor.py index f7c2153..c98b75e 100644 --- a/link_validator_executor.py +++ b/link_validator_executor.py @@ -8,7 +8,7 @@ from db.mongo_manager import MONGO_STORE_MANAGER from excel_reader import read_contacts from models.contact_pojo import ContactPojo from queue_message.link_validator_receiver import LinkValidatorReceiver -from queue_message.receiver import Receiver, filter_contacts +from queue_message.appointmentrequestsendor import AppointmentRequestSendor, filter_contacts from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER from workers.sender import Sender diff --git a/models/LinkPojo.py b/models/LinkPojo.py index 9eb75ec..c58b67e 100644 --- a/models/LinkPojo.py +++ b/models/LinkPojo.py @@ -1,13 +1,17 @@ class LinkPojo(): - def __init__(self, url, email, updated_at): + def __init__(self, url, email, updated_at, ip_country): self.url = url self.email = email self.updated_at = updated_at + self.ip_country = ip_country @staticmethod def from_firestore_dict(source): updated_at = source['updated_at'] email = source['email'] url = source['url'] - result = LinkPojo(email=email, url=url, updated_at=updated_at) + ip_country = "FR" + if source.get('ip_country'): + ip_country = source['ip_country'] + result = LinkPojo(email=email, url=url, updated_at=updated_at, ip_country=ip_country) return result diff --git a/queue_message/CookiesPublisher.py b/queue_message/CookiesPublisher.py index 883ef2f..d0eed07 100644 --- a/queue_message/CookiesPublisher.py +++ b/queue_message/CookiesPublisher.py @@ -2,7 +2,9 @@ import pika QUEUE_HOST = "appointment.lpaconsulting.fr" REQUEST_DATA_QUEUE = 'REQUEST_DATA' +# REQUEST_DATA_QUEUE_TEST = 'REQUEST_DATA_TEST' REQUEST_DATA_QUEUE_TEST = 'REQUEST_DATA_TEST' +REQUEST_DATA_QUEUE_DE = 'REQUEST_DATA_DE' credentials = pika.PlainCredentials('appointment', 'ZyuhJZ2xEYWhElhpJjy7YEpZGZwNYJz2fHIu') @@ -21,6 +23,7 @@ class CookiesPublisher: self.queue_method = self.channel.queue_declare(queue=self.to_queue, durable=True) def publish_body(self, body: str): + print("will push to queue {}".format(self.to_queue)) self.channel.basic_publish(exchange='', routing_key=self.to_queue, body=body) def message_count(self): diff --git a/queue_message/receiver.py b/queue_message/appointmentrequestsendor.py similarity index 91% rename from queue_message/receiver.py rename to queue_message/appointmentrequestsendor.py index 6084daa..3683062 100644 --- a/queue_message/receiver.py +++ b/queue_message/appointmentrequestsendor.py @@ -11,6 +11,7 @@ from models.contact_pojo import ContactPojo from queue_message.CookiesPublisher import CookiesPublisher from utiles import is_time_between from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER +from workers.proxies_constants import PROXY_LIST_DE from workers.sender import Sender QUEUE_HOST = "appointment.lpaconsulting.fr" @@ -72,14 +73,15 @@ def is_open(): return is_time_between(datetime.time(10, 30), datetime.time(19, 00)) -class Receiver(threading.Thread): - def __init__(self, sub_contact_list: list, cookiesPublisher: CookiesPublisher): +class AppointmentRequestSendor(threading.Thread): + def __init__(self, sub_contact_list: list, cookiesPublisher: CookiesPublisher, queue_name=REQUEST_DATA_QUEUE): super().__init__() self.connection = None self.cookiesPublisher = cookiesPublisher self.channel = None self.valid_csrf = None self.contact_list = sub_contact_list + self.queue_name = queue_name def set_up_connection(self): self.connection = pika.BlockingConnection( @@ -87,14 +89,16 @@ class Receiver(threading.Thread): self.channel = self.connection.channel() def listen_to_queue(self, callback): + print("listen to queue {}".format(self.queue_name)) self.channel.basic_qos(prefetch_count=1) - self.channel.basic_consume(queue=REQUEST_DATA_QUEUE, auto_ack=False, on_message_callback=callback) + self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback) self.channel.start_consuming() def on_message(self, ch, method, properties, body): print(f" [x] Received {body}") print("message count in queue is {}".format(self.cookiesPublisher.message_count())) - sender = Sender(body.decode("UTF-8"), cookiesPublisher=self.cookiesPublisher) + sender = Sender(body.decode("UTF-8"), cookiesPublisher=self.cookiesPublisher, + proxy_to_use=random.choice(PROXY_LIST_DE)) self.contact_list = filter_contacts(self.contact_list) # remove already booked contacts random.shuffle(self.contact_list) diff --git a/request_sender.py b/request_sender.py index 2fb4983..78e2a8f 100644 --- a/request_sender.py +++ b/request_sender.py @@ -7,7 +7,7 @@ from db.mongo_manager import MONGO_STORE_MANAGER from excel_reader import read_contacts from models.contact_pojo import ContactPojo from queue_message.CookiesPublisher import CookiesPublisher -from queue_message.receiver import Receiver +from queue_message.appointmentrequestsendor import AppointmentRequestSendor from utiles import is_time_between from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER from workers.sender import Sender @@ -71,8 +71,10 @@ def is_open(): if __name__ == '__main__': - cookiesPublisher = CookiesPublisher() + REQUEST_DATA_QUEUE_DE = 'REQUEST_DATA_DE' + cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_DE) cookiesPublisher.set_up_connection() - contact_list = read_contacts('/Users/panlei/Desktop/real_name_contacts_77_14_01_2024.xlsx') - receiver = Receiver(sub_contact_list=contact_list, cookiesPublisher=cookiesPublisher) + contact_list = read_contacts('/Users/lpan/Desktop/08_01_24_valid_de.xlsx') + receiver = AppointmentRequestSendor(sub_contact_list=contact_list, queue_name=REQUEST_DATA_QUEUE_DE, + cookiesPublisher=cookiesPublisher) receiver.run() diff --git a/workers/captcha_result_getter.py b/workers/captcha_result_getter.py index 365a5a1..900b651 100644 --- a/workers/captcha_result_getter.py +++ b/workers/captcha_result_getter.py @@ -5,7 +5,7 @@ from time import time from typing import Union import requests -from workers.proxies_constants import PROXY_LIST +from workers.proxies_constants import PROXY_LIST_FR API_KEY = "d66aaf490d8aa424a5175e1fbd1aadea" @@ -21,7 +21,7 @@ class CaptchaResultGetter: self.cookie_str = 'datadome=5Nq~NEP_qQSHC0g_lZNnZmEv36J8gVV~rpZ329xmCkTq2~H3meIoXr4h_b988qB2XW5Te7iEGsvq8BzA5KeFupyrZFh4kgrDyl8hT2UymSByKHzAcDaNIBPDsRu2g_KG; Max-Age=31536000; Domain=.hermes.com; Path=/; Secure; SameSite=None' pass - def get_csrf(self, cookie: str = None) -> Union[str, None]: + def get_csrf(self, proxy_to_use, cookie: str = None) -> Union[str, None]: if cookie is not None: headers = {'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36', @@ -39,8 +39,8 @@ class CaptchaResultGetter: 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Dest': 'document', 'Accept-Language': 'fr-FR,fr;q=0.6'} - proxy_to_use = random.choice(PROXY_LIST) print("received cookie is " + str(cookie)) + print(proxy_to_use) try: response = requests.get(url=HERMES_REGISTER, headers=headers, verify=False, proxies=proxy_to_use, timeout=15) @@ -62,7 +62,7 @@ class CaptchaResultGetter: return result_list[-1] return None - def get_valid_cookie(self, old_valid_cookie: str, dvm=4, hc=6) -> Union[str, None]: + def get_valid_cookie(self, proxy_to_use, old_valid_cookie: str, dvm=4, hc=6) -> Union[str, None]: headers = {'content-Type': 'application/x-www-form-urlencoded', 'user-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36', 'accept': '*/*', @@ -75,8 +75,8 @@ class CaptchaResultGetter: 'sec-ch-ua-platform': 'Android', 'accept-encoding': 'gzip, deflate, br', 'accept-language': 'fr-FR,fr;q=0.6'} - proxy_to_use = random.choice(PROXY_LIST) print("send request to get new cookie") + print(proxy_to_use) print(headers) try: # tag_pu = 10 * Math.random() @@ -97,7 +97,7 @@ class CaptchaResultGetter: print(error) return None - def get_valid_ch_cookie(self, old_valid_cookie: str = None, dvm=4, hc=6) -> Union[str, None]: + def get_valid_ch_cookie(self, proxy_to_use, old_valid_cookie: str = None, dvm=4, hc=6) -> Union[str, None]: headers = {'content-Type': 'application/x-www-form-urlencoded', 'user-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36', 'accept': '*/*', @@ -109,7 +109,7 @@ class CaptchaResultGetter: 'sec-fetch-dest': 'empty', 'accept-encoding': 'gzip, deflate, br', 'accept-language': 'fr-FR,fr;q=0.6'} - proxy_to_use = random.choice(PROXY_LIST) + print(proxy_to_use) print("send request to get new cookie") print(headers) try: diff --git a/workers/cookie_generator.py b/workers/cookie_generator.py index 4246a26..134ac71 100644 --- a/workers/cookie_generator.py +++ b/workers/cookie_generator.py @@ -13,12 +13,13 @@ DVM_LIST = [2, 3, 4, 6] class CookiesGenerator(threading.Thread): - def __init__(self, cookiesPublisher: CookiesPublisher): + def __init__(self, proxy_to_use_list: list, cookiesPublisher: CookiesPublisher): super().__init__() self.connection = None self.cookiesPublisher = cookiesPublisher self.channel = None self.valid_csrf = None + self.proxy_to_use_list = proxy_to_use_list def set_up_connection(self): self.connection = pika.BlockingConnection( @@ -40,22 +41,25 @@ class CookiesGenerator(threading.Thread): # generate 10 cookies to new_queue captchaResultGetter = CaptchaResultGetter() - _cookies_count = random.randint(2, 10) + _cookies_count = random.randint(8, 10) _received_cookies = body.decode("UTF-8") dvm = random.choice(DVM_LIST) hc = random.choice(DVM_LIST) if _message_count < 10: + _proxy_to_use = random.choice(self.proxy_to_use_list) for i in range(1, _cookies_count): - new_cookie = captchaResultGetter.get_valid_ch_cookie(dvm=dvm, hc=hc) + # new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, dvm=dvm, hc=hc) _app_sig = get_app_and_app_sig(_received_cookies) - if new_cookie is not None and _app_sig is not None: - new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie - new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace( - "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "") - print("new_cookie is " + new_cookie) - _received_cookies = new_cookie - self.cookiesPublisher.publish_body(new_cookie) - new_cookie = captchaResultGetter.get_valid_cookie(new_cookie, dvm=dvm, hc=hc) + # if new_cookie is not None and _app_sig is not None: + # new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie + # new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace( + # "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "") + # print("new_cookie is " + new_cookie) + # _received_cookies = new_cookie + # self.cookiesPublisher.publish_body(new_cookie) + new_cookie = captchaResultGetter.get_valid_cookie(proxy_to_use=_proxy_to_use, + old_valid_cookie=_received_cookies, dvm=dvm, + hc=hc) if new_cookie is not None and _app_sig is not None: new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace( @@ -86,9 +90,6 @@ def get_app_and_app_sig(cookies: str): if __name__ == '__main__': - # _list = get_app_and_app_sig( - # "app.sig=axk5aJ2c83dgV56DswNhw98y4SY;datadome=2dTtQDIHCadAbzh8JWv7MFKGH8~fkOoGlA1O3J7llv_sdYOSZYbaKruHPBbARchV1nnrstsaMX5E_XwbJmGiXzt5qZXOL0UCAV9TTHqBRh10JvO~GCGJv2JVO~6RnrcU;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiZEE2bEdnN04yd2t0eWVmZmVjVkxMY1dTIiwiYXBwb2ludG1lbnRfY29kZSI6Ik5DTlVaQiIsImJsb2NrX3JlZ2lzdHJhdGlvbiI6ZmFsc2V9;policy=accepted;lang=fr;") - # print(_list) cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_TEST) cookiesPublisher.set_up_connection() cookieGenerator = CookiesGenerator(cookiesPublisher) diff --git a/workers/de_cookie_generator.py b/workers/de_cookie_generator.py new file mode 100644 index 0000000..9acdaa0 --- /dev/null +++ b/workers/de_cookie_generator.py @@ -0,0 +1,9 @@ +from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher +from workers.cookie_generator import CookiesGenerator +from workers.proxies_constants import PROXY_LIST_DE + +if __name__ == '__main__': + cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_DE) + cookiesPublisher.set_up_connection() + cookieGenerator = CookiesGenerator(cookiesPublisher=cookiesPublisher, proxy_to_use_list=PROXY_LIST_DE) + cookieGenerator.run() diff --git a/workers/link_validator.py b/workers/link_validator.py index a5da2df..c675282 100644 --- a/workers/link_validator.py +++ b/workers/link_validator.py @@ -9,19 +9,22 @@ import requests from db.mongo_manager import MONGO_STORE_MANAGER from models.LinkPojo import LinkPojo from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST -from queue_message.receiver import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials -from workers.proxies_constants import PROXY_LIST +from queue_message.appointmentrequestsendor import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials +from workers.proxies_constants import PROXY_LIST_FR class LinkValidator(threading.Thread): def __init__(self, link_to_validate_list: list, - cookiesPublisher: CookiesPublisher, queue_to_listen=REQUEST_DATA_QUEUE): + cookiesPublisher: CookiesPublisher, proxy_to_use, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR"): super().__init__() self.cookie = SimpleCookie() self.cookiesPublisher = cookiesPublisher self.link_to_validate_list = link_to_validate_list self.queue_to_listen = queue_to_listen + self.ip_country = ip_country + self.filter_with_ip_country() + self.proxy_to_use = proxy_to_use # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;' def set_up_connection(self): @@ -48,29 +51,32 @@ class LinkValidator(threading.Thread): 'Sec-Fetch-Site': 'same-origin', 'Sec-Fetch-Dest': 'document', 'Accept-Language': 'fr-FR,fr;q=0.6'} - proxy_to_use = random.choice(PROXY_LIST) - print(proxy_to_use) + print(self.proxy_to_use) print("received cookie is " + str(self.cookie_str)) try: - response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=proxy_to_use, + response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use, timeout=15) print(response.status_code) if response.status_code == 200: + _content = response.text print(response.text) - print(response.url) - MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo) - # set new cookies - _cookies_to_set = response.headers['set-cookie'] - self.cookie.load(_cookies_to_set) - new_cookies = {k: v.value for k, v in self.cookie.items()} - new_coolies_str = "" - for key in new_cookies: - new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";" - print("will publish to queue {}".format(new_coolies_str)) - # upload the cookie to queue - self.cookiesPublisher.publish_body(new_coolies_str) - self.cookie_str = new_coolies_str - return True + if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content: + print(response.url) + MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo) + # set new cookies + _cookies_to_set = response.headers['set-cookie'] + self.cookie.load(_cookies_to_set) + new_cookies = {k: v.value for k, v in self.cookie.items()} + new_coolies_str = "" + for key in new_cookies: + new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";" + print("will publish to queue {}".format(new_coolies_str)) + # upload the cookie to queue + self.cookiesPublisher.publish_body(new_coolies_str) + self.cookie_str = new_coolies_str + return True + else: + return False else: return False except Exception as error: @@ -80,6 +86,7 @@ class LinkValidator(threading.Thread): def on_message(self, ch, method, properties, body): print(f" [x] Received {body}") self.link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate() + self.filter_with_ip_country() self.cookie_str = body.decode("UTF-8") random.shuffle(self.link_to_validate_list) if len(self.link_to_validate_list) > 0: @@ -101,12 +108,34 @@ class LinkValidator(threading.Thread): time.sleep(60) ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True) + def filter_with_ip_country(self): + _link_list_to_click = [] + for _link in self.link_to_validate_list: + if _link.ip_country == self.ip_country: + _link_list_to_click.append(_link) + self.link_to_validate_list = _link_list_to_click -if __name__ == '__main__': + +def validate_with_FR_ip(): + _queue_name = REQUEST_DATA_QUEUE_TEST link_list = MONGO_STORE_MANAGER.get_links_to_validate() - cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_TEST) + cookiesPublisher = CookiesPublisher(queue_name=_queue_name) cookiesPublisher.set_up_connection() - receiver = LinkValidator(link_to_validate_list=link_list, cookiesPublisher=cookiesPublisher, - queue_to_listen=REQUEST_DATA_QUEUE_TEST) + print("filter links with ip_country") + _link_list_to_click = [] + for _link in link_list: + if _link.ip_country == "FR": + _link_list_to_click.append(_link) + for _l in _link_list_to_click: + print(_l.ip_country) + _fr_proxy_to_use = random.choice(PROXY_LIST_FR) + receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher, + proxy_to_use=_fr_proxy_to_use, + queue_to_listen=_queue_name, ip_country="FR") receiver.set_up_connection() receiver.listen_to_queue(receiver.on_message) + pass + + +if __name__ == '__main__': + validate_with_FR_ip() diff --git a/workers/proxies_constants.py b/workers/proxies_constants.py index 946fd9e..5e4f81c 100644 --- a/workers/proxies_constants.py +++ b/workers/proxies_constants.py @@ -31,4 +31,5 @@ DE_PROXY_RES = { # FR_PROXY_MOBILE # FR_PROXY_RES # PROXY_LIST = [FR_PROXY_MOBILE, FR_PROXY_RES, DE_PROXY_RES, DE_PROXY_MOBILE, ES_PROXY_MOBILE, IT_PROXY_MOBILE] -PROXY_LIST = [FR_PROXY_MOBILE, FR_PROXY_RES] +PROXY_LIST_FR = [FR_PROXY_MOBILE, FR_PROXY_RES] +PROXY_LIST_DE = [DE_PROXY_RES, DE_PROXY_MOBILE] diff --git a/workers/sender.py b/workers/sender.py index 0e10a9c..e242695 100644 --- a/workers/sender.py +++ b/workers/sender.py @@ -8,18 +8,19 @@ import requests from db.mongo_manager import MONGO_STORE_MANAGER from models.ReserveResultPojo import ReserveResultPojo, PublishType from queue_message.CookiesPublisher import CookiesPublisher -from workers.proxies_constants import PROXY_LIST +from workers.proxies_constants import PROXY_LIST_FR class Sender: - def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher): + def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher, proxy_to_use): self.store_type = "random" self.cookie = SimpleCookie() self.cookiesPublisher = cookiesPublisher # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;' self.cookie_str = cookie_str self._csrf = None + self.proxy_to_use = proxy_to_use self.cookie.load(self.cookie_str) def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str): @@ -64,7 +65,7 @@ class Sender: 'passport_id': contact.passport, 'processing': 'on', 'cgu': 'on'} print(data) try: - proxy_to_use = random.choice(PROXY_LIST) + proxy_to_use = self.proxy_to_use # proxy_to_use = PROXY_LIST[0] print(proxy_to_use) response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data, diff --git a/workers/validate_links_DE.py b/workers/validate_links_DE.py new file mode 100644 index 0000000..ea5a70f --- /dev/null +++ b/workers/validate_links_DE.py @@ -0,0 +1,33 @@ +import random + +from db.mongo_manager import MONGO_STORE_MANAGER +from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher +from workers.link_validator import LinkValidator +from workers.proxies_constants import PROXY_LIST_DE + + +def validate_with_DE_ip(): + _queue_name = REQUEST_DATA_QUEUE_DE + link_list = MONGO_STORE_MANAGER.get_links_to_validate() + cookiesPublisher = CookiesPublisher(queue_name=_queue_name) + cookiesPublisher.set_up_connection() + print("filter links with ip_country") + _link_list_to_click = [] + for _link in link_list: + if _link.ip_country == "DE": + _link_list_to_click.append(_link) + for _l in _link_list_to_click: + print(_l.ip_country) + # else: + # print(_link.ip_country) + _de_proxy_to_use = random.choice(PROXY_LIST_DE) + receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher, + proxy_to_use=_de_proxy_to_use, + queue_to_listen=_queue_name, ip_country="DE") + receiver.set_up_connection() + receiver.listen_to_queue(receiver.on_message) + pass + + +if __name__ == '__main__': + validate_with_DE_ip()