diff --git a/link_validator_executor.py b/link_validator_executor.py index 0aee0eb..318f598 100644 --- a/link_validator_executor.py +++ b/link_validator_executor.py @@ -2,4 +2,4 @@ from workers.link_validator import validate_with_FR_ip if __name__ == '__main__': # link_list = MONGO_STORE_MANAGER.get_links_to_validate() - validate_with_FR_ip() \ No newline at end of file + validate_with_FR_ip(segment_position=2) diff --git a/proxy_manager/proxy_manager.py b/proxy_manager/proxy_manager.py index 9eb8c81..2c514cf 100644 --- a/proxy_manager/proxy_manager.py +++ b/proxy_manager/proxy_manager.py @@ -8,10 +8,20 @@ FR_PROXY_RES_OXY = { 'https': 'http://customer-rendezvous-cc-FR:Rdv202220212023@pr.oxylabs.io:7777' } +FR_PROXY_MOB_OXY = { + 'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@pr.oxylabs.io:7777', + 'https': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@pr.oxylabs.io:7777' +} + +FR_PROXY_ASOCK_RES_2 = { + 'http': 'http://10488120-res-country-FR:k94fsbn9a@217.23.6.161:11287', + 'https': 'http://10488120-res-country-FR:k94fsbn9a@217.23.6.161:11287' +} + class ProxyManager: def get_link_validate_proxy(self, links_to_validate: list) -> list: if len(links_to_validate) > 15: - return [FR_PROXY_RES_OXY] + return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY] else: - return [FR_ASOCKS_MOBILE_PROXY] + return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY] diff --git a/queue_message/link_validator_receiver.py b/queue_message/link_validator_receiver.py index 66abdef..02b588d 100644 --- a/queue_message/link_validator_receiver.py +++ b/queue_message/link_validator_receiver.py @@ -8,8 +8,6 @@ import pika from db.mongo_manager import MONGO_STORE_MANAGER from models.contact_pojo import ContactPojo from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER -from workers.link_validator import LinkValidator -from workers.sender import Sender QUEUE_HOST = "appointment.lpaconsulting.fr" REQUEST_DATA_QUEUE = 'REQUEST_DATA' diff --git a/workers/link_validator.py b/workers/link_validator.py index a2095f5..bb664e2 100644 --- a/workers/link_validator.py +++ b/workers/link_validator.py @@ -19,13 +19,16 @@ DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ce class LinkValidator(threading.Thread): - def __init__(self, link_to_validate_list: list, - cookiesPublisher: CookiesPublisher, proxy_manager:ProxyManager, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR", + def __init__(self, cookiesPublisher: CookiesPublisher, proxy_manager: ProxyManager, + queue_to_listen=REQUEST_DATA_QUEUE, + ip_country="FR", segment_position=1, limit=40): super().__init__() + self.link_to_validate_list = [] self.cookie = SimpleCookie() self.cookiesPublisher = cookiesPublisher - self.link_to_validate_list = link_to_validate_list + self.segment_position = segment_position + self.update_validate_list() self.queue_to_listen = queue_to_listen self.ip_country = ip_country self.filter_with_ip_country() @@ -62,7 +65,7 @@ class LinkValidator(threading.Thread): print("received cookie is " + str(self.cookie_str)) try: response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use, - timeout=10) + timeout=30) print(response.status_code) if response.status_code == 200: _content = response.text @@ -99,17 +102,35 @@ class LinkValidator(threading.Thread): return RequestResult.SUCCESS else: return RequestResult.UNKNOWN + elif response.status_code == 502: + return RequestResult.PROXY_ERROR else: return RequestResult.BLOCKED except Exception as error: print(error) return RequestResult.PROXY_ERROR + def update_validate_list(self): + # for the moment, max segment is 2 + _all_list = MONGO_STORE_MANAGER.get_links_to_validate() + list_length = len(_all_list) + middle = int(len(_all_list) / 2) + if self.segment_position == 1: + if list_length > 1: + self.link_to_validate_list = _all_list[0:middle] + else: + self.link_to_validate_list = _all_list + else: + if list_length > 1: + self.link_to_validate_list = _all_list[middle:] + else: + self.link_to_validate_list = [] + def on_message(self, ch, method, properties, body): print(f" [x] Received {body}") _message_in_queue_count = self.cookiesPublisher.message_count() print("message count in queue is {}".format(_message_in_queue_count)) - self.link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate() + self.update_validate_list() self.filter_with_ip_country() self.cookie_str = body.decode("UTF-8") random.shuffle(self.link_to_validate_list) @@ -141,24 +162,15 @@ class LinkValidator(threading.Thread): self.link_to_validate_list = _link_list_to_click -def validate_with_FR_ip(): +def validate_with_FR_ip(segment_position=1): _queue_name = REQUEST_DATA_QUEUE - link_list = MONGO_STORE_MANAGER.get_links_to_validate() cookiesPublisher = CookiesPublisher(queue_name=_queue_name) cookiesPublisher.set_up_connection() print("filter links with ip_country") - _link_list_to_click = [] - for _link in link_list: - if _link.ip_country == "FR": - _link_list_to_click.append(_link) - for _l in _link_list_to_click: - print(_l.ip_country) _proxy_manager = ProxyManager() - # _fr_proxy_to_use = random.choice(PROXY_LIST_FR) - # random.shuffle(_link_list_to_click) - receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher, + receiver = LinkValidator(cookiesPublisher=cookiesPublisher, proxy_manager=_proxy_manager, - queue_to_listen=_queue_name, ip_country="FR", limit=50) + queue_to_listen=_queue_name, ip_country="FR", segment_position=segment_position, limit=50) print("will connect to queue") receiver.set_up_connection() receiver.listen_to_queue(receiver.on_message) diff --git a/workers/validate_links_DE.py b/workers/validate_links_DE.py deleted file mode 100644 index 2f2578d..0000000 --- a/workers/validate_links_DE.py +++ /dev/null @@ -1,33 +0,0 @@ -import random - -from db.mongo_manager import MONGO_STORE_MANAGER -from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher -from workers.link_validator import LinkValidator -from workers.proxies_constants import PROXY_LIST_DE - - -def validate_with_DE_ip(): - _queue_name = REQUEST_DATA_QUEUE_DE - link_list = MONGO_STORE_MANAGER.get_links_to_validate() - cookiesPublisher = CookiesPublisher(queue_name=_queue_name) - cookiesPublisher.set_up_connection() - print("filter links with ip_country") - _link_list_to_click = [] - for _link in link_list: - if _link.ip_country == "DE": - _link_list_to_click.append(_link) - for _l in _link_list_to_click: - print(_l.ip_country) - # else: - # print(_link.ip_country) - _de_proxy_to_use = random.choice(PROXY_LIST_DE) - receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher, - proxy_to_use=_de_proxy_to_use, - queue_to_listen=_queue_name, ip_country="DE", limit=0) - receiver.set_up_connection() - receiver.listen_to_queue(receiver.on_message) - pass - - -if __name__ == '__main__': - validate_with_DE_ip()