panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support 502 error

Browse Source
This commit is contained in:
Lei PAN
2024-04-06 11:47:33 +02:00
parent 302294fe06
commit 10b184f3da
5 changed files with 42 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
link_validator_executor.py
+1 -1
View File
@@ -2,4 +2,4 @@ from workers.link_validator import validate_with_FR_ip
if __name__ == '__main__':
# link_list = MONGO_STORE_MANAGER.get_links_to_validate()
validate_with_FR_ip()
validate_with_FR_ip(segment_position=2)
proxy_manager/proxy_manager.py
+12 -2
View File
@@ -8,10 +8,20 @@ FR_PROXY_RES_OXY = {
'https': 'http://customer-rendezvous-cc-FR:Rdv202220212023@pr.oxylabs.io:7777'
}
FR_PROXY_MOB_OXY = {
'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@pr.oxylabs.io:7777',
'https': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@pr.oxylabs.io:7777'
}
FR_PROXY_ASOCK_RES_2 = {
'http': 'http://10488120-res-country-FR:k94fsbn9a@217.23.6.161:11287',
'https': 'http://10488120-res-country-FR:k94fsbn9a@217.23.6.161:11287'
}
class ProxyManager:
def get_link_validate_proxy(self, links_to_validate: list) -> list:
if len(links_to_validate) > 15:
return [FR_PROXY_RES_OXY]
return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY]
else:
return [FR_ASOCKS_MOBILE_PROXY]
return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY]
queue_message/link_validator_receiver.py
-2
View File
@@ -8,8 +8,6 @@ import pika
from db.mongo_manager import MONGO_STORE_MANAGER
from models.contact_pojo import ContactPojo
from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
from workers.link_validator import LinkValidator
from workers.sender import Sender
QUEUE_HOST = "appointment.lpaconsulting.fr"
REQUEST_DATA_QUEUE = 'REQUEST_DATA'
workers/link_validator.py
+29 -17
View File
@@ -19,13 +19,16 @@ DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ce
class LinkValidator(threading.Thread):
def __init__(self, link_to_validate_list: list,
cookiesPublisher: CookiesPublisher, proxy_manager:ProxyManager, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR",
def __init__(self, cookiesPublisher: CookiesPublisher, proxy_manager: ProxyManager,
queue_to_listen=REQUEST_DATA_QUEUE,
ip_country="FR", segment_position=1,
limit=40):
super().__init__()
self.link_to_validate_list = []
self.cookie = SimpleCookie()
self.cookiesPublisher = cookiesPublisher
self.link_to_validate_list = link_to_validate_list
self.segment_position = segment_position
self.update_validate_list()
self.queue_to_listen = queue_to_listen
self.ip_country = ip_country
self.filter_with_ip_country()
@@ -62,7 +65,7 @@ class LinkValidator(threading.Thread):
print("received cookie is " + str(self.cookie_str))
try:
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use,
timeout=10)
timeout=30)
print(response.status_code)
if response.status_code == 200:
_content = response.text
@@ -99,17 +102,35 @@ class LinkValidator(threading.Thread):
return RequestResult.SUCCESS
else:
return RequestResult.UNKNOWN
elif response.status_code == 502:
return RequestResult.PROXY_ERROR
else:
return RequestResult.BLOCKED
except Exception as error:
print(error)
return RequestResult.PROXY_ERROR
def update_validate_list(self):
# for the moment, max segment is 2
_all_list = MONGO_STORE_MANAGER.get_links_to_validate()
list_length = len(_all_list)
middle = int(len(_all_list) / 2)
if self.segment_position == 1:
if list_length > 1:
self.link_to_validate_list = _all_list[0:middle]
else:
self.link_to_validate_list = _all_list
else:
if list_length > 1:
self.link_to_validate_list = _all_list[middle:]
else:
self.link_to_validate_list = []
def on_message(self, ch, method, properties, body):
print(f" [x] Received {body}")
_message_in_queue_count = self.cookiesPublisher.message_count()
print("message count in queue is {}".format(_message_in_queue_count))
self.link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
self.update_validate_list()
self.filter_with_ip_country()
self.cookie_str = body.decode("UTF-8")
random.shuffle(self.link_to_validate_list)
@@ -141,24 +162,15 @@ class LinkValidator(threading.Thread):
self.link_to_validate_list = _link_list_to_click
def validate_with_FR_ip():
def validate_with_FR_ip(segment_position=1):
_queue_name = REQUEST_DATA_QUEUE
link_list = MONGO_STORE_MANAGER.get_links_to_validate()
cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
cookiesPublisher.set_up_connection()
print("filter links with ip_country")
_link_list_to_click = []
for _link in link_list:
if _link.ip_country == "FR":
_link_list_to_click.append(_link)
for _l in _link_list_to_click:
print(_l.ip_country)
_proxy_manager = ProxyManager()
# _fr_proxy_to_use = random.choice(PROXY_LIST_FR)
# random.shuffle(_link_list_to_click)
receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
receiver = LinkValidator(cookiesPublisher=cookiesPublisher,
proxy_manager=_proxy_manager,
queue_to_listen=_queue_name, ip_country="FR", limit=50)
queue_to_listen=_queue_name, ip_country="FR", segment_position=segment_position, limit=50)
print("will connect to queue")
receiver.set_up_connection()
receiver.listen_to_queue(receiver.on_message)
workers/validate_links_DE.py
-33
View File
@@ -1,33 +0,0 @@
import random
from db.mongo_manager import MONGO_STORE_MANAGER
from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher
from workers.link_validator import LinkValidator
from workers.proxies_constants import PROXY_LIST_DE
def validate_with_DE_ip():
_queue_name = REQUEST_DATA_QUEUE_DE
link_list = MONGO_STORE_MANAGER.get_links_to_validate()
cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
cookiesPublisher.set_up_connection()
print("filter links with ip_country")
_link_list_to_click = []
for _link in link_list:
if _link.ip_country == "DE":
_link_list_to_click.append(_link)
for _l in _link_list_to_click:
print(_l.ip_country)
# else:
# print(_link.ip_country)
_de_proxy_to_use = random.choice(PROXY_LIST_DE)
receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
proxy_to_use=_de_proxy_to_use,
queue_to_listen=_queue_name, ip_country="DE", limit=0)
receiver.set_up_connection()
receiver.listen_to_queue(receiver.on_message)
pass
if __name__ == '__main__':
validate_with_DE_ip()