panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support segment

Browse Source
This commit is contained in:
Lei PAN
2024-04-17 15:10:40 +02:00
parent 5d5d7afed5
commit 4450bb1afb
3 changed files with 11 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/mongo_manager.py
+2 -2
View File
@@ -84,7 +84,7 @@ class MongoDbManager:
return link_list return link_list
def link_validated_for_result(self, link: str, linkPojo: LinkPojo, state=True, is_duplicated=False, def link_validated_for_result(self, link: str, linkPojo: LinkPojo, state=True, is_duplicated=False,
is_invalid=False): is_invalid=False, segement_position=1):
print("link_validated_for_result() called with url = " + link) print("link_validated_for_result() called with url = " + link)
if is_duplicated: if is_duplicated:
_id = link.split("/")[-2] _id = link.split("/")[-2]
@@ -99,7 +99,7 @@ class MongoDbManager:
collection = self.db[collection_name] collection = self.db[collection_name]
validated_at = time.strftime("%H:%M:%S", time.localtime()) validated_at = time.strftime("%H:%M:%S", time.localtime())
validated_by = "requests" validated_by = "requests" + str(segement_position)
if is_invalid: if is_invalid:
validated_by = "Invalid" validated_by = "Invalid"
if is_duplicated: if is_duplicated:
proxy_manager/proxy_manager.py
+4 -4
View File
@@ -22,8 +22,8 @@ FR_PROXY_ASOCK_RES_2 = {
class ProxyManager: class ProxyManager:
def get_link_validate_proxy(self, links_to_validate: list) -> list: def get_link_validate_proxy(self, links_to_validate: list) -> list:
if len(links_to_validate) > 15: if len(links_to_validate) > 15:
# return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY] return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY]
return [FR_PROXY_RES_OXY] # return [FR_PROXY_RES_OXY]
else: else:
# return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY] return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY]
return [FR_PROXY_RES_OXY] # return [FR_PROXY_RES_OXY]
workers/link_validator.py
+5 -5
View File
@@ -67,14 +67,15 @@ class LinkValidator(threading.Thread):
print("send request for link: " + linkPojo.url) print("send request for link: " + linkPojo.url)
try: try:
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use, response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use,
timeout=30) timeout=60)
print(response.status_code) print(response.status_code)
if response.status_code == 200: if response.status_code == 200:
_content = response.text _content = response.text
print(response.text) print(response.text)
if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content: if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content:
print(response.url) print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo) MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo,
segement_position=self.segment_position)
# set new cookies # set new cookies
_cookies_to_set = response.headers['set-cookie'] _cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set) self.cookie.load(_cookies_to_set)
@@ -169,7 +170,7 @@ class LinkValidator(threading.Thread):
break break
time.sleep(random.randint(2, 5)) time.sleep(random.randint(2, 5))
print("can continue, continue") print("can continue, continue")
if can_continue == RequestResult.BAD_GATEWAY or can_continue == RequestResult.PROXY_ERROR: if can_continue == RequestResult.BAD_GATEWAY or can_continue == RequestResult.PROXY_ERROR or can_continue == RequestResult.SUCCESS:
print("will requeue the message") print("will requeue the message")
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True) ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
else: else:
@@ -189,8 +190,7 @@ class LinkValidator(threading.Thread):
def validate_with_FR_ip(segment_position=1): def validate_with_FR_ip(segment_position=1):
# _queue_name = REQUEST_DATA_QUEUE _queue_name = REQUEST_DATA_QUEUE
_queue_name = REQUEST_DATA_DE
cookiesPublisher = CookiesPublisher(queue_name=_queue_name) cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
cookiesPublisher.set_up_connection() cookiesPublisher.set_up_connection()
print("filter links with ip_country") print("filter links with ip_country")