panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/5_0_2' into feature/with_lan

Browse Source
This commit is contained in:
PAN Lei
2025-07-11 12:07:32 +02:00
parent 343a14f6a2 3d05f739b4
commit 8ae6a7593b
4 changed files with 75 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
models/ReserveResultPojo.py
+2 -2
View File
@@ -29,6 +29,7 @@ class ReserveResultPojo:
slot_position = None
sim_position = None
ccid: str = ""
model: str = ""
source_from: str = socket.gethostname()
store_type = 0
url_validated = None
@@ -122,8 +123,7 @@ class ReserveResultPojo:
u'email': self.email,
u'passport': self.passport,
u'url': self.url,
# u'sim_position': self.sim_position,
# u'slot_position': self.slot_position,
u'model': self.model,
u'source_from': self.source_from,
u'hostName': self.source_from,
u'created_at': self.created_at,
proxy_manager/proxy_manager.py
+21 -7
View File
@@ -1,5 +1,8 @@
import logging
import random
import requests
FR_ASOCKS_MOBILE_PROXY = {
'http': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046',
'https': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046',
@@ -25,8 +28,8 @@ FR_PROXY_ASOCK_RES_2 = {
'https': 'http://zd6fbrujot-res-country-FR-hold-query:8k8avNlnLHQaMsWg@217.23.6.161:9999'
}
FR_PROXY_MOB_OXY_STICKY = {
'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}',
'https': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}'
'http': 'http://customer-rendezvousmob-cc-fr-sessid-0{}-sesstime-2:Rdv+202220212023@pr.oxylabs.io:7777',
'https': 'http://customer-rendezvousmob-cc-fr-sessid-0{}-sesstime-2:Rdv+202220212023@pr.oxylabs.io:7777'
}
FR_PROXY_DATA_IMPULSE_STICKY = {
@@ -59,12 +62,11 @@ FR_MOBILE_ANY_IP_ROTATING = {
# FR_PROXY_DATA_IMPULSE_STICKY]
# MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY, FR_PROXY_MOB_OXY_STICKY]
# MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY]
MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY]
# MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY]
class ProxyManager:
def __init__(self, logger):
@@ -72,7 +74,8 @@ class ProxyManager:
def get_link_validate_proxy(self, links_to_validate: list) -> list:
# return [FR_PROXY_RES_PARIS_OXY]
return [FR_MOBILE_ANY_IP_ROTATING]
# return [FR_MOBILE_ANY_IP_ROTATING]
return [FR_PROXY_RES_OXY]
# if len(links_to_validate) > 15:
# return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
# # return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY]
@@ -88,7 +91,7 @@ class ProxyManager:
_chosen_proxy = random.choice(MOBILE_PROXY_LIST)
if "oxylabs" in _chosen_proxy["http"]:
self.logger.info("use oxylabs proxy")
_port = random.randint(40001, 49999)
_port = random.randint(900000000, 995869818)
elif "anyip" in _chosen_proxy["http"]:
self.logger.info("use anyip proxy")
_port = random.randint(40001, 49999)
@@ -100,3 +103,14 @@ class ProxyManager:
_proxy_to_use["http"] = _chosen_proxy["http"].format(_port)
_proxy_to_use["https"] = _chosen_proxy["https"].format(_port)
return _proxy_to_use
if __name__ == '__main__':
_logger = logging.getLogger()
_proxy = ProxyManager(logger=_logger)
_proxise = _proxy.get_proxy_for_appointment_request()
print(_proxise)
response = requests.get(
"https://ip.oxylabs.io/location",
proxies=_proxise)
print(response.content)
workers/link_validator_with_provided_list.py
+46 -39
View File
@@ -149,44 +149,51 @@ class LinkValidatorWithProvidedList(threading.Thread):
_message_in_queue_count = self.cookiesPublisher.message_count()
print("message count in queue is {}".format(_message_in_queue_count))
_received_object = body.decode("UTF-8")
_received_dict = json.loads(_received_object)
_received_cookies = _received_dict["cookiesStr"]
self.cookie_str = _received_cookies
_links_to_validate = filter_link_pojo_list_with_serial(_received_dict, self.link_to_validate_list)
random.shuffle(_links_to_validate)
if len(_links_to_validate) > 0 and _message_in_queue_count >= self.limit:
print("{}:links number is {}".format(threading.currentThread().name, len(_links_to_validate)))
can_continue = None
for link_to_validate in _links_to_validate:
print(link_to_validate)
self.proxy_to_use = random.choice(
self.proxy_manager.get_link_validate_proxy(_links_to_validate))
print("proxy to use is {}".format(self.proxy_to_use))
can_continue = self.send_request(link_to_validate, _received_dict)
# remove the tested link from link list
self.link_to_validate_list.remove(link_to_validate)
if can_continue == RequestResult.BLOCKED:
print("cannot continue, blocked, then skip")
break
else:
if can_continue == RequestResult.BAD_GATEWAY:
time.sleep(30)
try:
_received_dict = json.loads(_received_object)
_received_cookies = _received_dict["cookiesStr"]
self.cookie_str = _received_cookies
_links_to_validate = filter_link_pojo_list_with_serial(_received_dict, self.link_to_validate_list)
random.shuffle(_links_to_validate)
if len(_links_to_validate) > 0 and _message_in_queue_count >= self.limit:
print("{}:links number is {}".format(threading.currentThread().name, len(_links_to_validate)))
can_continue = None
for link_to_validate in _links_to_validate:
print(link_to_validate)
self.proxy_to_use = random.choice(
self.proxy_manager.get_link_validate_proxy(_links_to_validate))
print("proxy to use is {}".format(self.proxy_to_use))
can_continue = self.send_request(link_to_validate, _received_dict)
# remove the tested link from link list
self.link_to_validate_list.remove(link_to_validate)
if can_continue == RequestResult.BLOCKED:
print("cannot continue, blocked, then skip")
break
time.sleep(random.randint(2, 5))
print("can continue, continue")
if can_continue == RequestResult.BAD_GATEWAY or can_continue == RequestResult.PROXY_ERROR or can_continue == RequestResult.SUCCESS:
print("will requeue the message")
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
else:
if can_continue == RequestResult.BAD_GATEWAY:
time.sleep(30)
break
time.sleep(random.randint(2, 5))
print("can continue, continue")
if can_continue == RequestResult.BAD_GATEWAY or can_continue == RequestResult.PROXY_ERROR or can_continue == RequestResult.SUCCESS:
print("will requeue the message")
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
else:
print("will ack")
print("will wait for 40s")
time.sleep(20)
ch.basic_ack(delivery_tag=method.delivery_tag)
else:
print("will ack")
print("empty list, will republish message")
time.sleep(0)
print("body in json:{}".format(json.dumps(_received_dict)))
self.cookiesPublisher.publish_body(json.dumps(_received_dict))
ch.basic_ack(delivery_tag=method.delivery_tag)
else:
print("empty list, will republish message")
time.sleep(0)
print("body in json:{}".format(json.dumps(_received_dict)))
self.cookiesPublisher.publish_body(json.dumps(_received_dict))
self.channel.stop_consuming()
except Exception as error:
print(error)
print("not json format will ack")
ch.basic_ack(delivery_tag=method.delivery_tag)
self.channel.stop_consuming()
def filter_with_ip_country(self):
_link_list_to_click = []
@@ -213,7 +220,7 @@ def validate_all_links(_contact_serial_list):
link_to_validated = []
for _link in all_link_list:
# print("serial is "+_link.serial)
print("email is "+_link.email)
print("email is " + _link.email)
if _link.serial == "requests":
link_to_validated.append(_link)
# get the first 50 links
@@ -223,8 +230,8 @@ def validate_all_links(_contact_serial_list):
# default_segment_number = 20
_first_25_percent_links = link_to_validated[0:(int(len(all_link_list) / divided))]
_first_25_percent_links = all_link_list
# _queue_name = MORNING_DATA_CACHE
_queue_name = MORNING_DATA_CACHE_BAK
_queue_name = MORNING_DATA_CACHE
# _queue_name = MORNING_DATA_CACHE_BAK
# if len(all_link_list) > divided * default_segment_number:
# _segment_number = default_segment_number
# else:
@@ -234,10 +241,10 @@ def validate_all_links(_contact_serial_list):
# else:
# _segment_number = 1
_thread_list = []
if len(_first_25_percent_links) >=10:
if len(_first_25_percent_links) >= 10:
_segment_number = 10
else:
_segment_number = 1
_segment_number = len(_first_25_percent_links)
for i in range(0, _segment_number):
logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_25_percent_links)))
logger.info("segment is {}".format(i))
workers/sender.py
+6 -4
View File
@@ -38,7 +38,8 @@ class Sender:
self.proxy_to_use = proxy_to_use
self.cookie.load(self.cookie_str)
def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str, store_type: str):
def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str, store_type: str,
model: str = ""):
# create the message
if url == "https://rendezvousparis.hermes.com/client/welcome":
return
@@ -59,11 +60,12 @@ class Sender:
result.proxy = "data_impulse"
result.id = id
result.store_type = store_type
result.model = model
result.created_at = time.strftime("%H:%M:%S", time.localtime())
collection_name = str(datetime.date.today())
MONGO_STORE_MANAGER.insert_reserve_result(collection_name=collection_name, reserve=result)
def apply_redirect(self, response, old_headers, contact, js_data, selected_store):
def apply_redirect(self, response, old_headers, contact, js_data, selected_store, model=""):
# /client/register/5XD2E2
_res_headers = response.headers
_location = _res_headers['location']
@@ -87,7 +89,7 @@ class Sender:
contact.current_ip = get_address_ip(proxy_to_use=self.proxy_to_use)
_appointment_url = _redirect_url
self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=_appointment_url,
store_type=selected_store)
store_type=selected_store, model=model)
self.cookie.load(_cookies_to_set)
new_cookies = {k: v.value for k, v in self.cookie.items()}
new_coolies_str = ""
@@ -175,7 +177,7 @@ class Sender:
# add to mongodb
self.logger.info(response.text)
self.apply_redirect(response=response, old_headers=headers, contact=contact, js_data=js_data,
selected_store=_selected_store)
selected_store=_selected_store, model=model)
# self.logger.info("{}:{}".format(contact.mail, response.url))
# contact.ua = js_data.ua
# contact.current_ip = get_address_ip(proxy_to_use=proxy_to_use)