panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support ip_country field

Browse Source
This commit is contained in:
Lei PAN
2024-01-23 23:33:55 +01:00
parent 97c2b9ce8c
commit aa26e17e99
12 changed files with 147 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workers/captcha_result_getter.py
+7 -7
View File
@@ -5,7 +5,7 @@ from time import time
from typing import Union
import requests
from workers.proxies_constants import PROXY_LIST
from workers.proxies_constants import PROXY_LIST_FR
API_KEY = "d66aaf490d8aa424a5175e1fbd1aadea"
@@ -21,7 +21,7 @@ class CaptchaResultGetter:
self.cookie_str = 'datadome=5Nq~NEP_qQSHC0g_lZNnZmEv36J8gVV~rpZ329xmCkTq2~H3meIoXr4h_b988qB2XW5Te7iEGsvq8BzA5KeFupyrZFh4kgrDyl8hT2UymSByKHzAcDaNIBPDsRu2g_KG; Max-Age=31536000; Domain=.hermes.com; Path=/; Secure; SameSite=None'
pass
def get_csrf(self, cookie: str = None) -> Union[str, None]:
def get_csrf(self, proxy_to_use, cookie: str = None) -> Union[str, None]:
if cookie is not None:
headers = {'Content-Type': 'application/x-www-form-urlencoded',
'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
@@ -39,8 +39,8 @@ class CaptchaResultGetter:
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Dest': 'document',
'Accept-Language': 'fr-FR,fr;q=0.6'}
proxy_to_use = random.choice(PROXY_LIST)
print("received cookie is " + str(cookie))
print(proxy_to_use)
try:
response = requests.get(url=HERMES_REGISTER, headers=headers, verify=False, proxies=proxy_to_use,
timeout=15)
@@ -62,7 +62,7 @@ class CaptchaResultGetter:
return result_list[-1]
return None
def get_valid_cookie(self, old_valid_cookie: str, dvm=4, hc=6) -> Union[str, None]:
def get_valid_cookie(self, proxy_to_use, old_valid_cookie: str, dvm=4, hc=6) -> Union[str, None]:
headers = {'content-Type': 'application/x-www-form-urlencoded',
'user-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
'accept': '*/*',
@@ -75,8 +75,8 @@ class CaptchaResultGetter:
'sec-ch-ua-platform': 'Android',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'fr-FR,fr;q=0.6'}
proxy_to_use = random.choice(PROXY_LIST)
print("send request to get new cookie")
print(proxy_to_use)
print(headers)
try:
# tag_pu = 10 * Math.random()
@@ -97,7 +97,7 @@ class CaptchaResultGetter:
print(error)
return None
def get_valid_ch_cookie(self, old_valid_cookie: str = None, dvm=4, hc=6) -> Union[str, None]:
def get_valid_ch_cookie(self, proxy_to_use, old_valid_cookie: str = None, dvm=4, hc=6) -> Union[str, None]:
headers = {'content-Type': 'application/x-www-form-urlencoded',
'user-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
'accept': '*/*',
@@ -109,7 +109,7 @@ class CaptchaResultGetter:
'sec-fetch-dest': 'empty',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'fr-FR,fr;q=0.6'}
proxy_to_use = random.choice(PROXY_LIST)
print(proxy_to_use)
print("send request to get new cookie")
print(headers)
try:
workers/cookie_generator.py
+15 -14
View File
@@ -13,12 +13,13 @@ DVM_LIST = [2, 3, 4, 6]
class CookiesGenerator(threading.Thread):
def __init__(self, cookiesPublisher: CookiesPublisher):
def __init__(self, proxy_to_use_list: list, cookiesPublisher: CookiesPublisher):
super().__init__()
self.connection = None
self.cookiesPublisher = cookiesPublisher
self.channel = None
self.valid_csrf = None
self.proxy_to_use_list = proxy_to_use_list
def set_up_connection(self):
self.connection = pika.BlockingConnection(
@@ -40,22 +41,25 @@ class CookiesGenerator(threading.Thread):
# generate 10 cookies to new_queue
captchaResultGetter = CaptchaResultGetter()
_cookies_count = random.randint(2, 10)
_cookies_count = random.randint(8, 10)
_received_cookies = body.decode("UTF-8")
dvm = random.choice(DVM_LIST)
hc = random.choice(DVM_LIST)
if _message_count < 10:
_proxy_to_use = random.choice(self.proxy_to_use_list)
for i in range(1, _cookies_count):
new_cookie = captchaResultGetter.get_valid_ch_cookie(dvm=dvm, hc=hc)
# new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, dvm=dvm, hc=hc)
_app_sig = get_app_and_app_sig(_received_cookies)
if new_cookie is not None and _app_sig is not None:
new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
"Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "")
print("new_cookie is " + new_cookie)
_received_cookies = new_cookie
self.cookiesPublisher.publish_body(new_cookie)
new_cookie = captchaResultGetter.get_valid_cookie(new_cookie, dvm=dvm, hc=hc)
# if new_cookie is not None and _app_sig is not None:
# new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
# new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
# "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "")
# print("new_cookie is " + new_cookie)
# _received_cookies = new_cookie
# self.cookiesPublisher.publish_body(new_cookie)
new_cookie = captchaResultGetter.get_valid_cookie(proxy_to_use=_proxy_to_use,
old_valid_cookie=_received_cookies, dvm=dvm,
hc=hc)
if new_cookie is not None and _app_sig is not None:
new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
@@ -86,9 +90,6 @@ def get_app_and_app_sig(cookies: str):
if __name__ == '__main__':
# _list = get_app_and_app_sig(
# "app.sig=axk5aJ2c83dgV56DswNhw98y4SY;datadome=2dTtQDIHCadAbzh8JWv7MFKGH8~fkOoGlA1O3J7llv_sdYOSZYbaKruHPBbARchV1nnrstsaMX5E_XwbJmGiXzt5qZXOL0UCAV9TTHqBRh10JvO~GCGJv2JVO~6RnrcU;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiZEE2bEdnN04yd2t0eWVmZmVjVkxMY1dTIiwiYXBwb2ludG1lbnRfY29kZSI6Ik5DTlVaQiIsImJsb2NrX3JlZ2lzdHJhdGlvbiI6ZmFsc2V9;policy=accepted;lang=fr;")
# print(_list)
cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_TEST)
cookiesPublisher.set_up_connection()
cookieGenerator = CookiesGenerator(cookiesPublisher)
workers/de_cookie_generator.py
+9
View File
@@ -0,0 +1,9 @@
from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher
from workers.cookie_generator import CookiesGenerator
from workers.proxies_constants import PROXY_LIST_DE
if __name__ == '__main__':
cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_DE)
cookiesPublisher.set_up_connection()
cookieGenerator = CookiesGenerator(cookiesPublisher=cookiesPublisher, proxy_to_use_list=PROXY_LIST_DE)
cookieGenerator.run()
workers/link_validator.py
+53 -24
View File
@@ -9,19 +9,22 @@ import requests
from db.mongo_manager import MONGO_STORE_MANAGER
from models.LinkPojo import LinkPojo
from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST
from queue_message.receiver import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
from workers.proxies_constants import PROXY_LIST
from queue_message.appointmentrequestsendor import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
from workers.proxies_constants import PROXY_LIST_FR
class LinkValidator(threading.Thread):
def __init__(self, link_to_validate_list: list,
cookiesPublisher: CookiesPublisher, queue_to_listen=REQUEST_DATA_QUEUE):
cookiesPublisher: CookiesPublisher, proxy_to_use, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR"):
super().__init__()
self.cookie = SimpleCookie()
self.cookiesPublisher = cookiesPublisher
self.link_to_validate_list = link_to_validate_list
self.queue_to_listen = queue_to_listen
self.ip_country = ip_country
self.filter_with_ip_country()
self.proxy_to_use = proxy_to_use
# self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
def set_up_connection(self):
@@ -48,29 +51,32 @@ class LinkValidator(threading.Thread):
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Dest': 'document',
'Accept-Language': 'fr-FR,fr;q=0.6'}
proxy_to_use = random.choice(PROXY_LIST)
print(proxy_to_use)
print(self.proxy_to_use)
print("received cookie is " + str(self.cookie_str))
try:
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=proxy_to_use,
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
timeout=15)
print(response.status_code)
if response.status_code == 200:
_content = response.text
print(response.text)
print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo)
# set new cookies
_cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set)
new_cookies = {k: v.value for k, v in self.cookie.items()}
new_coolies_str = ""
for key in new_cookies:
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
print("will publish to queue {}".format(new_coolies_str))
# upload the cookie to queue
self.cookiesPublisher.publish_body(new_coolies_str)
self.cookie_str = new_coolies_str
return True
if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content:
print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo)
# set new cookies
_cookies_to_set = response.headers['set-cookie']
self.cookie.load(_cookies_to_set)
new_cookies = {k: v.value for k, v in self.cookie.items()}
new_coolies_str = ""
for key in new_cookies:
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
print("will publish to queue {}".format(new_coolies_str))
# upload the cookie to queue
self.cookiesPublisher.publish_body(new_coolies_str)
self.cookie_str = new_coolies_str
return True
else:
return False
else:
return False
except Exception as error:
@@ -80,6 +86,7 @@ class LinkValidator(threading.Thread):
def on_message(self, ch, method, properties, body):
print(f" [x] Received {body}")
self.link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
self.filter_with_ip_country()
self.cookie_str = body.decode("UTF-8")
random.shuffle(self.link_to_validate_list)
if len(self.link_to_validate_list) > 0:
@@ -101,12 +108,34 @@ class LinkValidator(threading.Thread):
time.sleep(60)
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
def filter_with_ip_country(self):
_link_list_to_click = []
for _link in self.link_to_validate_list:
if _link.ip_country == self.ip_country:
_link_list_to_click.append(_link)
self.link_to_validate_list = _link_list_to_click
if __name__ == '__main__':
def validate_with_FR_ip():
_queue_name = REQUEST_DATA_QUEUE_TEST
link_list = MONGO_STORE_MANAGER.get_links_to_validate()
cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_TEST)
cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
cookiesPublisher.set_up_connection()
receiver = LinkValidator(link_to_validate_list=link_list, cookiesPublisher=cookiesPublisher,
queue_to_listen=REQUEST_DATA_QUEUE_TEST)
print("filter links with ip_country")
_link_list_to_click = []
for _link in link_list:
if _link.ip_country == "FR":
_link_list_to_click.append(_link)
for _l in _link_list_to_click:
print(_l.ip_country)
_fr_proxy_to_use = random.choice(PROXY_LIST_FR)
receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
proxy_to_use=_fr_proxy_to_use,
queue_to_listen=_queue_name, ip_country="FR")
receiver.set_up_connection()
receiver.listen_to_queue(receiver.on_message)
pass
if __name__ == '__main__':
validate_with_FR_ip()
workers/proxies_constants.py
+2 -1
View File
@@ -31,4 +31,5 @@ DE_PROXY_RES = {
# FR_PROXY_MOBILE
# FR_PROXY_RES
# PROXY_LIST = [FR_PROXY_MOBILE, FR_PROXY_RES, DE_PROXY_RES, DE_PROXY_MOBILE, ES_PROXY_MOBILE, IT_PROXY_MOBILE]
PROXY_LIST = [FR_PROXY_MOBILE, FR_PROXY_RES]
PROXY_LIST_FR = [FR_PROXY_MOBILE, FR_PROXY_RES]
PROXY_LIST_DE = [DE_PROXY_RES, DE_PROXY_MOBILE]
workers/sender.py
+4 -3
View File
@@ -8,18 +8,19 @@ import requests
from db.mongo_manager import MONGO_STORE_MANAGER
from models.ReserveResultPojo import ReserveResultPojo, PublishType
from queue_message.CookiesPublisher import CookiesPublisher
from workers.proxies_constants import PROXY_LIST
from workers.proxies_constants import PROXY_LIST_FR
class Sender:
def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher):
def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher, proxy_to_use):
self.store_type = "random"
self.cookie = SimpleCookie()
self.cookiesPublisher = cookiesPublisher
# self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
self.cookie_str = cookie_str
self._csrf = None
self.proxy_to_use = proxy_to_use
self.cookie.load(self.cookie_str)
def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str):
@@ -64,7 +65,7 @@ class Sender:
'passport_id': contact.passport, 'processing': 'on', 'cgu': 'on'}
print(data)
try:
proxy_to_use = random.choice(PROXY_LIST)
proxy_to_use = self.proxy_to_use
# proxy_to_use = PROXY_LIST[0]
print(proxy_to_use)
response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data,
workers/validate_links_DE.py
+33
View File
@@ -0,0 +1,33 @@
import random
from db.mongo_manager import MONGO_STORE_MANAGER
from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher
from workers.link_validator import LinkValidator
from workers.proxies_constants import PROXY_LIST_DE
def validate_with_DE_ip():
_queue_name = REQUEST_DATA_QUEUE_DE
link_list = MONGO_STORE_MANAGER.get_links_to_validate()
cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
cookiesPublisher.set_up_connection()
print("filter links with ip_country")
_link_list_to_click = []
for _link in link_list:
if _link.ip_country == "DE":
_link_list_to_click.append(_link)
for _l in _link_list_to_click:
print(_l.ip_country)
# else:
# print(_link.ip_country)
_de_proxy_to_use = random.choice(PROXY_LIST_DE)
receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
proxy_to_use=_de_proxy_to_use,
queue_to_listen=_queue_name, ip_country="DE")
receiver.set_up_connection()
receiver.listen_to_queue(receiver.on_message)
pass
if __name__ == '__main__':
validate_with_DE_ip()