support ip_country field

This commit is contained in:
2024-01-23 23:33:55 +01:00
parent 97c2b9ce8c
commit aa26e17e99
12 changed files with 147 additions and 60 deletions
+1 -1
View File
@@ -8,7 +8,7 @@ from db.mongo_manager import MONGO_STORE_MANAGER
from excel_reader import read_contacts from excel_reader import read_contacts
from models.contact_pojo import ContactPojo from models.contact_pojo import ContactPojo
from queue_message.link_validator_receiver import LinkValidatorReceiver from queue_message.link_validator_receiver import LinkValidatorReceiver
from queue_message.receiver import Receiver, filter_contacts from queue_message.appointmentrequestsendor import AppointmentRequestSendor, filter_contacts
from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
from workers.sender import Sender from workers.sender import Sender
+6 -2
View File
@@ -1,13 +1,17 @@
class LinkPojo(): class LinkPojo():
def __init__(self, url, email, updated_at): def __init__(self, url, email, updated_at, ip_country):
self.url = url self.url = url
self.email = email self.email = email
self.updated_at = updated_at self.updated_at = updated_at
self.ip_country = ip_country
@staticmethod @staticmethod
def from_firestore_dict(source): def from_firestore_dict(source):
updated_at = source['updated_at'] updated_at = source['updated_at']
email = source['email'] email = source['email']
url = source['url'] url = source['url']
result = LinkPojo(email=email, url=url, updated_at=updated_at) ip_country = "FR"
if source.get('ip_country'):
ip_country = source['ip_country']
result = LinkPojo(email=email, url=url, updated_at=updated_at, ip_country=ip_country)
return result return result
+3
View File
@@ -2,7 +2,9 @@ import pika
QUEUE_HOST = "appointment.lpaconsulting.fr" QUEUE_HOST = "appointment.lpaconsulting.fr"
REQUEST_DATA_QUEUE = 'REQUEST_DATA' REQUEST_DATA_QUEUE = 'REQUEST_DATA'
# REQUEST_DATA_QUEUE_TEST = 'REQUEST_DATA_TEST'
REQUEST_DATA_QUEUE_TEST = 'REQUEST_DATA_TEST' REQUEST_DATA_QUEUE_TEST = 'REQUEST_DATA_TEST'
REQUEST_DATA_QUEUE_DE = 'REQUEST_DATA_DE'
credentials = pika.PlainCredentials('appointment', 'ZyuhJZ2xEYWhElhpJjy7YEpZGZwNYJz2fHIu') credentials = pika.PlainCredentials('appointment', 'ZyuhJZ2xEYWhElhpJjy7YEpZGZwNYJz2fHIu')
@@ -21,6 +23,7 @@ class CookiesPublisher:
self.queue_method = self.channel.queue_declare(queue=self.to_queue, durable=True) self.queue_method = self.channel.queue_declare(queue=self.to_queue, durable=True)
def publish_body(self, body: str): def publish_body(self, body: str):
print("will push to queue {}".format(self.to_queue))
self.channel.basic_publish(exchange='', routing_key=self.to_queue, body=body) self.channel.basic_publish(exchange='', routing_key=self.to_queue, body=body)
def message_count(self): def message_count(self):
@@ -11,6 +11,7 @@ from models.contact_pojo import ContactPojo
from queue_message.CookiesPublisher import CookiesPublisher from queue_message.CookiesPublisher import CookiesPublisher
from utiles import is_time_between from utiles import is_time_between
from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
from workers.proxies_constants import PROXY_LIST_DE
from workers.sender import Sender from workers.sender import Sender
QUEUE_HOST = "appointment.lpaconsulting.fr" QUEUE_HOST = "appointment.lpaconsulting.fr"
@@ -72,14 +73,15 @@ def is_open():
return is_time_between(datetime.time(10, 30), datetime.time(19, 00)) return is_time_between(datetime.time(10, 30), datetime.time(19, 00))
class Receiver(threading.Thread): class AppointmentRequestSendor(threading.Thread):
def __init__(self, sub_contact_list: list, cookiesPublisher: CookiesPublisher): def __init__(self, sub_contact_list: list, cookiesPublisher: CookiesPublisher, queue_name=REQUEST_DATA_QUEUE):
super().__init__() super().__init__()
self.connection = None self.connection = None
self.cookiesPublisher = cookiesPublisher self.cookiesPublisher = cookiesPublisher
self.channel = None self.channel = None
self.valid_csrf = None self.valid_csrf = None
self.contact_list = sub_contact_list self.contact_list = sub_contact_list
self.queue_name = queue_name
def set_up_connection(self): def set_up_connection(self):
self.connection = pika.BlockingConnection( self.connection = pika.BlockingConnection(
@@ -87,14 +89,16 @@ class Receiver(threading.Thread):
self.channel = self.connection.channel() self.channel = self.connection.channel()
def listen_to_queue(self, callback): def listen_to_queue(self, callback):
print("listen to queue {}".format(self.queue_name))
self.channel.basic_qos(prefetch_count=1) self.channel.basic_qos(prefetch_count=1)
self.channel.basic_consume(queue=REQUEST_DATA_QUEUE, auto_ack=False, on_message_callback=callback) self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback)
self.channel.start_consuming() self.channel.start_consuming()
def on_message(self, ch, method, properties, body): def on_message(self, ch, method, properties, body):
print(f" [x] Received {body}") print(f" [x] Received {body}")
print("message count in queue is {}".format(self.cookiesPublisher.message_count())) print("message count in queue is {}".format(self.cookiesPublisher.message_count()))
sender = Sender(body.decode("UTF-8"), cookiesPublisher=self.cookiesPublisher) sender = Sender(body.decode("UTF-8"), cookiesPublisher=self.cookiesPublisher,
proxy_to_use=random.choice(PROXY_LIST_DE))
self.contact_list = filter_contacts(self.contact_list) self.contact_list = filter_contacts(self.contact_list)
# remove already booked contacts # remove already booked contacts
random.shuffle(self.contact_list) random.shuffle(self.contact_list)
+6 -4
View File
@@ -7,7 +7,7 @@ from db.mongo_manager import MONGO_STORE_MANAGER
from excel_reader import read_contacts from excel_reader import read_contacts
from models.contact_pojo import ContactPojo from models.contact_pojo import ContactPojo
from queue_message.CookiesPublisher import CookiesPublisher from queue_message.CookiesPublisher import CookiesPublisher
from queue_message.receiver import Receiver from queue_message.appointmentrequestsendor import AppointmentRequestSendor
from utiles import is_time_between from utiles import is_time_between
from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
from workers.sender import Sender from workers.sender import Sender
@@ -71,8 +71,10 @@ def is_open():
if __name__ == '__main__': if __name__ == '__main__':
cookiesPublisher = CookiesPublisher() REQUEST_DATA_QUEUE_DE = 'REQUEST_DATA_DE'
cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_DE)
cookiesPublisher.set_up_connection() cookiesPublisher.set_up_connection()
contact_list = read_contacts('/Users/panlei/Desktop/real_name_contacts_77_14_01_2024.xlsx') contact_list = read_contacts('/Users/lpan/Desktop/08_01_24_valid_de.xlsx')
receiver = Receiver(sub_contact_list=contact_list, cookiesPublisher=cookiesPublisher) receiver = AppointmentRequestSendor(sub_contact_list=contact_list, queue_name=REQUEST_DATA_QUEUE_DE,
cookiesPublisher=cookiesPublisher)
receiver.run() receiver.run()
+7 -7
View File
@@ -5,7 +5,7 @@ from time import time
from typing import Union from typing import Union
import requests import requests
from workers.proxies_constants import PROXY_LIST from workers.proxies_constants import PROXY_LIST_FR
API_KEY = "d66aaf490d8aa424a5175e1fbd1aadea" API_KEY = "d66aaf490d8aa424a5175e1fbd1aadea"
@@ -21,7 +21,7 @@ class CaptchaResultGetter:
self.cookie_str = 'datadome=5Nq~NEP_qQSHC0g_lZNnZmEv36J8gVV~rpZ329xmCkTq2~H3meIoXr4h_b988qB2XW5Te7iEGsvq8BzA5KeFupyrZFh4kgrDyl8hT2UymSByKHzAcDaNIBPDsRu2g_KG; Max-Age=31536000; Domain=.hermes.com; Path=/; Secure; SameSite=None' self.cookie_str = 'datadome=5Nq~NEP_qQSHC0g_lZNnZmEv36J8gVV~rpZ329xmCkTq2~H3meIoXr4h_b988qB2XW5Te7iEGsvq8BzA5KeFupyrZFh4kgrDyl8hT2UymSByKHzAcDaNIBPDsRu2g_KG; Max-Age=31536000; Domain=.hermes.com; Path=/; Secure; SameSite=None'
pass pass
def get_csrf(self, cookie: str = None) -> Union[str, None]: def get_csrf(self, proxy_to_use, cookie: str = None) -> Union[str, None]:
if cookie is not None: if cookie is not None:
headers = {'Content-Type': 'application/x-www-form-urlencoded', headers = {'Content-Type': 'application/x-www-form-urlencoded',
'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36', 'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
@@ -39,8 +39,8 @@ class CaptchaResultGetter:
'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Dest': 'document',
'Accept-Language': 'fr-FR,fr;q=0.6'} 'Accept-Language': 'fr-FR,fr;q=0.6'}
proxy_to_use = random.choice(PROXY_LIST)
print("received cookie is " + str(cookie)) print("received cookie is " + str(cookie))
print(proxy_to_use)
try: try:
response = requests.get(url=HERMES_REGISTER, headers=headers, verify=False, proxies=proxy_to_use, response = requests.get(url=HERMES_REGISTER, headers=headers, verify=False, proxies=proxy_to_use,
timeout=15) timeout=15)
@@ -62,7 +62,7 @@ class CaptchaResultGetter:
return result_list[-1] return result_list[-1]
return None return None
def get_valid_cookie(self, old_valid_cookie: str, dvm=4, hc=6) -> Union[str, None]: def get_valid_cookie(self, proxy_to_use, old_valid_cookie: str, dvm=4, hc=6) -> Union[str, None]:
headers = {'content-Type': 'application/x-www-form-urlencoded', headers = {'content-Type': 'application/x-www-form-urlencoded',
'user-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36', 'user-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
'accept': '*/*', 'accept': '*/*',
@@ -75,8 +75,8 @@ class CaptchaResultGetter:
'sec-ch-ua-platform': 'Android', 'sec-ch-ua-platform': 'Android',
'accept-encoding': 'gzip, deflate, br', 'accept-encoding': 'gzip, deflate, br',
'accept-language': 'fr-FR,fr;q=0.6'} 'accept-language': 'fr-FR,fr;q=0.6'}
proxy_to_use = random.choice(PROXY_LIST)
print("send request to get new cookie") print("send request to get new cookie")
print(proxy_to_use)
print(headers) print(headers)
try: try:
# tag_pu = 10 * Math.random() # tag_pu = 10 * Math.random()
@@ -97,7 +97,7 @@ class CaptchaResultGetter:
print(error) print(error)
return None return None
def get_valid_ch_cookie(self, old_valid_cookie: str = None, dvm=4, hc=6) -> Union[str, None]: def get_valid_ch_cookie(self, proxy_to_use, old_valid_cookie: str = None, dvm=4, hc=6) -> Union[str, None]:
headers = {'content-Type': 'application/x-www-form-urlencoded', headers = {'content-Type': 'application/x-www-form-urlencoded',
'user-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36', 'user-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
'accept': '*/*', 'accept': '*/*',
@@ -109,7 +109,7 @@ class CaptchaResultGetter:
'sec-fetch-dest': 'empty', 'sec-fetch-dest': 'empty',
'accept-encoding': 'gzip, deflate, br', 'accept-encoding': 'gzip, deflate, br',
'accept-language': 'fr-FR,fr;q=0.6'} 'accept-language': 'fr-FR,fr;q=0.6'}
proxy_to_use = random.choice(PROXY_LIST) print(proxy_to_use)
print("send request to get new cookie") print("send request to get new cookie")
print(headers) print(headers)
try: try:
+15 -14
View File
@@ -13,12 +13,13 @@ DVM_LIST = [2, 3, 4, 6]
class CookiesGenerator(threading.Thread): class CookiesGenerator(threading.Thread):
def __init__(self, cookiesPublisher: CookiesPublisher): def __init__(self, proxy_to_use_list: list, cookiesPublisher: CookiesPublisher):
super().__init__() super().__init__()
self.connection = None self.connection = None
self.cookiesPublisher = cookiesPublisher self.cookiesPublisher = cookiesPublisher
self.channel = None self.channel = None
self.valid_csrf = None self.valid_csrf = None
self.proxy_to_use_list = proxy_to_use_list
def set_up_connection(self): def set_up_connection(self):
self.connection = pika.BlockingConnection( self.connection = pika.BlockingConnection(
@@ -40,22 +41,25 @@ class CookiesGenerator(threading.Thread):
# generate 10 cookies to new_queue # generate 10 cookies to new_queue
captchaResultGetter = CaptchaResultGetter() captchaResultGetter = CaptchaResultGetter()
_cookies_count = random.randint(2, 10) _cookies_count = random.randint(8, 10)
_received_cookies = body.decode("UTF-8") _received_cookies = body.decode("UTF-8")
dvm = random.choice(DVM_LIST) dvm = random.choice(DVM_LIST)
hc = random.choice(DVM_LIST) hc = random.choice(DVM_LIST)
if _message_count < 10: if _message_count < 10:
_proxy_to_use = random.choice(self.proxy_to_use_list)
for i in range(1, _cookies_count): for i in range(1, _cookies_count):
new_cookie = captchaResultGetter.get_valid_ch_cookie(dvm=dvm, hc=hc) # new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, dvm=dvm, hc=hc)
_app_sig = get_app_and_app_sig(_received_cookies) _app_sig = get_app_and_app_sig(_received_cookies)
if new_cookie is not None and _app_sig is not None: # if new_cookie is not None and _app_sig is not None:
new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie # new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace( # new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
"Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "") # "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "")
print("new_cookie is " + new_cookie) # print("new_cookie is " + new_cookie)
_received_cookies = new_cookie # _received_cookies = new_cookie
self.cookiesPublisher.publish_body(new_cookie) # self.cookiesPublisher.publish_body(new_cookie)
new_cookie = captchaResultGetter.get_valid_cookie(new_cookie, dvm=dvm, hc=hc) new_cookie = captchaResultGetter.get_valid_cookie(proxy_to_use=_proxy_to_use,
old_valid_cookie=_received_cookies, dvm=dvm,
hc=hc)
if new_cookie is not None and _app_sig is not None: if new_cookie is not None and _app_sig is not None:
new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace( new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
@@ -86,9 +90,6 @@ def get_app_and_app_sig(cookies: str):
if __name__ == '__main__': if __name__ == '__main__':
# _list = get_app_and_app_sig(
# "app.sig=axk5aJ2c83dgV56DswNhw98y4SY;datadome=2dTtQDIHCadAbzh8JWv7MFKGH8~fkOoGlA1O3J7llv_sdYOSZYbaKruHPBbARchV1nnrstsaMX5E_XwbJmGiXzt5qZXOL0UCAV9TTHqBRh10JvO~GCGJv2JVO~6RnrcU;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiZEE2bEdnN04yd2t0eWVmZmVjVkxMY1dTIiwiYXBwb2ludG1lbnRfY29kZSI6Ik5DTlVaQiIsImJsb2NrX3JlZ2lzdHJhdGlvbiI6ZmFsc2V9;policy=accepted;lang=fr;")
# print(_list)
cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_TEST) cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_TEST)
cookiesPublisher.set_up_connection() cookiesPublisher.set_up_connection()
cookieGenerator = CookiesGenerator(cookiesPublisher) cookieGenerator = CookiesGenerator(cookiesPublisher)
+9
View File
@@ -0,0 +1,9 @@
from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher
from workers.cookie_generator import CookiesGenerator
from workers.proxies_constants import PROXY_LIST_DE
if __name__ == '__main__':
cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_DE)
cookiesPublisher.set_up_connection()
cookieGenerator = CookiesGenerator(cookiesPublisher=cookiesPublisher, proxy_to_use_list=PROXY_LIST_DE)
cookieGenerator.run()
+53 -24
View File
@@ -9,19 +9,22 @@ import requests
from db.mongo_manager import MONGO_STORE_MANAGER from db.mongo_manager import MONGO_STORE_MANAGER
from models.LinkPojo import LinkPojo from models.LinkPojo import LinkPojo
from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST
from queue_message.receiver import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials from queue_message.appointmentrequestsendor import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
from workers.proxies_constants import PROXY_LIST from workers.proxies_constants import PROXY_LIST_FR
class LinkValidator(threading.Thread): class LinkValidator(threading.Thread):
def __init__(self, link_to_validate_list: list, def __init__(self, link_to_validate_list: list,
cookiesPublisher: CookiesPublisher, queue_to_listen=REQUEST_DATA_QUEUE): cookiesPublisher: CookiesPublisher, proxy_to_use, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR"):
super().__init__() super().__init__()
self.cookie = SimpleCookie() self.cookie = SimpleCookie()
self.cookiesPublisher = cookiesPublisher self.cookiesPublisher = cookiesPublisher
self.link_to_validate_list = link_to_validate_list self.link_to_validate_list = link_to_validate_list
self.queue_to_listen = queue_to_listen self.queue_to_listen = queue_to_listen
self.ip_country = ip_country
self.filter_with_ip_country()
self.proxy_to_use = proxy_to_use
# self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;' # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
def set_up_connection(self): def set_up_connection(self):
@@ -48,29 +51,32 @@ class LinkValidator(threading.Thread):
'Sec-Fetch-Site': 'same-origin', 'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Dest': 'document',
'Accept-Language': 'fr-FR,fr;q=0.6'} 'Accept-Language': 'fr-FR,fr;q=0.6'}
proxy_to_use = random.choice(PROXY_LIST) print(self.proxy_to_use)
print(proxy_to_use)
print("received cookie is " + str(self.cookie_str)) print("received cookie is " + str(self.cookie_str))
try: try:
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=proxy_to_use, response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
timeout=15) timeout=15)
print(response.status_code) print(response.status_code)
if response.status_code == 200: if response.status_code == 200:
_content = response.text
print(response.text) print(response.text)
print(response.url) if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content:
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo) print(response.url)
# set new cookies MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo)
_cookies_to_set = response.headers['set-cookie'] # set new cookies
self.cookie.load(_cookies_to_set) _cookies_to_set = response.headers['set-cookie']
new_cookies = {k: v.value for k, v in self.cookie.items()} self.cookie.load(_cookies_to_set)
new_coolies_str = "" new_cookies = {k: v.value for k, v in self.cookie.items()}
for key in new_cookies: new_coolies_str = ""
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";" for key in new_cookies:
print("will publish to queue {}".format(new_coolies_str)) new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
# upload the cookie to queue print("will publish to queue {}".format(new_coolies_str))
self.cookiesPublisher.publish_body(new_coolies_str) # upload the cookie to queue
self.cookie_str = new_coolies_str self.cookiesPublisher.publish_body(new_coolies_str)
return True self.cookie_str = new_coolies_str
return True
else:
return False
else: else:
return False return False
except Exception as error: except Exception as error:
@@ -80,6 +86,7 @@ class LinkValidator(threading.Thread):
def on_message(self, ch, method, properties, body): def on_message(self, ch, method, properties, body):
print(f" [x] Received {body}") print(f" [x] Received {body}")
self.link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate() self.link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
self.filter_with_ip_country()
self.cookie_str = body.decode("UTF-8") self.cookie_str = body.decode("UTF-8")
random.shuffle(self.link_to_validate_list) random.shuffle(self.link_to_validate_list)
if len(self.link_to_validate_list) > 0: if len(self.link_to_validate_list) > 0:
@@ -101,12 +108,34 @@ class LinkValidator(threading.Thread):
time.sleep(60) time.sleep(60)
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True) ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
def filter_with_ip_country(self):
_link_list_to_click = []
for _link in self.link_to_validate_list:
if _link.ip_country == self.ip_country:
_link_list_to_click.append(_link)
self.link_to_validate_list = _link_list_to_click
if __name__ == '__main__':
def validate_with_FR_ip():
_queue_name = REQUEST_DATA_QUEUE_TEST
link_list = MONGO_STORE_MANAGER.get_links_to_validate() link_list = MONGO_STORE_MANAGER.get_links_to_validate()
cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_TEST) cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
cookiesPublisher.set_up_connection() cookiesPublisher.set_up_connection()
receiver = LinkValidator(link_to_validate_list=link_list, cookiesPublisher=cookiesPublisher, print("filter links with ip_country")
queue_to_listen=REQUEST_DATA_QUEUE_TEST) _link_list_to_click = []
for _link in link_list:
if _link.ip_country == "FR":
_link_list_to_click.append(_link)
for _l in _link_list_to_click:
print(_l.ip_country)
_fr_proxy_to_use = random.choice(PROXY_LIST_FR)
receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
proxy_to_use=_fr_proxy_to_use,
queue_to_listen=_queue_name, ip_country="FR")
receiver.set_up_connection() receiver.set_up_connection()
receiver.listen_to_queue(receiver.on_message) receiver.listen_to_queue(receiver.on_message)
pass
if __name__ == '__main__':
validate_with_FR_ip()
+2 -1
View File
@@ -31,4 +31,5 @@ DE_PROXY_RES = {
# FR_PROXY_MOBILE # FR_PROXY_MOBILE
# FR_PROXY_RES # FR_PROXY_RES
# PROXY_LIST = [FR_PROXY_MOBILE, FR_PROXY_RES, DE_PROXY_RES, DE_PROXY_MOBILE, ES_PROXY_MOBILE, IT_PROXY_MOBILE] # PROXY_LIST = [FR_PROXY_MOBILE, FR_PROXY_RES, DE_PROXY_RES, DE_PROXY_MOBILE, ES_PROXY_MOBILE, IT_PROXY_MOBILE]
PROXY_LIST = [FR_PROXY_MOBILE, FR_PROXY_RES] PROXY_LIST_FR = [FR_PROXY_MOBILE, FR_PROXY_RES]
PROXY_LIST_DE = [DE_PROXY_RES, DE_PROXY_MOBILE]
+4 -3
View File
@@ -8,18 +8,19 @@ import requests
from db.mongo_manager import MONGO_STORE_MANAGER from db.mongo_manager import MONGO_STORE_MANAGER
from models.ReserveResultPojo import ReserveResultPojo, PublishType from models.ReserveResultPojo import ReserveResultPojo, PublishType
from queue_message.CookiesPublisher import CookiesPublisher from queue_message.CookiesPublisher import CookiesPublisher
from workers.proxies_constants import PROXY_LIST from workers.proxies_constants import PROXY_LIST_FR
class Sender: class Sender:
def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher): def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher, proxy_to_use):
self.store_type = "random" self.store_type = "random"
self.cookie = SimpleCookie() self.cookie = SimpleCookie()
self.cookiesPublisher = cookiesPublisher self.cookiesPublisher = cookiesPublisher
# self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;' # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
self.cookie_str = cookie_str self.cookie_str = cookie_str
self._csrf = None self._csrf = None
self.proxy_to_use = proxy_to_use
self.cookie.load(self.cookie_str) self.cookie.load(self.cookie_str)
def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str): def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str):
@@ -64,7 +65,7 @@ class Sender:
'passport_id': contact.passport, 'processing': 'on', 'cgu': 'on'} 'passport_id': contact.passport, 'processing': 'on', 'cgu': 'on'}
print(data) print(data)
try: try:
proxy_to_use = random.choice(PROXY_LIST) proxy_to_use = self.proxy_to_use
# proxy_to_use = PROXY_LIST[0] # proxy_to_use = PROXY_LIST[0]
print(proxy_to_use) print(proxy_to_use)
response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data, response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data,
+33
View File
@@ -0,0 +1,33 @@
import random
from db.mongo_manager import MONGO_STORE_MANAGER
from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher
from workers.link_validator import LinkValidator
from workers.proxies_constants import PROXY_LIST_DE
def validate_with_DE_ip():
_queue_name = REQUEST_DATA_QUEUE_DE
link_list = MONGO_STORE_MANAGER.get_links_to_validate()
cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
cookiesPublisher.set_up_connection()
print("filter links with ip_country")
_link_list_to_click = []
for _link in link_list:
if _link.ip_country == "DE":
_link_list_to_click.append(_link)
for _l in _link_list_to_click:
print(_l.ip_country)
# else:
# print(_link.ip_country)
_de_proxy_to_use = random.choice(PROXY_LIST_DE)
receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
proxy_to_use=_de_proxy_to_use,
queue_to_listen=_queue_name, ip_country="DE")
receiver.set_up_connection()
receiver.listen_to_queue(receiver.on_message)
pass
if __name__ == '__main__':
validate_with_DE_ip()