From 27411177f93625b63c72da2d6e0878a88f3b109d Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Fri, 5 Jan 2024 15:01:55 +0100 Subject: [PATCH] can use message queue --- request_sender.py | 120 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 89 insertions(+), 31 deletions(-) diff --git a/request_sender.py b/request_sender.py index d165e0f..1738513 100644 --- a/request_sender.py +++ b/request_sender.py @@ -9,12 +9,18 @@ from db.mongo_manager import MONGO_STORE_MANAGER from excel_reader import read_contacts from models.ReserveResultPojo import ReserveResultPojo, PublishType from models.contact_pojo import ContactPojo +from queue_message.receiver import Receiver FR_PROXY_MOBILE = { 'http': 'http://7039778-mobile-country-FR:2ji8e3r0nh@89.39.106.148:14806', 'https': 'http://7039778-mobile-country-FR:2ji8e3r0nh@89.39.106.148:14806' } +# FR_PROXY_MOBILE = { +# 'http': 'http://e99aff2d-409239:19elyiw1ge@93.190.141.102:31637', +# 'https': 'http://e99aff2d-409239:19elyiw1ge@93.190.141.102:31637' +# } + IPFIY = 'http://api.ipify.org' HERMES_REGISTER = "https://rendezvousparis.hermes.com/client/register" NGROK_TEST = "https://bcc6-193-164-156-53.ngrok-free.app" @@ -28,12 +34,27 @@ def is_already_sent(contact: ContactPojo) -> bool: return False +def filter_contacts(_contact_list: list) -> list: + already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day() + _contact_list_to_book = [] + for contact in _contact_list: + _to_add = True + for booked in already_sent_contacts: + if contact.mail == booked.email: + _to_add = False + if _to_add: + _contact_list_to_book.append(contact) + + return _contact_list_to_book + + class Sender: - def __init__(self): + def __init__(self, cookie_str): self.store_type = "random" self.cookie = SimpleCookie() - self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;' + # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;' + self.cookie_str = cookie_str self.cookie.load(self.cookie_str) def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str): @@ -53,7 +74,11 @@ class Sender: def send_request_for_list(self, contact_list: list): pass + def get_csrf(self): + pass + def send_request(self, url, contact: ContactPojo): + csrf = 'uWGIeylh-DjQFXI439Fc_ete5RcVA7eDsnQU' headers = {'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8', @@ -61,43 +86,76 @@ class Sender: 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Dest': 'document', 'Accept-Language': 'fr-FR,fr;q=0.6'} - data = {'check': '', '_csrf': 'SqI0z6xt-guBLrwEQLcqhTczFWh_EpcffX9w', 'prefer': '', + data = {'check': '', '_csrf': csrf, 'prefer': '', 'surname': contact.last_name.capitalize(), 'name': contact.first_name.capitalize(), 'phone_country': "FR", 'phone_number': contact.phone, 'email': contact.mail, 'passport_id': contact.passport, 'processing': 'on', 'cgu': 'on'} print(data) - response = requests.post(url=url, proxies=FR_PROXY_MOBILE, verify=False, headers=headers, data=data) - print(response.status_code) - if response.status_code == 200: - # add to mongodb - print(response.text) - print(response.url) - self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=response.url) - cookies_to_set = response.headers['set-cookie'] - self.cookie.load(cookies_to_set) - new_cookies = {k: v.value for k, v in self.cookie.items()} - new_coolies_str = "" - for key in new_cookies: - new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";" - print(new_coolies_str) - self.cookie_str = new_coolies_str - return True - else: + try: + response = requests.post(url=url, proxies=FR_PROXY_MOBILE, verify=False, headers=headers, data=data, + timeout=15) + print(response.status_code) + if response.status_code == 200: + # add to mongodb + print(response.text) + print(response.url) + self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=response.url) + cookies_to_set = response.headers['set-cookie'] + self.cookie.load(cookies_to_set) + new_cookies = {k: v.value for k, v in self.cookie.items()} + new_coolies_str = "" + for key in new_cookies: + new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";" + print(new_coolies_str) + self.cookie_str = new_coolies_str + return True + else: + return False + except Exception as Error: + print(Error) return False -if __name__ == '__main__': - sender = Sender() - contact_list = read_contacts('/Users/panlei/Desktop/yahoo_aol_valid_26-1.xlsx') - sub_contact_list = contact_list[200:300] - for con in sub_contact_list: - if not is_already_sent(con): - time.sleep(random.randint(2, 10)) +def on_message(ch, method, properties, body): + print(f" [x] Received {body}") + sender = Sender(body.decode("UTF-8")) + contact_list = read_contacts('/Users/panlei/Desktop/real_name_contacts_231_hotmail.xlsx') + # remove already booked contacts + sub_contact_list = filter_contacts(contact_list) + random.shuffle(sub_contact_list) + if len(sub_contact_list) > 0: + for con in sub_contact_list: + # if not is_already_sent(con): + print(con.mail) + time.sleep(random.randint(1, 5)) can_continue = sender.send_request(HERMES_REGISTER, con) if not can_continue: print("cannot continue") break - else: - print(con.mail + "--> skip") - print(con.mail) - # sender.send_request(HERMES_REGISTER, contact_list[-5]) + # else: + # print(con.mail + "--> skip") + ch.basic_ack(delivery_tag=method.delivery_tag) + else: + print("empty list") + + +if __name__ == '__main__': + receiver = Receiver() + receiver.set_up_connection() + receiver.listen_to_queue(on_message) + +# if __name__ == '__main__': +# sender = Sender() +# contact_list = read_contacts('/Users/panlei/Desktop/yahoo_aol_valid_26-1.xlsx') +# sub_contact_list = contact_list[200:300] +# for con in sub_contact_list: +# if not is_already_sent(con): +# time.sleep(random.randint(2, 10)) +# can_continue = sender.send_request(HERMES_REGISTER, con) +# if not can_continue: +# print("cannot continue") +# break +# else: +# print(con.mail + "--> skip") +# print(con.mail) +# sender.send_request(HERMES_REGISTER, contact_list[-5])