Merge branch 'develop' into feature/proxy_manager
This commit is contained in:
@@ -7,6 +7,7 @@ from pymongo import MongoClient
|
||||
from models.LinkPojo import LinkPojo
|
||||
from models.ReserveResultPojo import ReserveResultPojo
|
||||
from models.contact_pojo import ContactPojo
|
||||
from models.mail_pojo import MailAddress
|
||||
|
||||
MONGO_DB_URL = "mongo.lpaconsulting.fr"
|
||||
CAPTCHA_ERROR_COLLECTION_PREFIX = "CAPTCHA_ERROR_"
|
||||
@@ -72,6 +73,43 @@ class MongoDbManager:
|
||||
result_list.append(ContactPojo.from_firestore_dict(document))
|
||||
return result_list
|
||||
|
||||
def save_links_to_validate(self, link: str, mail_address: str, _all_contact_list: list):
|
||||
collection_to_use = self.db[LINKS_TO_VALIDATE]
|
||||
updated_at = time.strftime("%H:%M:%S", time.localtime())
|
||||
_ip_country = "FR"
|
||||
# find ip_country info
|
||||
for _contact in _all_contact_list:
|
||||
if _contact.mail == mail_address:
|
||||
_ip_country = _contact.ip_country
|
||||
|
||||
if len(mail_address) > 0:
|
||||
collection_to_use.replace_one(filter={'_id': mail_address, }, replacement={
|
||||
u'url': link,
|
||||
u'email': mail_address,
|
||||
u'ip_country': _ip_country,
|
||||
"updated_at": updated_at
|
||||
},
|
||||
upsert=True)
|
||||
else:
|
||||
collection_to_use.replace_one(filter={'_id': link, }, replacement={
|
||||
u'url': link,
|
||||
u'ip_country': _ip_country,
|
||||
"updated_at": updated_at
|
||||
},
|
||||
upsert=True)
|
||||
|
||||
def get_destination_emails(self) -> list:
|
||||
collection_name = DESTINATION_EMAIL_LIST
|
||||
email_list = []
|
||||
try:
|
||||
collection_to_use = self.db[collection_name]
|
||||
for document in collection_to_use.find():
|
||||
email_list.append(MailAddress.from_firestore_dict(document))
|
||||
except Exception as error:
|
||||
self.logger.info(error)
|
||||
print(error)
|
||||
return email_list
|
||||
|
||||
def get_links_to_validate(self) -> list:
|
||||
collection_name = LINKS_TO_VALIDATE
|
||||
link_list = []
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
{
|
||||
"opts": "endpoint,ajaxListenerPath",
|
||||
"ttst": 93,
|
||||
"ifov": false,
|
||||
"hc": 2,
|
||||
"br_oh": 663,
|
||||
"br_ow": 384,
|
||||
"ua": "Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36",
|
||||
"wbd": false,
|
||||
"tagpu": 9.185432887936322,
|
||||
"wdif": false,
|
||||
"wdifrm": false,
|
||||
"npmtm": false,
|
||||
"br_h": 663,
|
||||
"br_w": 384,
|
||||
"isf": true,
|
||||
"nddc": 1,
|
||||
"rs_h": 811,
|
||||
"rs_w": 384,
|
||||
"rs_cd": 24,
|
||||
"phe": false,
|
||||
"nm": false,
|
||||
"jsf": false,
|
||||
"lg": "fr-FR",
|
||||
"pr": 2.8125,
|
||||
"ars_h": 811,
|
||||
"ars_w": 384,
|
||||
"tz": -60,
|
||||
"str_ss": true,
|
||||
"str_ls": true,
|
||||
"str_idb": true,
|
||||
"str_odb": true,
|
||||
"plgod": false,
|
||||
"plg": 2,
|
||||
"plgne": "err",
|
||||
"plgre": "err",
|
||||
"plgof": "err",
|
||||
"plggt": "err",
|
||||
"pltod": false,
|
||||
"hcovdr": false,
|
||||
"hcovdr2": false,
|
||||
"plovdr": false,
|
||||
"plovdr2": false,
|
||||
"ftsovdr": false,
|
||||
"ftsovdr2": false,
|
||||
"lb": false,
|
||||
"eva": 33,
|
||||
"lo": false,
|
||||
"ts_mtp": 5,
|
||||
"ts_tec": true,
|
||||
"ts_tsa": true,
|
||||
"vnd": "Google Inc.",
|
||||
"bid": "NA",
|
||||
"mmt": "empty",
|
||||
"plu": "5gQIEKN,Iv26GLka",
|
||||
"hdn": false,
|
||||
"awe": false,
|
||||
"geb": false,
|
||||
"dat": false,
|
||||
"med": "defined",
|
||||
"aco": "probably",
|
||||
"acots": false,
|
||||
"acmp": "probably",
|
||||
"acmpts": true,
|
||||
"acw": "probably",
|
||||
"acwts": false,
|
||||
"acma": "maybe",
|
||||
"acmats": false,
|
||||
"acaa": "probably",
|
||||
"acaats": true,
|
||||
"ac3": "",
|
||||
"ac3ts": false,
|
||||
"acf": "probably",
|
||||
"acfts": false,
|
||||
"acmp4": "maybe",
|
||||
"acmp4ts": false,
|
||||
"acmp3": "probably",
|
||||
"acmp3ts": false,
|
||||
"acwm": "maybe",
|
||||
"acwmts": false,
|
||||
"ocpt": false,
|
||||
"vco": "",
|
||||
"vcots": false,
|
||||
"vch": "probably",
|
||||
"vchts": true,
|
||||
"vcw": "probably",
|
||||
"vcwts": true,
|
||||
"vc3": "maybe",
|
||||
"vc3ts": false,
|
||||
"vcmp": "",
|
||||
"vcmpts": false,
|
||||
"vcq": "",
|
||||
"vcqts": false,
|
||||
"vc1": "probably",
|
||||
"vc1ts": true,
|
||||
"dvm": 4,
|
||||
"sqt": false,
|
||||
"so": "portrait-primary",
|
||||
"wdw": true,
|
||||
"cokys": "bG9hZFRpbWVzY3NpL=",
|
||||
"ecpc": false,
|
||||
"lgs": true,
|
||||
"lgsod": false,
|
||||
"psn": true,
|
||||
"edp": false,
|
||||
"addt": true,
|
||||
"wsdc": true,
|
||||
"ccsr": true,
|
||||
"nuad": true,
|
||||
"bcda": true,
|
||||
"idn": true,
|
||||
"capi": false,
|
||||
"svde": false,
|
||||
"vpbq": true,
|
||||
"ucdv": false,
|
||||
"spwn": false,
|
||||
"emt": false,
|
||||
"bfr": false,
|
||||
"dbov": false,
|
||||
"cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2",
|
||||
"stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzMDIpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWIgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTA2MjIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxNzMz",
|
||||
"ckwa": true,
|
||||
"glrd": "Adreno (TM) 630",
|
||||
"glvd": "Qualcomm",
|
||||
"wwl": false,
|
||||
"jset": 1710598314
|
||||
}
|
||||
@@ -30,6 +30,7 @@ def read_contacts(file_name) -> list:
|
||||
first_name=first_name,
|
||||
passport_number=contact_dict['passport'],
|
||||
mail=contact_dict['email'])
|
||||
contact.store = contact_dict['store']
|
||||
contact_list.append(contact)
|
||||
return contact_list
|
||||
|
||||
|
||||
Executable
+106
@@ -0,0 +1,106 @@
|
||||
import imaplib
|
||||
|
||||
from imapclient import IMAPClient
|
||||
|
||||
DOMAIN_YAHOO = "yahoo.com"
|
||||
DOMAIN_SINA = "sina.com"
|
||||
DOMAIN_HOTMAIL = "hotmail.com"
|
||||
DOMAIN_TIM_IT = "tim.it"
|
||||
DOMAIN_163 = "163.com"
|
||||
DOMAIN_RAMBLER_RU = "rambler.ru"
|
||||
DOMAIN_ALICE_IT = "alice.it"
|
||||
DOMAIN_MARS_DTI_NE_JP = "mars.dti.ne.jp"
|
||||
DOMAN_BTVM_NE_JP = "btvm.ne.jp"
|
||||
DOMAN_AURORA_DTI_NE_JP = "aurora.dti.ne.jp"
|
||||
DOMAN_GMAIL = "gmail.com"
|
||||
DOMAIN_GMX = "gmx.com"
|
||||
DOMAIN_GMX_FR = "gmx.fr"
|
||||
DOMAIN_GMX_US = "gmx.us"
|
||||
DOMAIN_GMX_CH = "gmx.ch"
|
||||
DOMAIN_ONET = "onet.pl"
|
||||
DOMAIN_NAVER = "naver.com"
|
||||
DOMAIN_INBOX_LV = "inbox.lv"
|
||||
|
||||
DOMAIN_PISS_MAIL = "pissmail.com"
|
||||
DOMAIN_INCEL_EMAIL = "incel.email"
|
||||
DOMAIN_SHITPOSTING_EXPERT = "shitposting.expert"
|
||||
DOMAIN_HATESJE_WS = "hatesje.ws"
|
||||
DOMAIN_CHILD_PIZZA = "child.pizza"
|
||||
DOMAIN_GENOCIDE_FUN = "genocide.fun"
|
||||
DOMAIN_DMC_CHAT = "dmc.chat"
|
||||
|
||||
AOL_IMAP_SERVER = "imap.aol.com"
|
||||
IMAP_SERVER_163 = "imap.163.com"
|
||||
IMAP_SERVER_SINA = "imap.sina.com"
|
||||
YAHOO_IMAP_SERVER = "imap.mail.yahoo.com"
|
||||
HOTMAIL_IMAP_SERVER = "outlook.office365.com"
|
||||
RAMBLER_IMAP_SERVER = "imap.rambler.ru"
|
||||
ALICE_IMAP_SERVER = "in.alice.it"
|
||||
TIME_IT_SERVER = "imap.tim.it"
|
||||
MARS_DTI_NE_JP_SERVER = "imap.cm.dream.jp"
|
||||
NAVER_SERVER = "imap.naver.com"
|
||||
BTVM_NE_JP = "imap.btvm.ne.jp"
|
||||
SEREVER_GMAIL = "imap.gmail.com"
|
||||
SERVER_IMAGE_ONET = "imap.poczta.onet.pl"
|
||||
SERVER_GMX = "imap.gmx.com"
|
||||
SERVER_PISS_MAIL = "mail.pissmail.com"
|
||||
INBOX_LV = "mail.inbox.lv"
|
||||
|
||||
|
||||
def show_folders(imap) -> list:
|
||||
folders = []
|
||||
isImapClient = isinstance(imap, IMAPClient)
|
||||
if not isImapClient:
|
||||
for i in imap.list()[1]:
|
||||
l = i.decode().split(' "/" ')
|
||||
if len(l) > 1:
|
||||
folders.append(l[1])
|
||||
if len(folders) == 0:
|
||||
folders.append('INBOX')
|
||||
return folders
|
||||
else:
|
||||
list = imap.list_folders()
|
||||
for i in list:
|
||||
name = i[-1]
|
||||
folders.append(name)
|
||||
return folders
|
||||
|
||||
|
||||
def create_imap(login: str):
|
||||
# create an IMAP4 class with SSL
|
||||
if DOMAIN_163 in login:
|
||||
imap = IMAPClient(IMAP_SERVER_163, use_uid=True)
|
||||
elif DOMAIN_YAHOO in login:
|
||||
imap = imaplib.IMAP4_SSL(YAHOO_IMAP_SERVER)
|
||||
elif DOMAIN_GMX in login or DOMAIN_GMX_FR in login or DOMAIN_GMX_US in login or DOMAIN_GMX_CH in login:
|
||||
imap = imaplib.IMAP4_SSL(SERVER_GMX)
|
||||
elif DOMAIN_SINA in login:
|
||||
# imap = imaplib.IMAP4_SSL(IMAP_SERVER_SINA)
|
||||
imap = IMAPClient(IMAP_SERVER_SINA, use_uid=True)
|
||||
elif DOMAIN_HOTMAIL in login:
|
||||
imap = imaplib.IMAP4_SSL(HOTMAIL_IMAP_SERVER)
|
||||
elif DOMAIN_RAMBLER_RU in login:
|
||||
imap = imaplib.IMAP4_SSL(RAMBLER_IMAP_SERVER)
|
||||
elif DOMAN_BTVM_NE_JP in login:
|
||||
imap = imaplib.IMAP4_SSL(BTVM_NE_JP)
|
||||
elif DOMAN_GMAIL in login:
|
||||
imap = imaplib.IMAP4_SSL(SEREVER_GMAIL, port=993)
|
||||
elif DOMAIN_ONET in login:
|
||||
imap = IMAPClient(SERVER_IMAGE_ONET, use_uid=True)
|
||||
elif DOMAIN_TIM_IT in login:
|
||||
imap = imaplib.IMAP4(TIME_IT_SERVER)
|
||||
elif DOMAIN_ALICE_IT in login:
|
||||
imap = imaplib.IMAP4(ALICE_IMAP_SERVER, port=143)
|
||||
elif DOMAIN_MARS_DTI_NE_JP in login:
|
||||
imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143)
|
||||
elif DOMAN_AURORA_DTI_NE_JP in login:
|
||||
imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143)
|
||||
elif DOMAIN_NAVER in login:
|
||||
imap = imaplib.IMAP4_SSL(NAVER_SERVER, port=993)
|
||||
elif DOMAIN_INBOX_LV in login:
|
||||
imap = imaplib.IMAP4_SSL(INBOX_LV, port=993)
|
||||
elif DOMAIN_PISS_MAIL in login or DOMAIN_CHILD_PIZZA in login or DOMAIN_DMC_CHAT in login or DOMAIN_GENOCIDE_FUN in login or DOMAIN_HATESJE_WS in login or DOMAIN_INCEL_EMAIL in login or DOMAIN_SHITPOSTING_EXPERT in login:
|
||||
imap = imaplib.IMAP4_SSL(SERVER_PISS_MAIL, port=993)
|
||||
else:
|
||||
imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER)
|
||||
return imap
|
||||
Executable
+265
@@ -0,0 +1,265 @@
|
||||
import datetime
|
||||
import email
|
||||
import re
|
||||
from builtins import list
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import time
|
||||
from email.header import decode_header
|
||||
from email.message import Message
|
||||
|
||||
from imapclient import IMAPClient
|
||||
from db.mongo_manager import MONGO_STORE_MANAGER
|
||||
from mail.mail_constants import DOMAIN_HOTMAIL, create_imap
|
||||
from models.mail_pojo import MailPojo
|
||||
|
||||
VALIDATION_URL_SUBJECT_fr = 'Validation de votre demande de rendez-vous'
|
||||
VALIDATION_URL_SUBJECT_EN = 'Please confirm your appointment request'
|
||||
VALIDATION_URL_REGEX = """https:\/\/rendezvousparis.hermes.com\/client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+"""
|
||||
PART_VALIDATION_URL_REGEX = """client\/register\/[A-Z0-9]+\/validate.code=[A-Z0-9]+"""
|
||||
HERMES_EMAIL = "no-reply@hermes.com"
|
||||
EMAIL_ADDRESS_REGEX = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b'
|
||||
|
||||
date_format = "%d-%b-%Y" # DD-Mon-YYYY e.g., 3-Mar-2014
|
||||
REDIRECTION_MAILS = "appointment2022@aol.com, chenpeijun@aol.com,hongjiang176@aol.com,ciyuexie@aol.com,rutger.62@aol.com,ciccidaniel@aol.com,armasgoodman@aol.com,wknd.gemerine@aol.com,rafmail1981@aol.com,tonovichivanenaki@aol.com,hetland.ari@aol.com,mateusiversen@aol.com,lacerdaraffaello@aol.com,anasida76@aol.com,liamolinari@aol.com,sen70zib@aol.com,mezeiderrick@aol.com,stanisl49avchic@aol.com,damcvrobaneuron@aol.com,suyzanna_fleona@aol.com,dxealing.dissa@aol.com,hogg.karen@aol.com,obocharovamarina@aol.com,buchholzjohann@aol.com,orn.cecchini@aol.com,percivaltorgersen@aol.com,candalgudrun@aol.com,filimonis.76@aol.com,bengann_100@aol.com,axelhanne@aol.com,tiffanylarochelle@aol.com,nicoleta.r@aol.com,eichenbaum.1963@aol.com,kotensasharev@aol.com,samognat32@aol.com,edem_headshot@aol.com,kozmakuzmich1960@aol.com,damonsvensson@aol.com,anders.riva@aol.com,caiminwei123@gmail.com,yulingguo086@gmail.com,yingxiaolu086@gmail.com,lijiazhen0035@gmail.com,fangp370@gmail.com,huangyayu10086@gmail.com,fuziyuan110@gmail.com,xinyingdu886@gmail.com,yasiaforever.1971@aol.com,lukaszfidalgo@aol.com,zaichi29@aol.com,prostotakitak.1974@aol.com,mo90nroe@aol.com,blonde.87@aol.com,dimidrol.1969@aol.com"
|
||||
|
||||
|
||||
def check_email_address(email):
|
||||
# pass the regular expression
|
||||
# and the string into the fullmatch() method
|
||||
if (re.fullmatch(EMAIL_ADDRESS_REGEX, email)):
|
||||
print("Valid Email")
|
||||
return True
|
||||
else:
|
||||
print("Invalid Email")
|
||||
return False
|
||||
|
||||
|
||||
def find_from_mail(param):
|
||||
from_address, encoded_algo = param[0]
|
||||
if isinstance(from_address, bytes):
|
||||
from_address = from_address.decode(encoded_algo)
|
||||
if not check_email_address(from_address) and len(param) == 2:
|
||||
from_address, new_encode = param[1]
|
||||
if new_encode is None:
|
||||
new_encode = encoded_algo
|
||||
if isinstance(from_address, bytes):
|
||||
from_address = from_address.decode(new_encode)
|
||||
return from_address.strip(" ").strip(">").strip("<")
|
||||
return from_address.strip(" ").strip(">").strip("<")
|
||||
|
||||
|
||||
class MailReader():
|
||||
def __init__(self, login, password):
|
||||
self.login = login
|
||||
self.password = password
|
||||
|
||||
@staticmethod
|
||||
def show_folders(imap) -> list:
|
||||
folders = []
|
||||
isImapClient = isinstance(imap, IMAPClient)
|
||||
if not isImapClient:
|
||||
for i in imap.list()[1]:
|
||||
l = i.decode().split(' "/" ')
|
||||
folders.append(l[1])
|
||||
return folders
|
||||
else:
|
||||
list = imap.list_folders()
|
||||
for i in list:
|
||||
name = i[-1]
|
||||
folders.append(name)
|
||||
return folders
|
||||
|
||||
def read_emails(self, mails_messages: list) -> list:
|
||||
imap = create_imap(self.login)
|
||||
isImapClient = isinstance(imap, IMAPClient)
|
||||
print("isImapClient is " + str(isImapClient))
|
||||
if isImapClient:
|
||||
# authenticate
|
||||
dat = imap.login(self.login, str(self.password))
|
||||
print("type is {} for {}".format(dat, self.login))
|
||||
else:
|
||||
responseType, dat = imap.login(self.login, str(self.password))
|
||||
print("type is {} for {}".format(responseType, self.login))
|
||||
|
||||
mail_list = []
|
||||
print("read mails from {}".format(self.login))
|
||||
if not isImapClient:
|
||||
folder_list = self.show_folders(imap)
|
||||
for folder in folder_list:
|
||||
print("folder is {}".format(folder))
|
||||
mail_list.extend(self._get_messages_from_folder(imap, subject=VALIDATION_URL_SUBJECT_fr,
|
||||
folder=folder))
|
||||
mail_list.extend(self._get_messages_from_folder(imap, subject=VALIDATION_URL_SUBJECT_EN,
|
||||
folder=folder))
|
||||
else:
|
||||
folder_list = self.show_folders(imap)
|
||||
for folder in folder_list:
|
||||
print("folder is " + folder)
|
||||
mail_list.extend(self._get_messages_from_folder_for_imapclient(imap, folder=folder))
|
||||
if DOMAIN_HOTMAIL in self.login:
|
||||
mail_list.extend(
|
||||
self._get_messages_from_folder_for_imapclient(imap, folder="Junk"))
|
||||
if not isImapClient:
|
||||
imap.close()
|
||||
imap.logout()
|
||||
mails_messages.extend(mail_list)
|
||||
return mail_list
|
||||
|
||||
def _get_messages_from_folder(self, imap, subject, folder="INBOX") -> list:
|
||||
imap.select(folder)
|
||||
mail_messages = []
|
||||
typ, data = imap.search(None, '(SUBJECT "{}" SINCE "{}")'.format(subject,
|
||||
datetime.datetime.today().strftime(
|
||||
date_format)))
|
||||
for i in data[0].split():
|
||||
# fetch the email message by ID
|
||||
res, msg = imap.fetch(i.decode("utf-8"), "(RFC822)")
|
||||
body = ''
|
||||
for response in msg:
|
||||
if isinstance(response, tuple):
|
||||
# parse a bytes email into a message object
|
||||
msg = email.message_from_bytes(response[1])
|
||||
# decode the email subject
|
||||
subject, subject_encoded = decode_header(msg["Subject"])[0]
|
||||
received_date = msg["Date"]
|
||||
if isinstance(subject, bytes):
|
||||
# if it's a bytes, decode to str
|
||||
subject = subject.decode(subject_encoded)
|
||||
# decode email sender
|
||||
from_address = find_from_mail(decode_header(msg.get("From")))
|
||||
to_email = find_from_mail(decode_header(msg.get("To")))
|
||||
print("Email:", self.login)
|
||||
print("From:", from_address)
|
||||
print("To:", to_email)
|
||||
print("Subject:", subject)
|
||||
# if the email message is multipart
|
||||
if msg.is_multipart():
|
||||
# iterate over email parts
|
||||
for part in msg.walk():
|
||||
try:
|
||||
# get the email body
|
||||
payloads = part.get_payload()
|
||||
if isinstance(payloads, list):
|
||||
for payload in payloads:
|
||||
if isinstance(payload, Message):
|
||||
body = body + payload.get_payload(decode=True).decode("iso-8859-1")
|
||||
# print(body)
|
||||
except Exception as Error:
|
||||
print(Error)
|
||||
else:
|
||||
body = msg.get_payload(decode=True).decode()
|
||||
print(body)
|
||||
if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject:
|
||||
mail = MailPojo(subject=subject, body=body, from_address=from_address)
|
||||
if to_email is None:
|
||||
mail.to_address = self.login
|
||||
else:
|
||||
mail.to_address = to_email
|
||||
mail.mail_address = self.login
|
||||
mail_messages.append(mail)
|
||||
return mail_messages
|
||||
|
||||
def _get_messages_from_folder_for_imapclient(self, imap, folder="INBOX") -> list:
|
||||
mail_messages = []
|
||||
search_terms = 'SINCE "{}"'.format(
|
||||
datetime.datetime.today().strftime(
|
||||
date_format))
|
||||
print("search terms is " + search_terms)
|
||||
imap.select_folder(folder)
|
||||
messages = imap.search(['SINCE', datetime.datetime.today()])
|
||||
print("%d messages from our best friend" % len(messages))
|
||||
for uid, message_data in imap.fetch(messages, 'RFC822').items():
|
||||
try:
|
||||
email_message = email.message_from_bytes(message_data[b'RFC822'])
|
||||
from_address = email_message.get('FROM')
|
||||
subject = email_message.get('subject')
|
||||
# print("{}, {},{}".format(from_address, subject, email_message))
|
||||
body = ""
|
||||
if "no-reply@hermes.com" in from_address or "appointment2022@aol.com":
|
||||
for part in email_message.walk():
|
||||
print(part.get_content_type())
|
||||
if part.get_content_type() == "text/html":
|
||||
body = body + part.get_payload(decode=True).decode("utf-8")
|
||||
elif part.get_content_type() == "text/plain":
|
||||
body = body + part.get_payload()
|
||||
if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject:
|
||||
mail = MailPojo(subject=subject, body=body, from_address=from_address)
|
||||
mail.isImapClient = True
|
||||
print("email is {}".format(self.login))
|
||||
print("body is {}".format(body))
|
||||
print("subject is {}".format(subject))
|
||||
mail_messages.append(mail)
|
||||
except Exception as error:
|
||||
print(error)
|
||||
print("error trying to read email_Message for {}".format(self.login))
|
||||
return mail_messages
|
||||
|
||||
|
||||
def need_to_valid_url(url: str, successful_items) -> bool:
|
||||
# return True
|
||||
# if len(successful_items) == 0:
|
||||
# return False
|
||||
print("url is :" + url)
|
||||
parts = url.split('/')
|
||||
id = parts[5]
|
||||
if len(id) == 6:
|
||||
if id == "CS93VB":
|
||||
print("found")
|
||||
for item in successful_items:
|
||||
if item.id == id:
|
||||
if item.url_validated is not None:
|
||||
return not item.url_validated
|
||||
else:
|
||||
# if url_validated is None
|
||||
if item.url_validated is not None:
|
||||
return not item.url_validated
|
||||
return True
|
||||
return True
|
||||
else:
|
||||
print("id not valid:{}".format(id))
|
||||
return False
|
||||
|
||||
|
||||
def need_to_check_email(mail: str, successful_items) -> bool:
|
||||
print("successful_items size is " + str(len(successful_items)))
|
||||
filtered_items = list(filter(lambda item: item.email == mail, successful_items))
|
||||
# has validated value
|
||||
if len(filtered_items) > 0:
|
||||
validated_items = list(filter(
|
||||
lambda filtered_item: filtered_item.url_validated is not None and filtered_item.url_validated is True,
|
||||
filtered_items))
|
||||
if len(validated_items) > 0:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
else:
|
||||
return True
|
||||
|
||||
|
||||
def find_links_to_validate_from_mail_list(mail_list: list):
|
||||
# check time before start checking emails
|
||||
contact_to_book_list = MONGO_STORE_MANAGER.get_all_contact_to_book_list()
|
||||
successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
|
||||
mails_messages = []
|
||||
with ThreadPoolExecutor(max_workers=len(mail_list)) as executor:
|
||||
for mail in mail_list:
|
||||
# check whether we need to read mail
|
||||
if need_to_check_email(mail.mail, successful_items):
|
||||
mail_reader = MailReader(mail.mail, mail.password)
|
||||
executor.submit(mail_reader.read_emails, mails_messages)
|
||||
# for mail in mail_list:
|
||||
# # check whether we need to read mail
|
||||
# if need_to_check_email(mail.mail, successful_items):
|
||||
# mail_reader = MailReader(mail.mail, mail.password)
|
||||
# mail_reader.read_emails(mails_messages)
|
||||
|
||||
for mail in mails_messages:
|
||||
match = re.search(VALIDATION_URL_REGEX, mail.body)
|
||||
if match:
|
||||
url = match.group(0)
|
||||
if need_to_valid_url(url, successful_items):
|
||||
print("need to validate url: " + url)
|
||||
MONGO_STORE_MANAGER.save_links_to_validate(url, mail.to_address,
|
||||
_all_contact_list=contact_to_book_list)
|
||||
else:
|
||||
print("do not need to click url --> {}".format(mail.mail_address))
|
||||
@@ -10,6 +10,7 @@ class ContactPojo:
|
||||
mail: str
|
||||
ccid: str
|
||||
position: int
|
||||
store: str
|
||||
note: str
|
||||
|
||||
def __init__(self, phone_number: str, passport_number: str, last_name: str, first_name: str, mail: str,
|
||||
@@ -31,6 +32,7 @@ class ContactPojo:
|
||||
u'last_name': self.last_name,
|
||||
u'first_name': self.first_name,
|
||||
u'mail': self.mail,
|
||||
u'store': self.store,
|
||||
u'ccid': self.ccid,
|
||||
u'position': self.position
|
||||
}
|
||||
@@ -39,13 +41,15 @@ class ContactPojo:
|
||||
|
||||
@staticmethod
|
||||
def from_firestore_dict(source):
|
||||
ccid = source['ccid']
|
||||
phone = source['phone']
|
||||
position = source['position']
|
||||
passport = source['passport']
|
||||
email = source['mail']
|
||||
last_name = source['last_name']
|
||||
first_name = source['first_name']
|
||||
result = ContactPojo(ccid=ccid, phone_number=phone, passport_number=passport, position=position, mail=email,
|
||||
ip_country = "FR"
|
||||
if source.get('ip_country'):
|
||||
ip_country = source['ip_country']
|
||||
result = ContactPojo(phone_number=phone, passport_number=passport, mail=email,
|
||||
last_name=last_name, first_name=first_name)
|
||||
result.ip_country = ip_country
|
||||
return result
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import json
|
||||
|
||||
from models.jsdata_pojo import JsDataPojo
|
||||
|
||||
|
||||
# "log2": true,
|
||||
# "dcok": ".hermes.com",
|
||||
# "m_fmi": false,
|
||||
# "tbce": 0,
|
||||
# "es_sigmdn": null,
|
||||
# "es_mumdn": null,
|
||||
# "es_distmdn": null,
|
||||
# "es_angsmdn": null,
|
||||
# "es_angemdn": null,
|
||||
# "m_s_c": 0,
|
||||
# "m_m_c": 3,
|
||||
# "m_c_c": 3,
|
||||
# "m_cm_r": 1,
|
||||
# "m_ms_r": -1
|
||||
class JsDataLeTypePojo(JsDataPojo):
|
||||
|
||||
def __init__(self, glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu,
|
||||
vnd, pr, ts_mtp, dvm, m_s_c, m_m_c, m_c_c, m_cm_r, m_ms_r):
|
||||
super().__init__(glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu,
|
||||
vnd, pr, ts_mtp, dvm)
|
||||
self.log2 = True
|
||||
self.dcok = ".hermes.com"
|
||||
self.m_fmi = False
|
||||
self.tbce = 0
|
||||
self.es_sigmdn = None
|
||||
self.es_mumdn = None
|
||||
self.es_distmdn = None
|
||||
self.es_angsmdn = None
|
||||
self.es_angemdn = None
|
||||
self.m_s_c = m_s_c
|
||||
self.m_m_c = m_m_c
|
||||
self.m_c_c = m_c_c
|
||||
self.m_cm_r = m_cm_r
|
||||
self.m_ms_r = m_ms_r
|
||||
self.cfpfe = "RXJyb3I6IENhbm5vdCByZWFkIHByb3BlcnRpZXMgb2YgbnVsbA==" # 4.25.1
|
||||
self.stcfp = "ckFsbCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo3NjQ3MykKICAgIGF0IEhUTUxEb2N1bWVudC5yIChodHRwczovL3JlbmRlenZvdXNwYXJpcy5oZXJtZXMuY29tL2Rpc3QvdmVuZG9yLWM2Mjk1ZTljMjg3ZGExNzYwODllLmpzOjI6ODg2Nzkp" # 4.25.1
|
||||
|
||||
|
||||
|
||||
test_data_json = """{"glvd": "Qualcomm", "glrd": "Adreno (TM) 630", "hc": 2, "br_oh": 663, "br_ow": 384, "br_h": 663,
|
||||
"br_w": 384, "rs_h": 811, "rs_w": 384, "rs_cd": 24, "ars_h": 811, "ars_w": 384, "plg": 2, "eva": 33,
|
||||
"vnd": "Google Inc.", "plu": ["5gQIEKN", "Iv26GLka"],
|
||||
"ua": "Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36",
|
||||
"dvm": 4, "acw": "probably", "pr": 2.8125, "ts_mtp": 5,
|
||||
"cookiesStr": "app.sig=Arp0SzXCTBF2ordHf3pPTd6cS14;datadome=o_7W8TjHcQjqvnFjFBGAZZ0OmNOd0fnJXfzgqOajAHBBNv7zNMFi2~Jn7wQbTEIZdr2qvg0IFFQQr~Zk5ifzM154V1XxZVhgDZeVkNyRKrQewLBIzVeSagWs587fBqdX;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoieVVRWmJBWnlRTnQ5bFlNbDdNN3R6eHhqIn0=;policy=accepted;lang=fr;"}"""
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_data_dict = json.loads(test_data_json)
|
||||
le_js_data = JsDataLeTypePojo(glrd=test_data_dict['glrd'], glvd=test_data_dict['glvd'], hc=test_data_dict['hc'],
|
||||
ua=test_data_dict['ua'], br_oh=test_data_dict['br_oh'], br_ow=test_data_dict['br_ow'],
|
||||
ars_h=test_data_dict['ars_h'], ars_w=test_data_dict['ars_w'], pr=test_data_dict['pr'],
|
||||
plg=test_data_dict['plg'], br_h=test_data_dict['br_h'], br_w=test_data_dict['br_w'],
|
||||
plu=test_data_dict['plu'], vnd=test_data_dict['vnd'], dvm=test_data_dict['dvm'],
|
||||
ts_mtp=test_data_dict['ts_mtp'], eva=test_data_dict['eva'],
|
||||
rs_h=test_data_dict['rs_h'],
|
||||
rs_w=test_data_dict['rs_w'], rs_cd=test_data_dict['rs_cd'], m_s_c=0, m_m_c=3, m_c_c=3,
|
||||
m_cm_r=1, m_ms_r=-1)
|
||||
print(le_js_data.to_url_encoded_json())
|
||||
@@ -9,7 +9,7 @@ import jsonpickle
|
||||
|
||||
@dataclass
|
||||
class JsDataPojo:
|
||||
opts: str
|
||||
# opts: str
|
||||
ttst: int
|
||||
ifov: bool
|
||||
hc: int
|
||||
@@ -137,8 +137,8 @@ class JsDataPojo:
|
||||
|
||||
def __init__(self, glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu,
|
||||
vnd, pr, ts_mtp, dvm):
|
||||
self.opts = "endpoint,ajaxListenerPath"
|
||||
self.ttst = random.randint(38, 94)
|
||||
# self.opts = "endpoint,ajaxListenerPath"
|
||||
self.ttst = random.randint(38, 148)
|
||||
self.ifov = False
|
||||
self.hc = hc
|
||||
self.br_oh = br_oh
|
||||
@@ -166,7 +166,7 @@ class JsDataPojo:
|
||||
self.pr = pr
|
||||
self.ars_h = ars_h
|
||||
self.ars_w = ars_w
|
||||
self.tz = -60
|
||||
self.tz = -120
|
||||
self.str_ss = True
|
||||
self.str_ls = True
|
||||
self.str_idb = True
|
||||
@@ -236,8 +236,8 @@ class JsDataPojo:
|
||||
self.vc1ts = True
|
||||
self.dvm = dvm
|
||||
self.set_default_values()
|
||||
self.glrd = glrd
|
||||
self.glvd = glvd
|
||||
self.glrd = glrd
|
||||
self.wwl = False
|
||||
self.jset = int(time.time())
|
||||
|
||||
@@ -265,8 +265,8 @@ class JsDataPojo:
|
||||
self.emt = False
|
||||
self.bfr = False
|
||||
self.dbov = False
|
||||
self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.24.1
|
||||
self.stcfp = "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzMDIpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWIgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTA2MjIpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxNzMz" # 4.24.1
|
||||
self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.25.1
|
||||
self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzY0NzMpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hYl0gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTAxMTgpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUyMjY2" # 4.25.1
|
||||
self.ckwa = True
|
||||
|
||||
def to_url_encoded_json(self):
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
class MailAddress:
|
||||
def __init__(self, mail, password):
|
||||
self.mail = mail
|
||||
self.password = password
|
||||
|
||||
def __repr__(self):
|
||||
return "邮箱:{}, 密码:{}".format(self.mail, self.password)
|
||||
|
||||
def to_firestore_dict(self):
|
||||
dest = {
|
||||
u'mail': self.mail,
|
||||
u'password': self.password
|
||||
}
|
||||
return dest
|
||||
|
||||
@staticmethod
|
||||
def from_firestore_dict(source):
|
||||
password = source['password']
|
||||
mail = source['mail']
|
||||
return MailAddress(mail=mail, password=password)
|
||||
|
||||
|
||||
class MailPojo:
|
||||
from_address: str
|
||||
to_address: str
|
||||
body: str
|
||||
subject: str
|
||||
mail_address: str = ""
|
||||
isImapClient = False
|
||||
|
||||
def __init__(self, from_address, body, subject):
|
||||
self.body = body
|
||||
self.subject = subject
|
||||
self.from_address = from_address
|
||||
self.isImapClient = False
|
||||
self.to_address = ""
|
||||
@@ -8,3 +8,4 @@ class RequestResult(Enum):
|
||||
COOKIES_ERROR = "COOKIES_ERROR"
|
||||
UNKNOWN = "UNKNOWN"
|
||||
BAD_GATEWAY = "BAD_GATEWAY"
|
||||
CTRF_ERROR = "CTRF_ERROR"
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import datetime
|
||||
import logging
|
||||
import sys
|
||||
from threading import Thread
|
||||
|
||||
from db.mongo_manager import MONGO_STORE_MANAGER
|
||||
from excel_reader import read_contacts
|
||||
from models.contact_pojo import ContactPojo
|
||||
from queue_message.CookiesPublisher import CookiesPublisher, SHARED_OBJECT, TEST_QUEUE
|
||||
from queue_message.appointmentrequestsender import AppointmentRequestSender
|
||||
from queue_message.parallel_requestsender import ParallelRequestSender
|
||||
from utiles import is_time_between
|
||||
from utils.AppLogging import init_logger
|
||||
from workers.proxies_constants import MOBILE_PROXY_LIST_FR
|
||||
|
||||
IPFIY = 'http://api.ipify.org'
|
||||
NGROK_TEST = "https://bcc6-193-164-156-53.ngrok-free.app"
|
||||
|
||||
|
||||
def is_already_sent(contact: ContactPojo) -> bool:
|
||||
already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
|
||||
for required_contact in already_sent_contacts:
|
||||
if contact.mail == required_contact.email:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def filter_contacts(_contact_list: list) -> list:
|
||||
already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
|
||||
_link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
|
||||
_contact_list_to_book = []
|
||||
for contact in _contact_list:
|
||||
_to_add = True
|
||||
for booked in already_sent_contacts:
|
||||
if contact.mail == booked.email:
|
||||
_to_add = False
|
||||
# 如果已经收到链接了，就不要再请求
|
||||
for link_to_validate in _link_to_validate_list:
|
||||
if contact.mail == link_to_validate.email:
|
||||
logger.info("{}: link already received".format(contact.mail))
|
||||
_to_add = False
|
||||
if _to_add:
|
||||
_contact_list_to_book.append(contact)
|
||||
|
||||
return _contact_list_to_book
|
||||
|
||||
|
||||
def is_open():
|
||||
return is_time_between(datetime.time(10, 30), datetime.time(19, 00))
|
||||
|
||||
|
||||
count = 0
|
||||
init_logger()
|
||||
logger = logging.getLogger()
|
||||
|
||||
logger.addHandler(logging.StreamHandler(stream=sys.stdout))
|
||||
|
||||
|
||||
def send_appointment_request(message_queue_name, _contact_list):
|
||||
global count
|
||||
count = count + 1
|
||||
for _contact in _contact_list:
|
||||
logger.info(_contact)
|
||||
_cookiesPublisher = CookiesPublisher(queue_name=message_queue_name)
|
||||
_cookiesPublisher.set_up_connection()
|
||||
receiver = ParallelRequestSender(sub_contact_list=_contact_list, proxy_to_use_list=MOBILE_PROXY_LIST_FR,
|
||||
queue_name=message_queue_name, just_send=True,
|
||||
cookiesPublisher=_cookiesPublisher, logger=logger)
|
||||
print("count is " + str(count))
|
||||
receiver.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
contacts_file_path = '~/Desktop/31_03_to_test.xlsx'
|
||||
_contact_list = read_contacts(contacts_file_path)[0:20]
|
||||
_contact_list_to_book = filter_contacts(_contact_list)
|
||||
_segment_number = 1
|
||||
logger.info("{} contacts to book".format(len(_contact_list_to_book)))
|
||||
last_thread = None
|
||||
for i in range(0, _segment_number):
|
||||
logger.info("segment is {}".format(i))
|
||||
_step = int(len(_contact_list_to_book) / _segment_number)
|
||||
_sublist = _contact_list_to_book[i * _step:_step * (i + 1)]
|
||||
_thread1 = Thread(target=send_appointment_request, args=(TEST_QUEUE, _sublist))
|
||||
last_thread = _thread1
|
||||
_thread1.start()
|
||||
last_thread.join()
|
||||
@@ -5,8 +5,11 @@ REQUEST_DATA_QUEUE = 'REQUEST_DATA'
|
||||
# REQUEST_DATA_QUEUE_TEST = 'REQUEST_DATA_TEST'
|
||||
REQUEST_DATA_QUEUE_TEST = 'REQUEST_DATA_TEST'
|
||||
TEST_QUEUE = 'TEST_QUEUE'
|
||||
SHARED_OBJECT = 'SHARED_OBJECT'
|
||||
MORNING_DATA_CACHE = 'MORNING_DATA_CACHE'
|
||||
REQUEST_DATA_QUEUE_DE = 'REQUEST_DATA_DE'
|
||||
REQUEST_DATA_OBJECT = 'REQUEST_DATA_OBJECT'
|
||||
REGISTER_QUEUE = 'REGISTER_QUEUE'
|
||||
credentials = pika.PlainCredentials('appointment', 'ZyuhJZ2xEYWhElhpJjy7YEpZGZwNYJz2fHIu')
|
||||
|
||||
|
||||
|
||||
@@ -3,12 +3,14 @@ import json
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from http.cookies import SimpleCookie
|
||||
|
||||
import pika
|
||||
|
||||
from db.mongo_manager import MONGO_STORE_MANAGER
|
||||
from mail.mail_reader_all_contacts import find_links_to_validate_from_mail_list
|
||||
from models.ReserveResultPojo import ReserveResultPojo
|
||||
from models.contact_pojo import ContactPojo
|
||||
from models.jsdata_le_pojo import JsDataLeTypePojo
|
||||
from models.jsdata_pojo import JsDataPojo
|
||||
from models.result_pojo import RequestResult
|
||||
from queue_message.CookiesPublisher import CookiesPublisher
|
||||
@@ -30,63 +32,53 @@ def is_already_sent(contact: ContactPojo) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def filter_contacts(_contact_list: list) -> list:
|
||||
def filter_contacts(_contact_list: list, provided_list=[]) -> list:
|
||||
already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
|
||||
already_sent_contacts.extend(provided_list)
|
||||
_link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
|
||||
_contact_list_to_book = []
|
||||
for contact in _contact_list:
|
||||
_to_add = True
|
||||
for booked in already_sent_contacts:
|
||||
if isinstance(booked, ReserveResultPojo):
|
||||
if contact.mail == booked.email:
|
||||
_to_add = False
|
||||
else:
|
||||
if contact.mail == booked.mail:
|
||||
_to_add = False
|
||||
# 如果已经收到链接了，就不要再请求
|
||||
for link_to_validate in _link_to_validate_list:
|
||||
if contact.mail == link_to_validate.email:
|
||||
print("{}: link already received".format(contact.mail))
|
||||
_to_add = False
|
||||
if _to_add:
|
||||
_contact_list_to_book.append(contact)
|
||||
|
||||
return _contact_list_to_book
|
||||
|
||||
|
||||
def get_valid_csrf() -> str:
|
||||
captchaResultGetter = CaptchaResultGetter()
|
||||
_valid_cookie = captchaResultGetter.get_valid_cookie()
|
||||
# while _valid_cookie is None:
|
||||
# _valid_cookie = captchaResultGetter.get_valid_cookie()
|
||||
new_csrf = None
|
||||
while new_csrf is None:
|
||||
valid_cookie = None
|
||||
if _valid_cookie is not None:
|
||||
simple_cookie = SimpleCookie()
|
||||
simple_cookie.load(_valid_cookie)
|
||||
new_cookies = {k: v.value for k, v in simple_cookie.items()}
|
||||
new_coolies_str = ""
|
||||
for key in new_cookies:
|
||||
print(key)
|
||||
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
|
||||
print(new_coolies_str)
|
||||
valid_cookie = new_coolies_str + "app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;policy=accepted;lang=fr;"
|
||||
print(valid_cookie)
|
||||
new_csrf = captchaResultGetter.get_csrf(valid_cookie)
|
||||
if new_csrf is None:
|
||||
_valid_cookie = None
|
||||
while _valid_cookie is None:
|
||||
_valid_cookie = captchaResultGetter.get_valid_cookie()
|
||||
time.sleep(2)
|
||||
return new_csrf
|
||||
|
||||
|
||||
def is_open():
|
||||
return is_time_between(datetime.time(10, 30), datetime.time(19, 00))
|
||||
|
||||
|
||||
class AppointmentRequestSender(threading.Thread):
|
||||
def __init__(self, sub_contact_list: list, proxy_to_use_list, cookiesPublisher: CookiesPublisher,
|
||||
def __init__(self, sub_contact_list: list, proxy_to_use_list, logger, cookiesPublisher: CookiesPublisher,
|
||||
just_send=False,
|
||||
queue_name=REQUEST_DATA_QUEUE):
|
||||
super().__init__()
|
||||
self.connection = None
|
||||
self.just_send = just_send
|
||||
self.logger = logger
|
||||
self.already_tried_contact_list = []
|
||||
self.cookiesPublisher = cookiesPublisher
|
||||
self.channel = None
|
||||
self.valid_csrf = None
|
||||
self.list_to_retrieve_mails = sub_contact_list
|
||||
self.initial_contact_list = sub_contact_list
|
||||
self.contact_list = sub_contact_list
|
||||
self.queue_name = queue_name
|
||||
self.proxy_to_use_list = proxy_to_use_list
|
||||
self.already_read_emails = False
|
||||
|
||||
def set_up_connection(self):
|
||||
self.connection = pika.BlockingConnection(
|
||||
@@ -94,16 +86,22 @@ class AppointmentRequestSender(threading.Thread):
|
||||
self.channel = self.connection.channel()
|
||||
|
||||
def listen_to_queue(self, callback):
|
||||
print("listen to queue {}".format(self.queue_name))
|
||||
self.logger.info("listen to queue {}".format(self.queue_name))
|
||||
self.channel.basic_qos(prefetch_count=1)
|
||||
self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback)
|
||||
self.channel.start_consuming()
|
||||
|
||||
def on_message(self, ch, method, properties, body):
|
||||
_message_count = self.cookiesPublisher.message_count()
|
||||
print("message count in queue is {}".format(_message_count))
|
||||
self.logger.info("message count in queue is {}".format(_message_count))
|
||||
_received_object = body.decode("UTF-8")
|
||||
print(f" [x] Received {_received_object}")
|
||||
self.logger.info(f" [x] Received {_received_object}")
|
||||
_port = random.randint(40001, 49999)
|
||||
self.logger.info("generated port is {}".format(_port))
|
||||
_chosen_proxy = random.choice(self.proxy_to_use_list)
|
||||
_proxy_to_use = {}
|
||||
_proxy_to_use["http"] = _chosen_proxy["http"].format(_port)
|
||||
_proxy_to_use["https"] = _chosen_proxy["https"].format(_port)
|
||||
if "glrd" in _received_object:
|
||||
_received_dict = json.loads(_received_object)
|
||||
js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'],
|
||||
@@ -116,55 +114,128 @@ class AppointmentRequestSender(threading.Thread):
|
||||
rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'])
|
||||
_received_cookies = _received_dict["cookiesStr"]
|
||||
sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict,
|
||||
proxy_to_use=random.choice(self.proxy_to_use_list))
|
||||
proxy_to_use=_proxy_to_use, logger=self.logger)
|
||||
# remove already sent contacts
|
||||
if self.just_send:
|
||||
self.contact_list = filter_contacts(self.contact_list, self.already_tried_contact_list)
|
||||
else:
|
||||
self.contact_list = filter_contacts(self.contact_list)
|
||||
# remove already booked contacts
|
||||
random.shuffle(self.contact_list)
|
||||
if len(self.contact_list) > 0 and is_open():
|
||||
captchaResultGetter = CaptchaResultGetter()
|
||||
print("contact number is {}".format(len(self.contact_list)))
|
||||
self.contact_list = filter_contacts(self.contact_list)
|
||||
self.logger.info("contact number is {}".format(len(self.contact_list)))
|
||||
# self.contact_list = filter_contacts(self.contact_list)
|
||||
for con in self.contact_list:
|
||||
# if not is_already_sent(con):
|
||||
print(con.mail)
|
||||
# time.sleep(random.randint(1, 5))
|
||||
self.logger.info(con.mail)
|
||||
if self.valid_csrf is None:
|
||||
self.valid_csrf = captchaResultGetter.get_csrf(
|
||||
proxy_to_use=random.choice(self.proxy_to_use_list),
|
||||
cookie=body.decode("UTF-8"))
|
||||
csrf_result = captchaResultGetter.get_csrf(
|
||||
proxy_to_use=_proxy_to_use, js_data=js_data,
|
||||
cookie=_received_cookies)
|
||||
if isinstance(csrf_result, str):
|
||||
self.valid_csrf = csrf_result
|
||||
else:
|
||||
self.logger.info("csrf is {}".format(csrf_result))
|
||||
if csrf_result == RequestResult.BLOCKED:
|
||||
break
|
||||
_new_cookies = captchaResultGetter.get_valid_ch_cookie(sender.proxy_to_use, js_data,
|
||||
old_valid_cookie=_received_cookies)
|
||||
if _received_cookies is not None:
|
||||
print("new cookie is " + _received_cookies)
|
||||
sender.cookie_str = _received_cookies
|
||||
time.sleep(random.randint(1, 5))
|
||||
can_continue = sender.send_request(HERMES_REGISTER, con, csrf=self.valid_csrf)
|
||||
if _new_cookies is not None:
|
||||
self.logger.info("new cookie is " + _new_cookies)
|
||||
time.sleep(random.randint(1, 3))
|
||||
# m_s_c = f.scroll
|
||||
m_s_c = random.randint(0, 3)
|
||||
m_c_c = random.randint(3, 5) # click count
|
||||
m_m_c = random.randint(3, 5) # move count
|
||||
m_cm_r = m_c_c / m_m_c
|
||||
m_ms_r = random.randint(-1, 1)
|
||||
|
||||
js_le_data = JsDataLeTypePojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'],
|
||||
hc=_received_dict['hc'],
|
||||
ua=_received_dict['ua'], br_oh=_received_dict['br_oh'],
|
||||
br_ow=_received_dict['br_ow'],
|
||||
ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'],
|
||||
pr=_received_dict['pr'],
|
||||
plg=_received_dict['plg'], br_h=_received_dict['br_h'],
|
||||
br_w=_received_dict['br_w'],
|
||||
plu=_received_dict['plu'], vnd=_received_dict['vnd'],
|
||||
dvm=_received_dict['dvm'],
|
||||
ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'],
|
||||
rs_h=_received_dict['rs_h'],
|
||||
rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'],
|
||||
m_s_c=m_s_c, m_m_c=m_m_c, m_c_c=m_c_c,
|
||||
m_cm_r=m_cm_r, m_ms_r=m_ms_r)
|
||||
time.sleep(random.randint(1, 4))
|
||||
_new_le_cookies = captchaResultGetter.get_le_valid_cookie(proxy_to_use=_proxy_to_use,
|
||||
js_le_type_data=js_le_data,
|
||||
old_valid_cookie=_new_cookies)
|
||||
if _new_le_cookies is not None:
|
||||
# self.logger.info("new le type cookie is " + _new_le_cookies)
|
||||
sender.cookie_str = _new_le_cookies
|
||||
time.sleep(random.randint(1, 3))
|
||||
self.already_tried_contact_list.append(con)
|
||||
can_continue = sender.send_request(HERMES_REGISTER, js_data, con, csrf=self.valid_csrf)
|
||||
if can_continue == RequestResult.SUCCESS:
|
||||
# 让服务器读取成功的约会
|
||||
try:
|
||||
self.logger.info("try to remove success contact from list to retrieve mails")
|
||||
self.list_to_retrieve_mails.remove(con)
|
||||
except Exception as e:
|
||||
self.logger.info(
|
||||
"exception while remove success contact from list to retrieve mails")
|
||||
print(e)
|
||||
else:
|
||||
can_continue = RequestResult.COOKIES_ERROR
|
||||
else:
|
||||
can_continue = RequestResult.COOKIES_ERROR
|
||||
if can_continue == RequestResult.BLOCKED:
|
||||
print("cannot continue, valid_csrf is " + str(self.valid_csrf))
|
||||
self.logger.info("BLOCKED, valid_csrf is " + str(self.valid_csrf))
|
||||
break
|
||||
elif can_continue == RequestResult.PROXY_ERROR:
|
||||
print("PROXY_ERROR, will not reset valid_csrf")
|
||||
self.logger.info("PROXY_ERROR, will not reset valid_csrf")
|
||||
elif can_continue == RequestResult.COOKIES_ERROR:
|
||||
print("COOKIES_ERROR, will not reset valid_csrf")
|
||||
self.logger.info("COOKIES_ERROR, will not reset valid_csrf")
|
||||
else:
|
||||
print("can continue, will reset valid_csrf")
|
||||
self.logger.info("can continue, will reset valid_csrf")
|
||||
self.valid_csrf = None
|
||||
time.sleep(random.randint(1, 2))
|
||||
print("will ack method.delivery_tag: " + str(method.delivery_tag))
|
||||
self.logger.info("will ack method.delivery_tag: " + str(method.delivery_tag))
|
||||
ch.basic_ack(delivery_tag=method.delivery_tag)
|
||||
else:
|
||||
print("empty list")
|
||||
time.sleep(120)
|
||||
print("will basic_reject method.delivery_tag: " + str(method.delivery_tag))
|
||||
self.retrieve_invalidate_urls()
|
||||
self.logger.info("empty list")
|
||||
time.sleep(60)
|
||||
self.logger.info("will basic_reject method.delivery_tag: " + str(method.delivery_tag))
|
||||
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
|
||||
else:
|
||||
print("not a valid object")
|
||||
self.logger.info("not a valid object")
|
||||
ch.basic_ack(delivery_tag=method.delivery_tag)
|
||||
|
||||
def run(self):
|
||||
print(threading.currentThread().name + " starts")
|
||||
self.logger.info(threading.currentThread().name + " starts")
|
||||
self.set_up_connection()
|
||||
self.listen_to_queue(self.on_message)
|
||||
self.channel.start_consuming()
|
||||
|
||||
def retrieve_invalidate_urls(self):
|
||||
if not self.already_read_emails and len(self.list_to_retrieve_mails) > 0:
|
||||
self.logger.info("will retrieve validate urls")
|
||||
time.sleep(30)
|
||||
_mail_list = MONGO_STORE_MANAGER.get_destination_emails()
|
||||
_mail_list_filtered = []
|
||||
for mail in _mail_list:
|
||||
for _contact in self.list_to_retrieve_mails:
|
||||
if _contact.mail == mail.mail:
|
||||
_mail_list_filtered.append(mail)
|
||||
self.logger.info("will call find_links_to_validate_from_mail_list, size = " + str(len(_mail_list_filtered)))
|
||||
find_links_to_validate_from_mail_list(_mail_list_filtered)
|
||||
self.already_read_emails = True
|
||||
else:
|
||||
self.logger.info("already read emails, is there any contacts to use")
|
||||
self.logger.info("reset already_tried_contact_list")
|
||||
self.already_tried_contact_list = []
|
||||
self.contact_list = filter_contacts(self.initial_contact_list, self.already_tried_contact_list)
|
||||
self.logger.info("contact_list size is " + str(len(self.contact_list)))
|
||||
if len(self.contact_list) > 0:
|
||||
self.logger.info("set already_read_emails to False")
|
||||
self.already_read_emails = False
|
||||
|
||||
@@ -36,34 +36,6 @@ def filter_contacts(_contact_list: list) -> list:
|
||||
return _contact_list_to_book
|
||||
|
||||
|
||||
def get_valid_csrf() -> str:
|
||||
captchaResultGetter = CaptchaResultGetter()
|
||||
_valid_cookie = captchaResultGetter.get_valid_cookie()
|
||||
# while _valid_cookie is None:
|
||||
# _valid_cookie = captchaResultGetter.get_valid_cookie()
|
||||
new_csrf = None
|
||||
while new_csrf is None:
|
||||
valid_cookie = None
|
||||
if _valid_cookie is not None:
|
||||
simple_cookie = SimpleCookie()
|
||||
simple_cookie.load(_valid_cookie)
|
||||
new_cookies = {k: v.value for k, v in simple_cookie.items()}
|
||||
new_coolies_str = ""
|
||||
for key in new_cookies:
|
||||
print(key)
|
||||
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
|
||||
print(new_coolies_str)
|
||||
valid_cookie = new_coolies_str + "app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;policy=accepted;lang=fr;"
|
||||
print(valid_cookie)
|
||||
new_csrf = captchaResultGetter.get_csrf(valid_cookie)
|
||||
if new_csrf is None:
|
||||
_valid_cookie = None
|
||||
while _valid_cookie is None:
|
||||
_valid_cookie = captchaResultGetter.get_valid_cookie()
|
||||
time.sleep(2)
|
||||
return new_csrf
|
||||
|
||||
|
||||
class LinkValidatorReceiver(threading.Thread):
|
||||
def __init__(self, linkpojo_list: list):
|
||||
self.connection = None
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
import datetime
|
||||
import json
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
import pika
|
||||
|
||||
from db.mongo_manager import MONGO_STORE_MANAGER
|
||||
from mail.mail_reader_all_contacts import find_links_to_validate_from_mail_list
|
||||
from models.ReserveResultPojo import ReserveResultPojo
|
||||
from models.contact_pojo import ContactPojo
|
||||
from models.jsdata_le_pojo import JsDataLeTypePojo
|
||||
from models.jsdata_pojo import JsDataPojo
|
||||
from models.result_pojo import RequestResult
|
||||
from queue_message.CookiesPublisher import CookiesPublisher
|
||||
from queue_message.appointmentrequestsender import filter_contacts, is_open
|
||||
from utiles import is_time_between
|
||||
from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
|
||||
from workers.sender import Sender
|
||||
|
||||
QUEUE_HOST = "appointment.lpaconsulting.fr"
|
||||
REQUEST_DATA_QUEUE = 'REQUEST_DATA'
|
||||
credentials = pika.PlainCredentials('appointment', 'ZyuhJZ2xEYWhElhpJjy7YEpZGZwNYJz2fHIu')
|
||||
|
||||
|
||||
def is_already_sent(contact: ContactPojo) -> bool:
|
||||
already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
|
||||
for required_contact in already_sent_contacts:
|
||||
if contact.mail == required_contact.email:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class ParallelRequestSender(threading.Thread):
|
||||
def __init__(self, sub_contact_list: list, proxy_to_use_list, logger, cookiesPublisher: CookiesPublisher,
|
||||
just_send=False,
|
||||
queue_name=REQUEST_DATA_QUEUE):
|
||||
super().__init__()
|
||||
self.connection = None
|
||||
self.just_send = just_send
|
||||
self.logger = logger
|
||||
self.already_tried_contact_list = []
|
||||
self.cookiesPublisher = cookiesPublisher
|
||||
self.channel = None
|
||||
self.valid_csrf = None
|
||||
self.list_to_retrieve_mails = sub_contact_list
|
||||
self.contact_list = sub_contact_list
|
||||
self.queue_name = queue_name
|
||||
self.proxy_to_use_list = proxy_to_use_list
|
||||
self.already_read_emails = False
|
||||
|
||||
def set_up_connection(self):
|
||||
self.connection = pika.BlockingConnection(
|
||||
pika.ConnectionParameters(host=QUEUE_HOST, port=5672, credentials=credentials))
|
||||
self.channel = self.connection.channel()
|
||||
|
||||
def listen_to_queue(self, callback):
|
||||
self.logger.info("listen to queue {}".format(self.queue_name))
|
||||
self.channel.basic_qos(prefetch_count=1)
|
||||
self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback)
|
||||
self.channel.start_consuming()
|
||||
|
||||
def send_request(self, _received_cookies, _received_dict, js_data: JsDataPojo, logger,
|
||||
_contact) -> RequestResult:
|
||||
_proxy_to_use = self.generate_proxy()
|
||||
logger.info("send_request for contact: {}, cookies: {}".format(_contact.mail, _received_cookies))
|
||||
logger.info("proxy to use is {}".format(_proxy_to_use))
|
||||
sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict,
|
||||
proxy_to_use=_proxy_to_use, logger=logger)
|
||||
# remove already sent contacts
|
||||
if is_open():
|
||||
captchaResultGetter = CaptchaResultGetter()
|
||||
_new_cookies = captchaResultGetter.get_valid_ch_cookie(sender.proxy_to_use, js_data,
|
||||
old_valid_cookie=_received_cookies)
|
||||
# self.contact_list = filter_contacts(self.contact_list)
|
||||
logger.info(_contact.mail)
|
||||
valid_csrf = captchaResultGetter.get_csrf(
|
||||
proxy_to_use=_proxy_to_use, js_data=js_data,
|
||||
cookie=_new_cookies)
|
||||
if isinstance(valid_csrf, str):
|
||||
if _new_cookies is not None:
|
||||
logger.info("new cookie is " + _new_cookies)
|
||||
# m_s_c = f.scroll
|
||||
m_s_c = random.randint(0, 3)
|
||||
m_c_c = random.randint(3, 5) # click count
|
||||
m_m_c = random.randint(3, 5) # move count
|
||||
m_cm_r = m_c_c / m_m_c
|
||||
m_ms_r = random.randint(-1, 1)
|
||||
|
||||
js_le_data = JsDataLeTypePojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'],
|
||||
hc=_received_dict['hc'],
|
||||
ua=_received_dict['ua'], br_oh=_received_dict['br_oh'],
|
||||
br_ow=_received_dict['br_ow'],
|
||||
ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'],
|
||||
pr=_received_dict['pr'],
|
||||
plg=_received_dict['plg'], br_h=_received_dict['br_h'],
|
||||
br_w=_received_dict['br_w'],
|
||||
plu=_received_dict['plu'], vnd=_received_dict['vnd'],
|
||||
dvm=_received_dict['dvm'],
|
||||
ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'],
|
||||
rs_h=_received_dict['rs_h'],
|
||||
rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'],
|
||||
m_s_c=m_s_c, m_m_c=m_m_c, m_c_c=m_c_c,
|
||||
m_cm_r=m_cm_r, m_ms_r=m_ms_r)
|
||||
time.sleep(random.randint(1, 4))
|
||||
_new_le_cookies = captchaResultGetter.get_le_valid_cookie(proxy_to_use=_proxy_to_use,
|
||||
js_le_type_data=js_le_data,
|
||||
old_valid_cookie=_new_cookies)
|
||||
if _new_le_cookies is not None:
|
||||
# self.logger.info("new le type cookie is " + _new_le_cookies)
|
||||
sender.cookie_str = _new_le_cookies
|
||||
time.sleep(random.randint(1, 3))
|
||||
self.already_tried_contact_list.append(_contact)
|
||||
can_continue = sender.send_request(HERMES_REGISTER, js_data, _contact, csrf=valid_csrf)
|
||||
if can_continue == RequestResult.SUCCESS:
|
||||
# 让服务器读取成功的约会
|
||||
try:
|
||||
self.logger.info("try to remove success contact from list to retrieve mails")
|
||||
self.list_to_retrieve_mails.remove(_contact)
|
||||
except Exception as e:
|
||||
self.logger.info(
|
||||
"exception while remove success contact from list to retrieve mails")
|
||||
print(e)
|
||||
else:
|
||||
can_continue = RequestResult.COOKIES_ERROR
|
||||
else:
|
||||
can_continue = RequestResult.COOKIES_ERROR
|
||||
if can_continue == RequestResult.BLOCKED:
|
||||
self.logger.info("cannot continue, we are blocked " + str(self.valid_csrf))
|
||||
elif can_continue == RequestResult.PROXY_ERROR:
|
||||
self.logger.info("PROXY_ERROR, will not reset valid_csrf")
|
||||
elif can_continue == RequestResult.COOKIES_ERROR:
|
||||
self.logger.info("COOKIES_ERROR, will not reset valid_csrf")
|
||||
else:
|
||||
self.logger.info("can continue, will reset valid_csrf")
|
||||
self.valid_csrf = None
|
||||
return can_continue
|
||||
else:
|
||||
return valid_csrf
|
||||
# return RequestResult.CTRF_ERROR
|
||||
|
||||
def getChTypeJsDataFromDict(self, _received_dict) -> JsDataPojo:
|
||||
return JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'],
|
||||
ua=_received_dict['ua'], br_oh=_received_dict['br_oh'], br_ow=_received_dict['br_ow'],
|
||||
ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'], pr=_received_dict['pr'],
|
||||
plg=_received_dict['plg'], br_h=_received_dict['br_h'], br_w=_received_dict['br_w'],
|
||||
plu=_received_dict['plu'], vnd=_received_dict['vnd'], dvm=_received_dict['dvm'],
|
||||
ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'],
|
||||
rs_h=_received_dict['rs_h'],
|
||||
rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'])
|
||||
|
||||
def generate_proxy(self):
|
||||
_port = random.randint(40001, 49999)
|
||||
_chosen_proxy = random.choice(self.proxy_to_use_list)
|
||||
self.logger.info("generated port is {}".format(_port))
|
||||
_proxy_to_use = {}
|
||||
_proxy_to_use["http"] = _chosen_proxy["http"].format(_port)
|
||||
_proxy_to_use["https"] = _chosen_proxy["https"].format(_port)
|
||||
return _proxy_to_use
|
||||
|
||||
def on_message(self, ch, method, properties, body):
|
||||
_message_count = self.cookiesPublisher.message_count()
|
||||
self.logger.info("message count in queue is {}".format(_message_count))
|
||||
# prepare the contact list
|
||||
if self.just_send:
|
||||
self.contact_list = filter_contacts(self.contact_list, self.already_tried_contact_list)
|
||||
else:
|
||||
self.contact_list = filter_contacts(self.contact_list)
|
||||
# remove already booked contacts
|
||||
random.shuffle(self.contact_list)
|
||||
_received_object = body.decode("UTF-8")
|
||||
self.logger.info(f" [x] Received {_received_object}")
|
||||
step = 5
|
||||
_received_dict = json.loads(_received_object)
|
||||
js_data = self.getChTypeJsDataFromDict(_received_dict)
|
||||
_received_cookies = _received_dict["cookiesStr"]
|
||||
if len(self.contact_list) > step:
|
||||
_sub_list = self.contact_list[0:step]
|
||||
result = None
|
||||
for con in _sub_list:
|
||||
with ThreadPoolExecutor(max_workers=step) as executor:
|
||||
result = executor.submit(self.send_request, _received_cookies, _received_dict, js_data, self.logger,
|
||||
con)
|
||||
self.logger.info("result is: " + str(result.result()))
|
||||
if result.result() == RequestResult.SUCCESS:
|
||||
self.logger.info("Success for {}, with cookies{}".format(con.mail, _received_cookies))
|
||||
if result.result() == RequestResult.BLOCKED or result.result() == RequestResult.CTRF_ERROR:
|
||||
ch.basic_ack(delivery_tag=method.delivery_tag)
|
||||
|
||||
else:
|
||||
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
|
||||
else:
|
||||
self.retrieve_invalidate_urls()
|
||||
self.logger.info("empty list")
|
||||
time.sleep(120)
|
||||
self.logger.info("will basic_reject method.delivery_tag: " + str(method.delivery_tag))
|
||||
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
|
||||
|
||||
def run(self):
|
||||
self.logger.info(threading.currentThread().name + " starts")
|
||||
self.set_up_connection()
|
||||
self.listen_to_queue(self.on_message)
|
||||
self.channel.start_consuming()
|
||||
|
||||
def retrieve_invalidate_urls(self):
|
||||
if not self.already_read_emails and len(self.list_to_retrieve_mails) > 0:
|
||||
self.logger.info("will retrieve validate urls")
|
||||
time.sleep(30)
|
||||
_mail_list = MONGO_STORE_MANAGER.get_destination_emails()
|
||||
_mail_list_filtered = []
|
||||
for mail in _mail_list:
|
||||
for _contact in self.list_to_retrieve_mails:
|
||||
if _contact.mail == mail.mail:
|
||||
_mail_list_filtered.append(mail)
|
||||
self.logger.info("will call find_links_to_validate_from_mail_list, size = " + str(len(_mail_list_filtered)))
|
||||
find_links_to_validate_from_mail_list(_mail_list_filtered)
|
||||
self.already_read_emails = True
|
||||
else:
|
||||
self.logger.info("already read emails, will not retrieve validate urls")
|
||||
+36
-47
@@ -1,19 +1,16 @@
|
||||
import datetime
|
||||
import random
|
||||
import time
|
||||
from http.cookies import SimpleCookie
|
||||
import logging
|
||||
import sys
|
||||
from threading import Thread
|
||||
|
||||
from db.mongo_manager import MONGO_STORE_MANAGER
|
||||
from excel_reader import read_contacts
|
||||
from models.contact_pojo import ContactPojo
|
||||
from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_DE, REQUEST_DATA_QUEUE_TEST, \
|
||||
REQUEST_DATA_OBJECT, TEST_QUEUE
|
||||
from queue_message.appointmentrequestsender import AppointmentRequestSender, REQUEST_DATA_QUEUE
|
||||
from queue_message.CookiesPublisher import CookiesPublisher, SHARED_OBJECT, TEST_QUEUE
|
||||
from queue_message.appointmentrequestsender import AppointmentRequestSender
|
||||
from utiles import is_time_between
|
||||
from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER
|
||||
from workers.proxies_constants import PROXY_LIST_DE, PROXY_LIST_FR, MOBILE_PROXY_LIST_FR
|
||||
from workers.sender import Sender
|
||||
from utils.AppLogging import init_logger
|
||||
from workers.proxies_constants import MOBILE_PROXY_LIST_FR
|
||||
|
||||
IPFIY = 'http://api.ipify.org'
|
||||
NGROK_TEST = "https://bcc6-193-164-156-53.ngrok-free.app"
|
||||
@@ -29,74 +26,66 @@ def is_already_sent(contact: ContactPojo) -> bool:
|
||||
|
||||
def filter_contacts(_contact_list: list) -> list:
|
||||
already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
|
||||
_link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
|
||||
_contact_list_to_book = []
|
||||
for contact in _contact_list:
|
||||
_to_add = True
|
||||
for booked in already_sent_contacts:
|
||||
if contact.mail == booked.email:
|
||||
_to_add = False
|
||||
# 如果已经收到链接了，就不要再请求
|
||||
for link_to_validate in _link_to_validate_list:
|
||||
if contact.mail == link_to_validate.email:
|
||||
logger.info("{}: link already received".format(contact.mail))
|
||||
_to_add = False
|
||||
if _to_add:
|
||||
_contact_list_to_book.append(contact)
|
||||
|
||||
return _contact_list_to_book
|
||||
|
||||
|
||||
def get_valid_csrf() -> str:
|
||||
captchaResultGetter = CaptchaResultGetter()
|
||||
_valid_cookie = captchaResultGetter.get_valid_cookie()
|
||||
# while _valid_cookie is None:
|
||||
# _valid_cookie = captchaResultGetter.get_valid_cookie()
|
||||
new_csrf = None
|
||||
while new_csrf is None and is_open():
|
||||
valid_cookie = None
|
||||
if _valid_cookie is not None:
|
||||
simple_cookie = SimpleCookie()
|
||||
simple_cookie.load(_valid_cookie)
|
||||
new_cookies = {k: v.value for k, v in simple_cookie.items()}
|
||||
new_coolies_str = ""
|
||||
for key in new_cookies:
|
||||
print(key)
|
||||
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
|
||||
print(new_coolies_str)
|
||||
valid_cookie = new_coolies_str + "app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;policy=accepted;lang=fr;"
|
||||
print(valid_cookie)
|
||||
new_csrf = captchaResultGetter.get_csrf(valid_cookie)
|
||||
if new_csrf is None:
|
||||
_valid_cookie = None
|
||||
while _valid_cookie is None:
|
||||
_valid_cookie = captchaResultGetter.get_valid_cookie()
|
||||
time.sleep(2)
|
||||
return new_csrf
|
||||
|
||||
|
||||
def is_open():
|
||||
return is_time_between(datetime.time(10, 30), datetime.time(19, 00))
|
||||
|
||||
|
||||
count = 0
|
||||
init_logger()
|
||||
logger = logging.getLogger()
|
||||
|
||||
logger.addHandler(logging.StreamHandler(stream=sys.stdout))
|
||||
|
||||
|
||||
def send_appointment_request(message_queue_name, _contact_list):
|
||||
global count
|
||||
count = count + 1
|
||||
for _contact in _contact_list:
|
||||
logger.info(_contact)
|
||||
_cookiesPublisher = CookiesPublisher(queue_name=message_queue_name)
|
||||
_cookiesPublisher.set_up_connection()
|
||||
receiver = AppointmentRequestSender(sub_contact_list=_contact_list, proxy_to_use_list=MOBILE_PROXY_LIST_FR,
|
||||
queue_name=message_queue_name,
|
||||
cookiesPublisher=_cookiesPublisher)
|
||||
queue_name=message_queue_name, just_send=True,
|
||||
cookiesPublisher=_cookiesPublisher, logger=logger)
|
||||
print("count is " + str(count))
|
||||
receiver.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
contacts_file_path = '~/Desktop/contact_list_2024-03-09.xlsx'
|
||||
def start_send_requests():
|
||||
print("start send requests")
|
||||
contacts_file_path = '~/Desktop/contact_list_2024-04-16_2.xlsx'
|
||||
_contact_list = read_contacts(contacts_file_path)
|
||||
_contact_list_to_book = filter_contacts(_contact_list)
|
||||
_segment_number = 10
|
||||
logger.info("{} contacts to book".format(len(_contact_list_to_book)))
|
||||
last_thread = None
|
||||
for i in range(0, _segment_number):
|
||||
print(i)
|
||||
_step = int(len(_contact_list) / _segment_number)
|
||||
_sublist = _contact_list[i:_step * (i + 1)]
|
||||
print(_sublist[0])
|
||||
print(_sublist[-1])
|
||||
_thread1 = Thread(target=send_appointment_request, args=(REQUEST_DATA_OBJECT, _sublist))
|
||||
logger.info("segment is {}".format(i))
|
||||
_step = int(len(_contact_list_to_book) / _segment_number)
|
||||
_sublist = _contact_list_to_book[i * _step:_step * (i + 1)]
|
||||
_thread1 = Thread(target=send_appointment_request, args=(SHARED_OBJECT, _sublist))
|
||||
last_thread = _thread1
|
||||
_thread1.start()
|
||||
last_thread.join()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
start_send_requests()
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import datetime
|
||||
import logging
|
||||
import sys
|
||||
from threading import Thread
|
||||
|
||||
from db.mongo_manager import MONGO_STORE_MANAGER
|
||||
from excel_reader import read_contacts
|
||||
from models.contact_pojo import ContactPojo
|
||||
from queue_message.CookiesPublisher import CookiesPublisher, SHARED_OBJECT, TEST_QUEUE, MORNING_DATA_CACHE
|
||||
from queue_message.appointmentrequestsender import AppointmentRequestSender
|
||||
from utiles import is_time_between
|
||||
from utils.AppLogging import init_logger
|
||||
from workers.proxies_constants import MOBILE_PROXY_LIST_FR
|
||||
|
||||
IPFIY = 'http://api.ipify.org'
|
||||
NGROK_TEST = "https://bcc6-193-164-156-53.ngrok-free.app"
|
||||
|
||||
|
||||
def is_already_sent(contact: ContactPojo) -> bool:
|
||||
already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
|
||||
for required_contact in already_sent_contacts:
|
||||
if contact.mail == required_contact.email:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def filter_contacts(_contact_list: list) -> list:
|
||||
already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
|
||||
_link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
|
||||
_contact_list_to_book = []
|
||||
for contact in _contact_list:
|
||||
_to_add = True
|
||||
for booked in already_sent_contacts:
|
||||
if contact.mail == booked.email:
|
||||
_to_add = False
|
||||
# 如果已经收到链接了，就不要再请求
|
||||
for link_to_validate in _link_to_validate_list:
|
||||
if contact.mail == link_to_validate.email:
|
||||
logger.info("{}: link already received".format(contact.mail))
|
||||
_to_add = False
|
||||
if _to_add:
|
||||
_contact_list_to_book.append(contact)
|
||||
|
||||
return _contact_list_to_book
|
||||
|
||||
|
||||
def is_open():
|
||||
return is_time_between(datetime.time(10, 30), datetime.time(19, 00))
|
||||
|
||||
|
||||
count = 0
|
||||
init_logger()
|
||||
logger = logging.getLogger()
|
||||
|
||||
logger.addHandler(logging.StreamHandler(stream=sys.stdout))
|
||||
|
||||
|
||||
def send_appointment_request(message_queue_name, _contact_list):
|
||||
global count
|
||||
count = count + 1
|
||||
for _contact in _contact_list:
|
||||
logger.info(_contact)
|
||||
_cookiesPublisher = CookiesPublisher(queue_name=message_queue_name)
|
||||
_cookiesPublisher.set_up_connection()
|
||||
receiver = AppointmentRequestSender(sub_contact_list=_contact_list, proxy_to_use_list=MOBILE_PROXY_LIST_FR,
|
||||
queue_name=message_queue_name, just_send=True,
|
||||
cookiesPublisher=_cookiesPublisher, logger=logger)
|
||||
print("count is " + str(count))
|
||||
receiver.run()
|
||||
|
||||
|
||||
def start_send_requests():
|
||||
print("start send requests")
|
||||
contacts_file_path = '~/Desktop/contact_list_merged.xlsx'
|
||||
_contact_list = read_contacts(contacts_file_path)
|
||||
_contact_list_to_book = filter_contacts(_contact_list)
|
||||
_segment_number = 20
|
||||
logger.info("{} contacts to book".format(len(_contact_list_to_book)))
|
||||
last_thread = None
|
||||
for i in range(0, _segment_number):
|
||||
logger.info("segment is {}".format(i))
|
||||
_step = int(len(_contact_list_to_book) / _segment_number)
|
||||
_sublist = _contact_list_to_book[i * _step:_step * (i + 1)]
|
||||
_thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE, _sublist))
|
||||
last_thread = _thread1
|
||||
_thread1.start()
|
||||
last_thread.join()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
start_send_requests()
|
||||
@@ -0,0 +1,30 @@
|
||||
from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor
|
||||
from apscheduler.schedulers.blocking import BlockingScheduler
|
||||
|
||||
from request_sender import start_send_requests
|
||||
|
||||
|
||||
def start_book_appointment():
|
||||
start_send_requests()
|
||||
|
||||
|
||||
def start_check_results_job(sched):
|
||||
sched.add_job(start_book_appointment, 'cron', day_of_week='mon-sat', hour='10',
|
||||
minute='30',
|
||||
misfire_grace_time=10,
|
||||
second='0', timezone='Europe/Paris', max_instances=1, args=[])
|
||||
|
||||
|
||||
def config_and_start_jobs():
|
||||
executors = {
|
||||
'default': ThreadPoolExecutor(30),
|
||||
'processpool': ProcessPoolExecutor(12)
|
||||
}
|
||||
sched = BlockingScheduler(executors=executors)
|
||||
start_check_results_job(sched)
|
||||
sched.print_jobs()
|
||||
sched.start()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
config_and_start_jobs()
|
||||
@@ -0,0 +1,30 @@
|
||||
from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor
|
||||
from apscheduler.schedulers.blocking import BlockingScheduler
|
||||
|
||||
from request_sender_test import start_send_requests
|
||||
|
||||
|
||||
def start_book_appointment():
|
||||
start_send_requests()
|
||||
|
||||
|
||||
def start_check_results_job(sched):
|
||||
sched.add_job(start_book_appointment, 'cron', day_of_week='mon-sat', hour='10',
|
||||
minute='30',
|
||||
misfire_grace_time=10,
|
||||
second='0', timezone='Europe/Paris', max_instances=1, args=[])
|
||||
|
||||
|
||||
def config_and_start_jobs():
|
||||
executors = {
|
||||
'default': ThreadPoolExecutor(30),
|
||||
'processpool': ProcessPoolExecutor(12)
|
||||
}
|
||||
sched = BlockingScheduler(executors=executors)
|
||||
start_check_results_job(sched)
|
||||
sched.print_jobs()
|
||||
sched.start()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
config_and_start_jobs()
|
||||
@@ -0,0 +1,13 @@
|
||||
import datetime
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
LOGS_DIR = str(Path.home())
|
||||
|
||||
|
||||
def init_logger():
|
||||
logging.basicConfig(filename=LOGS_DIR + "/request_{}.log".format(str(datetime.date.today())),
|
||||
filemode='a',
|
||||
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S',
|
||||
level=logging.INFO)
|
||||
@@ -0,0 +1,70 @@
|
||||
from http.cookies import SimpleCookie
|
||||
|
||||
|
||||
def get_datadome_cookies(cookiesStr: str):
|
||||
_cookies = SimpleCookie()
|
||||
_cookies.load(cookiesStr)
|
||||
new_cookies = {k: v.value for k, v in _cookies.items()}
|
||||
_new_coolies_str = None
|
||||
for key in new_cookies:
|
||||
if key == "datadome":
|
||||
if _new_coolies_str is None:
|
||||
_new_coolies_str = ""
|
||||
_new_coolies_str = _new_coolies_str + new_cookies[key]
|
||||
return _new_coolies_str
|
||||
|
||||
|
||||
def retain_only_dataome_cookies(cookiesStr: str):
|
||||
_cookies = SimpleCookie()
|
||||
_cookies.load(cookiesStr)
|
||||
new_cookies = {k: v.value for k, v in _cookies.items()}
|
||||
new_coolies_str = None
|
||||
for key in new_cookies:
|
||||
if key == "datadome":
|
||||
if new_coolies_str is None:
|
||||
new_coolies_str = ""
|
||||
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
|
||||
return new_coolies_str
|
||||
|
||||
|
||||
def get_app_cookies(cookiesStr: str):
|
||||
_cookies = SimpleCookie()
|
||||
_cookies.load(cookiesStr)
|
||||
new_cookies = {k: v.value for k, v in _cookies.items()}
|
||||
new_coolies_str = None
|
||||
for key in new_cookies:
|
||||
if key == "app.sig":
|
||||
if new_coolies_str is None:
|
||||
new_coolies_str = ""
|
||||
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
|
||||
return new_coolies_str
|
||||
|
||||
|
||||
def get_lang_cookies(cookiesStr: str):
|
||||
_cookies = SimpleCookie()
|
||||
_cookies.load(cookiesStr)
|
||||
new_cookies = {k: v.value for k, v in _cookies.items()}
|
||||
new_coolies_str = None
|
||||
for key in new_cookies:
|
||||
if key == "app":
|
||||
if new_coolies_str is None:
|
||||
new_coolies_str = ""
|
||||
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
|
||||
if key == "policy":
|
||||
if new_coolies_str is None:
|
||||
new_coolies_str = ""
|
||||
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
|
||||
if key == "lang":
|
||||
if new_coolies_str is None:
|
||||
new_coolies_str = ""
|
||||
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
|
||||
return new_coolies_str
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print(get_datadome_cookies(
|
||||
"datadome=HS7HPdbsFTYfxD60ogUwxjplOuqrveIjcTaucXVIWCko6pRayhXuKxC~44Ky5eNR2wnx4PEd2P2AadewmeXgxi4vn3x_QKdxLb6H1a9MbaMLNwmxfMEjQQ~CK8Y_nspJ;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiNXhvX3ZFaWc0aURla0dYMHZGTHNJX2p4In0=;policy=accepted;lang=fr;"))
|
||||
print(get_app_cookies(
|
||||
"datadome=HS7HPdbsFTYfxD60ogUwxjplOuqrveIjcTaucXVIWCko6pRayhXuKxC~44Ky5eNR2wnx4PEd2P2AadewmeXgxi4vn3x_QKdxLb6H1a9MbaMLNwmxfMEjQQ~CK8Y_nspJ;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiNXhvX3ZFaWc0aURla0dYMHZGTHNJX2p4In0=;policy=accepted;lang=fr;"))
|
||||
print(get_lang_cookies(
|
||||
"datadome=HS7HPdbsFTYfxD60ogUwxjplOuqrveIjcTaucXVIWCko6pRayhXuKxC~44Ky5eNR2wnx4PEd2P2AadewmeXgxi4vn3x_QKdxLb6H1a9MbaMLNwmxfMEjQQ~CK8Y_nspJ;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiNXhvX3ZFaWc0aURla0dYMHZGTHNJX2p4In0=;policy=accepted;lang=fr;"))
|
||||
@@ -1,11 +1,14 @@
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
from time import time
|
||||
from typing import Union
|
||||
import requests
|
||||
|
||||
from models.jsdata_le_pojo import JsDataLeTypePojo
|
||||
from models.jsdata_pojo import JsDataPojo
|
||||
from models.result_pojo import RequestResult
|
||||
from utils.get_only_datadome_cookies import get_datadome_cookies, get_app_cookies, get_lang_cookies, \
|
||||
retain_only_dataome_cookies
|
||||
from workers.proxies_constants import PROXY_LIST_FR
|
||||
|
||||
API_KEY = "d66aaf490d8aa424a5175e1fbd1aadea"
|
||||
@@ -22,10 +25,10 @@ class CaptchaResultGetter:
|
||||
self.cookie_str = 'datadome=5Nq~NEP_qQSHC0g_lZNnZmEv36J8gVV~rpZ329xmCkTq2~H3meIoXr4h_b988qB2XW5Te7iEGsvq8BzA5KeFupyrZFh4kgrDyl8hT2UymSByKHzAcDaNIBPDsRu2g_KG; Max-Age=31536000; Domain=.hermes.com; Path=/; Secure; SameSite=None'
|
||||
pass
|
||||
|
||||
def get_csrf(self, proxy_to_use, cookie: str = None) -> Union[str, None]:
|
||||
def get_csrf(self, proxy_to_use, js_data: JsDataPojo, cookie: str = None) -> Union[str, RequestResult]:
|
||||
if cookie is not None:
|
||||
headers = {'Content-Type': 'application/x-www-form-urlencoded',
|
||||
'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
|
||||
'User-Agent': js_data.ua,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
|
||||
'Referer': 'https://www.google.fr',
|
||||
'Cookie': cookie,
|
||||
@@ -34,7 +37,7 @@ class CaptchaResultGetter:
|
||||
'Accept-Language': 'fr-FR,fr;q=0.6'}
|
||||
else:
|
||||
headers = {'Content-Type': 'application/x-www-form-urlencoded',
|
||||
'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
|
||||
'User-Agent': js_data.ua,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
|
||||
'Referer': 'https://www.google.fr',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
@@ -47,13 +50,16 @@ class CaptchaResultGetter:
|
||||
timeout=15)
|
||||
print(response.status_code)
|
||||
if response.status_code == 200:
|
||||
# print(response.text)
|
||||
print(response.url)
|
||||
print(response.text)
|
||||
return self.extract_csrf_from_html(response.text)
|
||||
elif response.status_code == 403:
|
||||
return RequestResult.BLOCKED
|
||||
else:
|
||||
return None
|
||||
print(response.text)
|
||||
return RequestResult.UNKNOWN
|
||||
except Exception as error:
|
||||
print(error)
|
||||
return RequestResult.PROXY_ERROR
|
||||
|
||||
def extract_csrf_from_html(self, html: str) -> Union[str, None]:
|
||||
result = re.findall(r'_csrf" value="[A-Za-z0-9-_]+', html)
|
||||
@@ -63,9 +69,10 @@ class CaptchaResultGetter:
|
||||
return result_list[-1]
|
||||
return None
|
||||
|
||||
def get_valid_cookie(self, proxy_to_use, old_valid_cookie: str, dvm=4, hc=6) -> Union[str, None]:
|
||||
def get_le_valid_cookie(self, proxy_to_use, js_le_type_data: JsDataLeTypePojo, old_valid_cookie: str) -> Union[
|
||||
str, None]:
|
||||
headers = {'content-Type': 'application/x-www-form-urlencoded',
|
||||
'user-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
|
||||
'user-Agent': js_le_type_data.ua,
|
||||
'accept': '*/*',
|
||||
'referer': 'https://rendezvousparis.hermes.com/',
|
||||
'origin': 'https://rendezvousparis.hermes.com/',
|
||||
@@ -76,16 +83,15 @@ class CaptchaResultGetter:
|
||||
'sec-ch-ua-platform': 'Android',
|
||||
'accept-encoding': 'gzip, deflate, br',
|
||||
'accept-language': 'fr-FR,fr;q=0.6'}
|
||||
print("send request to get new cookie")
|
||||
print("send request to get le type cookie")
|
||||
print(proxy_to_use)
|
||||
print(headers)
|
||||
try:
|
||||
# tag_pu = 10 * Math.random()
|
||||
raw_data = self.get_raw_data(old_valid_cookie=old_valid_cookie, dvm=dvm, hc=hc)
|
||||
raw_data = self.get_le_type_raw_data(old_valid_cookie=old_valid_cookie, js_le_type_data=js_le_type_data)
|
||||
response = requests.post(url="https://d.digital.hermes/js/", headers=headers, verify=False,
|
||||
data=raw_data,
|
||||
proxies=proxy_to_use, timeout=15)
|
||||
print(response.status_code)
|
||||
# print(response.status_code)
|
||||
if response.status_code == 200:
|
||||
print(response.text)
|
||||
returned_result = json.loads(response.text)
|
||||
@@ -99,20 +105,32 @@ class CaptchaResultGetter:
|
||||
return None
|
||||
|
||||
def get_ch_raw_data_from_js_data(self, js_data: JsDataPojo, old_valid_cookie) -> str:
|
||||
_tag_version = "4.24.1"
|
||||
_tag_version = "4.25.1"
|
||||
_raw_data = "jsData={}&eventCounters=%5B%5D&jsType=ch&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv={}".format(
|
||||
js_data.to_url_encoded_json(), old_valid_cookie, _tag_version)
|
||||
print("raw data is " + _raw_data)
|
||||
print("raw ch data is " + _raw_data)
|
||||
return _raw_data
|
||||
|
||||
def get_valid_ch_cookie(self, proxy_to_use, js_data: JsDataPojo, old_valid_cookie: str = None) -> Union[str, None]:
|
||||
# extract datadome cookie
|
||||
# get new cookie
|
||||
_cookies_to_use = get_datadome_cookies(old_valid_cookie)
|
||||
_app_cookies = get_app_cookies(old_valid_cookie)
|
||||
_lang_cookies = get_lang_cookies(old_valid_cookie)
|
||||
_origin = "https://rendezvousparis.hermes.com/"
|
||||
# _origin = "https://www.hermes.com"
|
||||
_referer = "https://rendezvousparis.hermes.com/"
|
||||
# _referer = "https://www.hermes.com"
|
||||
|
||||
headers = {'content-Type': 'application/x-www-form-urlencoded',
|
||||
'user-Agent': js_data.ua,
|
||||
'accept': '*/*',
|
||||
'referer': 'https://rendezvousparis.hermes.com/',
|
||||
'origin': 'https://rendezvousparis.hermes.com/',
|
||||
'referer': _referer,
|
||||
'origin': _origin,
|
||||
'request': _origin,
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
'sec-fetch-site': 'cross-site',
|
||||
'sec-ch-ua-mobile': '?1',
|
||||
'sec-gpc': "1",
|
||||
'sec-fetch-dest': 'empty',
|
||||
'accept-encoding': 'gzip, deflate, br',
|
||||
@@ -121,10 +139,9 @@ class CaptchaResultGetter:
|
||||
print("send request to get new cookie")
|
||||
print(headers)
|
||||
try:
|
||||
# tag_pu = 10 * Math.random()
|
||||
response = requests.post(url="https://d.digital.hermes/js/", headers=headers, verify=False,
|
||||
data=self.get_ch_raw_data_from_js_data(js_data=js_data,
|
||||
old_valid_cookie=old_valid_cookie),
|
||||
old_valid_cookie=_cookies_to_use),
|
||||
proxies=proxy_to_use, timeout=15)
|
||||
print(response.status_code)
|
||||
if response.status_code == 200:
|
||||
@@ -132,6 +149,9 @@ class CaptchaResultGetter:
|
||||
returned_result = json.loads(response.text)
|
||||
new_cookie = returned_result['cookie']
|
||||
print(response.url)
|
||||
new_cookie = retain_only_dataome_cookies(new_cookie)
|
||||
if _app_cookies is not None:
|
||||
new_cookie = _app_cookies + new_cookie + _lang_cookies
|
||||
print(new_cookie)
|
||||
return new_cookie
|
||||
except Exception as error:
|
||||
@@ -139,27 +159,25 @@ class CaptchaResultGetter:
|
||||
print(error)
|
||||
return None
|
||||
|
||||
def get_raw_data(self, old_valid_cookie: str = None, dvm=4, hc=6):
|
||||
_raw_data = None
|
||||
|
||||
mousemove_count = random.randint(5, 10)
|
||||
click_count = random.randint(5, 10)
|
||||
click_count = mousemove_count
|
||||
key_count = random.randint(5, 10)
|
||||
def get_le_type_raw_data(self, old_valid_cookie: str, js_le_type_data: JsDataLeTypePojo):
|
||||
# m_m_c = f.mousemove
|
||||
mousemove_count = js_le_type_data.m_m_c
|
||||
# m_c_c = f.click
|
||||
click_count = js_le_type_data.m_c_c
|
||||
# m_s_c = f.scroll
|
||||
scroll_count = js_le_type_data.m_s_c
|
||||
key_count = random.randint(5, 12)
|
||||
touch_count = mousemove_count
|
||||
js_set = int(time())
|
||||
tag_pu = random.uniform(1, 10)
|
||||
while len(str(tag_pu)) != 17:
|
||||
tag_pu = random.uniform(1, 10)
|
||||
|
||||
if old_valid_cookie is None:
|
||||
_raw_data = "jsData=%7B%22log1%22%3Afalse%2C%22opts%22%3A%22endpoint%2CajaxListenerPath%22%2C%22ttst%22%3A38%2C%22ifov%22%3Afalse%2C%22hc%22%3A{}%2C%22br_oh%22%3A646%2C%22br_ow%22%3A360%2C%22ua%22%3A%22Mozilla%2F5.0%20(Linux%3B%20Android%2010%3B%20K)%20AppleWebKit%2F537.36%20(KHTML%2C%20like%20Gecko)%20Chrome%2F115.0.0.0%20Mobile%20Safari%2F537.36%22%2C%22wbd%22%3Afalse%2C%22tagpu%22%3A{}%2C%22wdif%22%3Afalse%2C%22wdifrm%22%3Afalse%2C%22npmtm%22%3Afalse%2C%22br_h%22%3A646%2C%22br_w%22%3A360%2C%22nddc%22%3A1%2C%22rs_h%22%3A800%2C%22rs_w%22%3A360%2C%22rs_cd%22%3A24%2C%22phe%22%3Afalse%2C%22nm%22%3Afalse%2C%22jsf%22%3Afalse%2C%22lg%22%3A%22fr-FR%22%2C%22pr%22%3A3%2C%22ars_h%22%3A800%2C%22ars_w%22%3A360%2C%22tz%22%3A-60%2C%22str_ss%22%3Atrue%2C%22str_ls%22%3Atrue%2C%22str_idb%22%3Atrue%2C%22str_odb%22%3Atrue%2C%22plgod%22%3Afalse%2C%22plg%22%3A2%2C%22plgne%22%3A%22err%22%2C%22plgre%22%3A%22err%22%2C%22plgof%22%3A%22err%22%2C%22plggt%22%3A%22err%22%2C%22pltod%22%3Afalse%2C%22hcovdr%22%3Afalse%2C%22hcovdr2%22%3Afalse%2C%22plovdr%22%3Afalse%2C%22plovdr2%22%3Afalse%2C%22ftsovdr%22%3Afalse%2C%22ftsovdr2%22%3Afalse%2C%22lb%22%3Afalse%2C%22eva%22%3A33%2C%22lo%22%3Afalse%2C%22ts_mtp%22%3A5%2C%22ts_tec%22%3Atrue%2C%22ts_tsa%22%3Atrue%2C%22vnd%22%3A%22Google%20Inc.%22%2C%22bid%22%3A%22NA%22%2C%22mmt%22%3A%22empty%22%2C%22plu%22%3A%226GLkaVK%2CjRvAfXq0%22%2C%22hdn%22%3Afalse%2C%22awe%22%3Afalse%2C%22geb%22%3Afalse%2C%22dat%22%3Afalse%2C%22med%22%3A%22defined%22%2C%22aco%22%3A%22probably%22%2C%22acots%22%3Afalse%2C%22acmp%22%3A%22probably%22%2C%22acmpts%22%3Atrue%2C%22acw%22%3A%22probably%22%2C%22acwts%22%3Afalse%2C%22acma%22%3A%22maybe%22%2C%22acmats%22%3Afalse%2C%22acaa%22%3A%22probably%22%2C%22acaats%22%3Atrue%2C%22ac3%22%3A%22%22%2C%22ac3ts%22%3Afalse%2C%22acf%22%3A%22probably%22%2C%22acfts%22%3Afalse%2C%22acmp4%22%3A%22maybe%22%2C%22acmp4ts%22%3Afalse%2C%22acmp3%22%3A%22probably%22%2C%22acmp3ts%22%3Afalse%2C%22acwm%22%3A%22maybe%22%2C%22acwmts%22%3Afalse%2C%22ocpt%22%3Afalse%2C%22vco%22%3A%22%22%2C%22vcots%22%3Afalse%2C%22vch%22%3A%22probably%22%2C%22vchts%22%3Atrue%2C%22vcw%22%3A%22probably%22%2C%22vcwts%22%3Atrue%2C%22vc3%22%3A%22maybe%22%2C%22vc3ts%22%3Afalse%2C%22vcmp%22%3A%22%22%2C%22vcmpts%22%3Afalse%2C%22vcq%22%3A%22%22%2C%22vcqts%22%3Afalse%2C%22vc1%22%3A%22probably%22%2C%22vc1ts%22%3Atrue%2C%22dvm%22%3A{}%2C%22sqt%22%3Afalse%2C%22so%22%3A%22portrait-primary%22%2C%22wdw%22%3Atrue%2C%22cokys%22%3A%22bG9hZFRpbWVzY3NpL%3D%22%2C%22ecpc%22%3Afalse%2C%22lgs%22%3Atrue%2C%22lgsod%22%3Afalse%2C%22psn%22%3Atrue%2C%22edp%22%3Afalse%2C%22addt%22%3Atrue%2C%22wsdc%22%3Atrue%2C%22ccsr%22%3Atrue%2C%22nuad%22%3Atrue%2C%22bcda%22%3Atrue%2C%22idn%22%3Atrue%2C%22capi%22%3Afalse%2C%22svde%22%3Afalse%2C%22vpbq%22%3Atrue%2C%22ucdv%22%3Afalse%2C%22spwn%22%3Afalse%2C%22emt%22%3Afalse%2C%22bfr%22%3Afalse%2C%22dbov%22%3Afalse%2C%22cfpfe%22%3A%22ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ3MCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1snXHg3M1x4NjhceDYxXHg2%22%2C%22stcfp%22%3A%22Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzUyMzcpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWEgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODg0NDkpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMTY1%22%2C%22ckwa%22%3Atrue%2C%22glvd%22%3A%22Google%20Inc.%20(Qualcomm)%22%2C%22glrd%22%3A%22ANGLE%20(Qualcomm%2C%20Adreno%20(TM)%20650%2C%20OpenGL%20ES%203.2)%22%2C%22wwl%22%3Afalse%2C%22jset%22%3A{}%2C%22dcok%22%3A%22.hermes.com%22%2C%22log2%22%3Atrue%2C%22tbce%22%3A0%2C%22es_sigmdn%22%3Anull%2C%22es_mumdn%22%3Anull%2C%22es_distmdn%22%3Anull%2C%22es_angsmdn%22%3Anull%2C%22es_angemdn%22%3Anull%2C%22m_s_c%22%3A3%2C%22m_m_c%22%3A5%2C%22m_c_c%22%3A5%2C%22m_cm_r%22%3A1%2C%22m_ms_r%22%3A1.6666666666666667%7D&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A3%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A0%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid=S2ilUoO80StZ2OfedRZnvyMiJ9hmD_DyIWj11O2eak~8TOek7JTTeJeKWDLzGgMZaH2IOix3BX1ocB4TwnLQFx~76pAQAD0JnVZzoxwbxOPL2SR834BfbiK0Nu4c0Pxh&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.19.3".format(
|
||||
hc, tag_pu, dvm, js_set, mousemove_count, click_count, touch_count, touch_count, key_count, key_count)
|
||||
else:
|
||||
_raw_data = "jsData=%7B%22log1%22%3Afalse%2C%22opts%22%3A%22endpoint%2CajaxListenerPath%22%2C%22ttst%22%3A38%2C%22ifov%22%3Afalse%2C%22hc%22%3A{}%2C%22br_oh%22%3A646%2C%22br_ow%22%3A360%2C%22ua%22%3A%22Mozilla%2F5.0%20(Linux%3B%20Android%2010%3B%20K)%20AppleWebKit%2F537.36%20(KHTML%2C%20like%20Gecko)%20Chrome%2F115.0.0.0%20Mobile%20Safari%2F537.36%22%2C%22wbd%22%3Afalse%2C%22tagpu%22%3A{}%2C%22wdif%22%3Afalse%2C%22wdifrm%22%3Afalse%2C%22npmtm%22%3Afalse%2C%22br_h%22%3A646%2C%22br_w%22%3A360%2C%22nddc%22%3A1%2C%22rs_h%22%3A800%2C%22rs_w%22%3A360%2C%22rs_cd%22%3A24%2C%22phe%22%3Afalse%2C%22nm%22%3Afalse%2C%22jsf%22%3Afalse%2C%22lg%22%3A%22fr-FR%22%2C%22pr%22%3A3%2C%22ars_h%22%3A800%2C%22ars_w%22%3A360%2C%22tz%22%3A-60%2C%22str_ss%22%3Atrue%2C%22str_ls%22%3Atrue%2C%22str_idb%22%3Atrue%2C%22str_odb%22%3Atrue%2C%22plgod%22%3Afalse%2C%22plg%22%3A2%2C%22plgne%22%3A%22err%22%2C%22plgre%22%3A%22err%22%2C%22plgof%22%3A%22err%22%2C%22plggt%22%3A%22err%22%2C%22pltod%22%3Afalse%2C%22hcovdr%22%3Afalse%2C%22hcovdr2%22%3Afalse%2C%22plovdr%22%3Afalse%2C%22plovdr2%22%3Afalse%2C%22ftsovdr%22%3Afalse%2C%22ftsovdr2%22%3Afalse%2C%22lb%22%3Afalse%2C%22eva%22%3A33%2C%22lo%22%3Afalse%2C%22ts_mtp%22%3A5%2C%22ts_tec%22%3Atrue%2C%22ts_tsa%22%3Atrue%2C%22vnd%22%3A%22Google%20Inc.%22%2C%22bid%22%3A%22NA%22%2C%22mmt%22%3A%22empty%22%2C%22plu%22%3A%226GLkaVK%2CjRvAfXq0%22%2C%22hdn%22%3Afalse%2C%22awe%22%3Afalse%2C%22geb%22%3Afalse%2C%22dat%22%3Afalse%2C%22med%22%3A%22defined%22%2C%22aco%22%3A%22probably%22%2C%22acots%22%3Afalse%2C%22acmp%22%3A%22probably%22%2C%22acmpts%22%3Atrue%2C%22acw%22%3A%22probably%22%2C%22acwts%22%3Afalse%2C%22acma%22%3A%22maybe%22%2C%22acmats%22%3Afalse%2C%22acaa%22%3A%22probably%22%2C%22acaats%22%3Atrue%2C%22ac3%22%3A%22%22%2C%22ac3ts%22%3Afalse%2C%22acf%22%3A%22probably%22%2C%22acfts%22%3Afalse%2C%22acmp4%22%3A%22maybe%22%2C%22acmp4ts%22%3Afalse%2C%22acmp3%22%3A%22probably%22%2C%22acmp3ts%22%3Afalse%2C%22acwm%22%3A%22maybe%22%2C%22acwmts%22%3Afalse%2C%22ocpt%22%3Afalse%2C%22vco%22%3A%22%22%2C%22vcots%22%3Afalse%2C%22vch%22%3A%22probably%22%2C%22vchts%22%3Atrue%2C%22vcw%22%3A%22probably%22%2C%22vcwts%22%3Atrue%2C%22vc3%22%3A%22maybe%22%2C%22vc3ts%22%3Afalse%2C%22vcmp%22%3A%22%22%2C%22vcmpts%22%3Afalse%2C%22vcq%22%3A%22%22%2C%22vcqts%22%3Afalse%2C%22vc1%22%3A%22probably%22%2C%22vc1ts%22%3Atrue%2C%22dvm%22%3A{}%2C%22sqt%22%3Afalse%2C%22so%22%3A%22portrait-primary%22%2C%22wdw%22%3Atrue%2C%22cokys%22%3A%22bG9hZFRpbWVzY3NpL%3D%22%2C%22ecpc%22%3Afalse%2C%22lgs%22%3Atrue%2C%22lgsod%22%3Afalse%2C%22psn%22%3Atrue%2C%22edp%22%3Afalse%2C%22addt%22%3Atrue%2C%22wsdc%22%3Atrue%2C%22ccsr%22%3Atrue%2C%22nuad%22%3Atrue%2C%22bcda%22%3Atrue%2C%22idn%22%3Atrue%2C%22capi%22%3Afalse%2C%22svde%22%3Afalse%2C%22vpbq%22%3Atrue%2C%22ucdv%22%3Afalse%2C%22spwn%22%3Afalse%2C%22emt%22%3Afalse%2C%22bfr%22%3Afalse%2C%22dbov%22%3Afalse%2C%22cfpfe%22%3A%22ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ3MCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1snXHg3M1x4NjhceDYxXHg2%22%2C%22stcfp%22%3A%22Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzUyMzcpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWEgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODg0NDkpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMTY1%22%2C%22ckwa%22%3Atrue%2C%22glvd%22%3A%22Google%20Inc.%20(Qualcomm)%22%2C%22glrd%22%3A%22ANGLE%20(Qualcomm%2C%20Adreno%20(TM)%20650%2C%20OpenGL%20ES%203.2)%22%2C%22wwl%22%3Afalse%2C%22jset%22%3A{}%2C%22dcok%22%3A%22.hermes.com%22%2C%22log2%22%3Atrue%2C%22tbce%22%3A0%2C%22es_sigmdn%22%3Anull%2C%22es_mumdn%22%3Anull%2C%22es_distmdn%22%3Anull%2C%22es_angsmdn%22%3Anull%2C%22es_angemdn%22%3Anull%2C%22m_s_c%22%3A3%2C%22m_m_c%22%3A5%2C%22m_c_c%22%3A5%2C%22m_cm_r%22%3A1%2C%22m_ms_r%22%3A1.6666666666666667%7D&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A3%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A0%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.19.3".format(
|
||||
hc, tag_pu, dvm, js_set, mousemove_count, click_count, touch_count, touch_count, key_count,
|
||||
key_count, old_valid_cookie)
|
||||
print("raw data is " + _raw_data)
|
||||
touch_move = touch_count + 2
|
||||
# _le_js_raw_data = self.get_le_raw_data_from_js_data(js_le_type_data=js_le_type_data,
|
||||
# old_valid_cookie=old_valid_cookie)
|
||||
_cid = get_datadome_cookies(old_valid_cookie)
|
||||
_raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.25.1".format(
|
||||
js_le_type_data.to_url_encoded_json(), mousemove_count, click_count, scroll_count, touch_count, touch_count,
|
||||
touch_move,
|
||||
key_count,
|
||||
key_count, _cid)
|
||||
print("raw le data is " + _raw_data)
|
||||
return _raw_data
|
||||
|
||||
def extract_cid_from_html(self, html_text):
|
||||
@@ -182,48 +200,6 @@ class CaptchaResultGetter:
|
||||
|
||||
#
|
||||
if __name__ == '__main__':
|
||||
# print(int(time()))
|
||||
captchaResultGetter = CaptchaResultGetter()
|
||||
_url_encoded_data = captchaResultGetter.get_valid_cookie()
|
||||
print(_url_encoded_data)
|
||||
|
||||
# print(urllib.parse.parse_qs((_url_encoded_data)))
|
||||
# valid_cookie = captchaResultGetter.get_valid_cookie()
|
||||
# simple_cookie = SimpleCookie()
|
||||
# simple_cookie.load(valid_cookie)
|
||||
# new_cookies = {k: v.value for k, v in simple_cookie.items()}
|
||||
# new_coolies_str = ""
|
||||
# for key in new_cookies:
|
||||
# print(key)
|
||||
# new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
|
||||
# print(new_coolies_str)
|
||||
# valid_cookie = new_coolies_str + "app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;policy=accepted;lang=fr;"
|
||||
# print(valid_cookie)
|
||||
# # csrf = captchaResultGetter.extract_csrf_from_html(
|
||||
# # """strong>Français</strong> / <a href="?lang=en">English</a> / <a href="?lang=cn">中文</a> / <a href="?lang=jp">日本語</a> / <a href="?lang=kr">한국어</a> / <a href="?lang=ru">Русский</a></div><div class="row"><div class="col-sm-6 offset-sm-3"></div></div><div class="row"><div class="col-md-6 offset-md-3"><div class="text-center"><h1>Bienvenue chez Hermès</h1><div class="client-nav register-for text-center">Demande de rendez-vous pour le vendredi 12 janvier</div><hr/></div><form id="client-registration-form" method="post" action="/client/register" class=""><input type="hidden" name="check" id="check" /><input type="hidden" name="_csrf" value="Qw5FkdRf--yiWD6S527G7Qt40DCLAzk1q21Y" /><div class="mb-3"><label for="prefer" class="control-label">Magasin préféré</label><select id="prefer" name="prefer" class="form-control" data-value=""><option value="">Sans préférence</option><option value="faubourg">Hermès Faubourg Saint-Honoré</option><option value="georgev">Hermès George V</option><option value="sevres">Hermès Sèvres</option></select></div><div class="mb-3">""")
|
||||
# # print(csrf)
|
||||
# contact_list = read_contacts('/Users/panlei/Desktop/yahoo_aol_valid_26-2.xlsx')
|
||||
# # remove already booked contacts
|
||||
# sub_contact_list = filter_contacts(contact_list)
|
||||
# random.shuffle(sub_contact_list)
|
||||
# valid_cookie = captchaResultGetter.get_valid_cookie()
|
||||
# contact = random.choice(sub_contact_list)
|
||||
# if valid_cookie is not None:
|
||||
# # new_csrf = captchaResultGetter.get_csrf(valid_cookie)
|
||||
# new_csrf = None
|
||||
# # if new_csrf is not None:
|
||||
# # print(new_csrf)
|
||||
# simple_cookie = SimpleCookie()
|
||||
# simple_cookie.load(valid_cookie)
|
||||
# new_cookies = {k: v.value for k, v in simple_cookie.items()}
|
||||
# new_coolies_str = ""
|
||||
# for key in new_cookies:
|
||||
# print(key)
|
||||
# new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
|
||||
# print(new_coolies_str)
|
||||
# valid_cookie = new_coolies_str + "app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;policy=accepted;lang=fr;"
|
||||
# print(valid_cookie)
|
||||
# new_csrf = captchaResultGetter.get_csrf(valid_cookie)
|
||||
# if new_csrf is not None:
|
||||
# sender = Sender(valid_cookie)
|
||||
# sender.send_request(HERMES_REGISTER, contact=contact, csrf=new_csrf)
|
||||
|
||||
+30
-13
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
@@ -10,8 +11,8 @@ from db.mongo_manager import MONGO_STORE_MANAGER
|
||||
from models.LinkPojo import LinkPojo
|
||||
from models.result_pojo import RequestResult
|
||||
from proxy_manager.proxy_manager import ProxyManager
|
||||
from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST
|
||||
from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials, REQUEST_DATA_DE
|
||||
from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST, TEST_QUEUE, SHARED_OBJECT
|
||||
from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
|
||||
from workers.proxies_constants import PROXY_LIST_FR
|
||||
|
||||
DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ces coordonnées"
|
||||
@@ -35,7 +36,6 @@ class LinkValidator(threading.Thread):
|
||||
self.filter_with_ip_country()
|
||||
self.proxy_manager = proxy_manager
|
||||
self.limit = limit
|
||||
# self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
|
||||
|
||||
def set_up_connection(self):
|
||||
self.connection = pika.BlockingConnection(
|
||||
@@ -47,10 +47,13 @@ class LinkValidator(threading.Thread):
|
||||
self.channel.basic_consume(queue=self.queue_to_listen, auto_ack=False, on_message_callback=callback)
|
||||
self.channel.start_consuming()
|
||||
|
||||
def send_request(self, linkPojo: LinkPojo) -> RequestResult:
|
||||
def send_request(self, linkPojo: LinkPojo, _received_dict=None) -> RequestResult:
|
||||
_ua = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36'
|
||||
if _received_dict is not None:
|
||||
_ua = _received_dict['ua']
|
||||
self.cookie.load(self.cookie_str)
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
|
||||
'User-Agent': _ua,
|
||||
'Accept': '*/*',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Cache-Control': 'max-age=0',
|
||||
@@ -66,7 +69,9 @@ class LinkValidator(threading.Thread):
|
||||
print("received cookie is " + str(self.cookie_str))
|
||||
print("send request for link: " + linkPojo.url)
|
||||
try:
|
||||
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=_proxy_to_use,
|
||||
print("will send request with ua {}".format(_ua))
|
||||
print("will send request with cookie {}".format(self.cookie_str))
|
||||
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
|
||||
timeout=60)
|
||||
print(response.status_code)
|
||||
if response.status_code == 200:
|
||||
@@ -85,6 +90,11 @@ class LinkValidator(threading.Thread):
|
||||
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
|
||||
print("will publish to queue {}".format(new_coolies_str))
|
||||
# upload the cookie to queue
|
||||
if _received_dict is not None:
|
||||
_received_dict['cookiesStr'] = new_coolies_str
|
||||
print("body in json:{}".format(json.dumps(_received_dict)))
|
||||
self.cookiesPublisher.publish_body(json.dumps(_received_dict))
|
||||
else:
|
||||
self.cookiesPublisher.publish_body(new_coolies_str)
|
||||
self.cookie_str = new_coolies_str
|
||||
return RequestResult.SUCCESS
|
||||
@@ -151,16 +161,24 @@ class LinkValidator(threading.Thread):
|
||||
print("message count in queue is {}".format(_message_in_queue_count))
|
||||
self.update_validate_list()
|
||||
self.filter_with_ip_country()
|
||||
self.cookie_str = body.decode("UTF-8")
|
||||
_received_object = body.decode("UTF-8")
|
||||
js_data = None
|
||||
_received_dict = None
|
||||
if "glrd" in _received_object:
|
||||
_received_dict = json.loads(_received_object)
|
||||
_received_cookies = _received_dict["cookiesStr"]
|
||||
else:
|
||||
_received_cookies = _received_object
|
||||
self.cookie_str = _received_cookies
|
||||
random.shuffle(self.link_to_validate_list)
|
||||
if len(self.link_to_validate_list) > 0 and _message_in_queue_count > self.limit:
|
||||
if len(self.link_to_validate_list) > 0 and _message_in_queue_count >= self.limit:
|
||||
print("links number is {}".format(len(self.link_to_validate_list)))
|
||||
can_continue = None
|
||||
for con in self.link_to_validate_list:
|
||||
# if not is_already_sent(con):
|
||||
print(con.email)
|
||||
self.proxy_to_use = random.choice(PROXY_LIST_FR)
|
||||
can_continue = self.send_request(con)
|
||||
can_continue = self.send_request(con, _received_dict)
|
||||
if can_continue == RequestResult.BLOCKED:
|
||||
print("cannot continue, blocked, then skip")
|
||||
break
|
||||
@@ -190,13 +208,12 @@ class LinkValidator(threading.Thread):
|
||||
|
||||
|
||||
def validate_with_FR_ip(segment_position=1):
|
||||
_queue_name = REQUEST_DATA_QUEUE
|
||||
cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
|
||||
_queue_name = TEST_QUEUE
|
||||
cookiesPublisher = CookiesPublisher(queue_name=TEST_QUEUE)
|
||||
cookiesPublisher.set_up_connection()
|
||||
print("filter links with ip_country")
|
||||
_proxy_manager = ProxyManager()
|
||||
receiver = LinkValidator(cookiesPublisher=cookiesPublisher,
|
||||
proxy_manager=_proxy_manager,
|
||||
receiver = LinkValidator(cookiesPublisher=cookiesPublisher, proxy_manager=_proxy_manager,
|
||||
queue_to_listen=_queue_name, ip_country="FR", segment_position=segment_position, limit=0)
|
||||
print("will connect to queue")
|
||||
receiver.set_up_connection()
|
||||
|
||||
@@ -42,12 +42,21 @@ FR_PROXY_RES_OXY = {
|
||||
'https': 'http://customer-rendezvous-cc-FR:Rdv202220212023@pr.oxylabs.io:7777'
|
||||
}
|
||||
|
||||
|
||||
FR_PROXY_MOB_OXY = {
|
||||
'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@pr.oxylabs.io:7777',
|
||||
'https': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@pr.oxylabs.io:7777'
|
||||
}
|
||||
|
||||
FR_PROXY_MOB_OXY_STICKY = {
|
||||
'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}',
|
||||
'https': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}'
|
||||
}
|
||||
|
||||
FR_PROXY_RES_OXY_STICKY = {
|
||||
'http': 'http://customer-rendezvous-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}',
|
||||
'https': 'http://customer-rendezvous-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}'
|
||||
}
|
||||
|
||||
DE_PROXY_MOBILE = {
|
||||
'http': 'http://8153587-mobile-country-DE:jp50x9jmo@194.88.106.169:11842',
|
||||
'https': 'http://8153587-mobile-country-DE:jp50x9jmo@194.88.106.169:11842'
|
||||
@@ -57,7 +66,6 @@ DE_PROXY_RES = {
|
||||
'https': 'http://8254565-res-country-DE:1st57izbhe@185.21.60.181:14648'
|
||||
}
|
||||
|
||||
|
||||
# FR_PROXY_MOBILE
|
||||
# FR_PROXY_RES
|
||||
# PROXY_LIST = [FR_PROXY_MOBILE, FR_PROXY_RES, DE_PROXY_RES, DE_PROXY_MOBILE, ES_PROXY_MOBILE, IT_PROXY_MOBILE]
|
||||
@@ -65,7 +73,4 @@ DE_PROXY_RES = {
|
||||
# PROXY_LIST_FR = [FR_PROXY_RES_OXY, FR_PROXY_MOBILE_3]
|
||||
# PROXY_LIST_FR = [ FR_PROXY_MOBILE_3]
|
||||
PROXY_LIST_FR = [FR_PROXY_RES_OXY]
|
||||
MOBILE_PROXY_LIST_FR = [FR_PROXY_MOB_OXY]
|
||||
# PROXY_LIST = [FR_PROXY_MOBILE, FR_
|
||||
# PROXY_RES]
|
||||
PROXY_LIST_DE = [DE_PROXY_RES, DE_PROXY_MOBILE]
|
||||
MOBILE_PROXY_LIST_FR = [FR_PROXY_MOB_OXY_STICKY]
|
||||
|
||||
+31
-15
@@ -8,6 +8,7 @@ import requests
|
||||
|
||||
from db.mongo_manager import MONGO_STORE_MANAGER
|
||||
from models.ReserveResultPojo import ReserveResultPojo, PublishType
|
||||
from models.jsdata_pojo import JsDataPojo
|
||||
from models.result_pojo import RequestResult
|
||||
from queue_message.CookiesPublisher import CookiesPublisher
|
||||
from workers.proxies_constants import PROXY_LIST_FR
|
||||
@@ -15,18 +16,19 @@ from workers.proxies_constants import PROXY_LIST_FR
|
||||
|
||||
class Sender:
|
||||
|
||||
def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher, received_dict, proxy_to_use):
|
||||
def __init__(self, cookie_str, cookiesPublisher: CookiesPublisher, received_dict, proxy_to_use, logger):
|
||||
self.store_type = "random"
|
||||
self.cookie = SimpleCookie()
|
||||
self.cookiesPublisher = cookiesPublisher
|
||||
# self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
|
||||
self.cookie_str = cookie_str
|
||||
self.logger = logger
|
||||
self._csrf = None
|
||||
self.received_dict = received_dict
|
||||
self.proxy_to_use = proxy_to_use
|
||||
self.cookie.load(self.cookie_str)
|
||||
|
||||
def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str):
|
||||
def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str, store_type: str):
|
||||
# create the message
|
||||
if url == "https://rendezvousparis.hermes.com/client/welcome":
|
||||
return
|
||||
@@ -35,7 +37,7 @@ class Sender:
|
||||
firstName=contact.first_name, lastName=contact.last_name, email=contact.mail,
|
||||
passport=contact.passport, ccid=contact.ccid)
|
||||
result.id = id
|
||||
result.store_type = self.store_type
|
||||
result.store_type = store_type
|
||||
result.created_at = time.strftime("%H:%M:%S", time.localtime())
|
||||
collection_name = str(datetime.date.today())
|
||||
MONGO_STORE_MANAGER.insert_reserve_result(collection_name=collection_name, reserve=result)
|
||||
@@ -46,7 +48,7 @@ class Sender:
|
||||
def get_csrf(self):
|
||||
pass
|
||||
|
||||
def send_request(self, url, contact: ContactPojo, csrf: str = None) -> RequestResult:
|
||||
def send_request(self, url, js_data: JsDataPojo, contact: ContactPojo, csrf: str = None) -> RequestResult:
|
||||
if csrf is None:
|
||||
if self._csrf is None:
|
||||
_csrf = '8Bs2dBwb-nHONOzo9Tei2CcMZglEfsRqUz8E'
|
||||
@@ -55,44 +57,58 @@ class Sender:
|
||||
else:
|
||||
self._csrf = csrf
|
||||
_csrf = csrf
|
||||
# 默认为空，也就是随机
|
||||
_selected_store = ''
|
||||
if contact.store is not None and contact.store != "random":
|
||||
_selected_store = contact.store
|
||||
self.logger.info("store is:{}".format(_selected_store))
|
||||
headers = {'Content-Type': 'application/x-www-form-urlencoded',
|
||||
'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
|
||||
'User-Agent': js_data.ua,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
|
||||
'Cookie': self.cookie_str, 'Referer': 'https://rendezvousparis.hermes.com/client/register',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-Dest': 'document',
|
||||
'Accept-Language': 'fr-FR,fr;q=0.6'}
|
||||
data = {'check': '', '_csrf': _csrf, 'prefer': '',
|
||||
data = {'check': '', '_csrf': _csrf, 'prefer': _selected_store,
|
||||
'surname': contact.last_name.capitalize(), 'name': contact.first_name.capitalize(),
|
||||
'phone_country': "FR", 'phone_number': "+33 " + contact.phone, 'email': contact.mail,
|
||||
'passport_id': contact.passport, 'processing': 'on', 'cgu': 'on'}
|
||||
print(data)
|
||||
self.logger.info(data)
|
||||
self.logger.info("try to request for mail:{}".format(contact.mail))
|
||||
print("send request with cookie:{}".format(self.cookie_str))
|
||||
try:
|
||||
proxy_to_use = self.proxy_to_use
|
||||
# proxy_to_use = PROXY_LIST[0]
|
||||
print(proxy_to_use)
|
||||
# print(proxy_to_use)
|
||||
response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data,
|
||||
timeout=15)
|
||||
print(response.status_code)
|
||||
self.logger.info(response.status_code)
|
||||
if response.status_code == 200:
|
||||
# add to mongodb
|
||||
print(response.text)
|
||||
print(response.url)
|
||||
self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=response.url)
|
||||
self.logger.info(response.text)
|
||||
self.logger.info("{}:{}".format(contact.mail, response.url))
|
||||
self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=response.url,
|
||||
store_type=_selected_store)
|
||||
cookies_to_set = response.headers['set-cookie']
|
||||
self.cookie.load(cookies_to_set)
|
||||
new_cookies = {k: v.value for k, v in self.cookie.items()}
|
||||
new_coolies_str = ""
|
||||
for key in new_cookies:
|
||||
new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
|
||||
print("will publish to queue {}".format(new_coolies_str))
|
||||
self.logger.info("will publish to queue {}".format(new_coolies_str))
|
||||
# upload the cookie to queue
|
||||
self.received_dict['cookiesStr'] = new_coolies_str
|
||||
print("body in json:{}".format(json.dumps(self.received_dict)))
|
||||
self.logger.info("body in json:{}".format(json.dumps(self.received_dict)))
|
||||
self.cookiesPublisher.publish_body(json.dumps(self.received_dict))
|
||||
# self.cookie_str = new_coolies_str
|
||||
self.logger.info("successful request, mail:{}".format(contact.mail))
|
||||
return RequestResult.SUCCESS
|
||||
else:
|
||||
print(response.text)
|
||||
if response.status_code == 403:
|
||||
# try to load seesion
|
||||
# session = HTMLSession()
|
||||
self.logger.info("blocked by hermes, mail:{}".format(contact.mail))
|
||||
return RequestResult.BLOCKED
|
||||
return RequestResult.BLOCKED
|
||||
except Exception as Error:
|
||||
print(Error)
|
||||
|
||||
Reference in New Issue
Block a user