panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: panleicim/appointment_request:feature/5_0_2
Branches Tags
panleicim/appointment_request:master panleicim/appointment_request:feature/read_mail_issue panleicim/appointment_request:feature/password_in_env panleicim/appointment_request:api/app_api panleicim/appointment_request:feature/get_ip_geo_info panleicim/appointment_request:feature/links_filter_with_model panleicim/appointment_request:feature/v_5_1_15 panleicim/appointment_request:feature/with_lan panleicim/appointment_request:feature/token_from_cookies panleicim/appointment_request:feature/5_0_2 panleicim/appointment_request:feature/4_40_0 panleicim/appointment_request:feature/test panleicim/appointment_request:develop panleicim/appointment_request:feature/account_creation
..
pull from: panleicim/appointment_request:feature/test
Branches Tags
panleicim/appointment_request:feature/read_mail_issue panleicim/appointment_request:master panleicim/appointment_request:feature/password_in_env panleicim/appointment_request:api/app_api panleicim/appointment_request:feature/get_ip_geo_info panleicim/appointment_request:feature/links_filter_with_model panleicim/appointment_request:feature/v_5_1_15 panleicim/appointment_request:feature/with_lan panleicim/appointment_request:feature/token_from_cookies panleicim/appointment_request:feature/5_0_2 panleicim/appointment_request:feature/4_40_0 panleicim/appointment_request:feature/test panleicim/appointment_request:develop panleicim/appointment_request:feature/account_creation

1 Commits
feature/5_0_2 .. feature/test

Author SHA1 Message Date
panleicim 8bba5eb2f1 added method to remove message 2024-12-03 18:46:42 +01:00
30 changed files with 224 additions and 1174 deletions
Expand all files Collapse all files
.gitignore
-1
View File
@@ -4,4 +4,3 @@ __pycache__/
.idea .idea
db/__pycache__/* db/__pycache__/*
*.iml *.iml
venv
db/mongo_manager.py
+1 -15
View File
@@ -20,7 +20,6 @@ EMAIL_LIST = "EMAIL_LIST"
DESTINATION_EMAIL_LIST = "DESTINATION_EMAIL_LIST" DESTINATION_EMAIL_LIST = "DESTINATION_EMAIL_LIST"
LINKS_TO_VALIDATE = "LINKS_TO_VALIDATE" LINKS_TO_VALIDATE = "LINKS_TO_VALIDATE"
INVALID_EMAIL_LIST = "INVALID_EMAIL_LIST" INVALID_EMAIL_LIST = "INVALID_EMAIL_LIST"
CONTACT_LIST_SERIAL_MAP = "CONTACT_LIST_SERIAL_MAP"
class MongoDbManager: class MongoDbManager:
@@ -76,18 +75,10 @@ class MongoDbManager:
result_list.append(ContactPojo.from_firestore_dict(document)) result_list.append(ContactPojo.from_firestore_dict(document))
return result_list return result_list
def get_all_contact_serial_list(self) -> list:
result_list = []
cursor = self.db[CONTACT_LIST_SERIAL_MAP]
for document in cursor.find():
result_list.append(ContactPojo.from_firestore_dict(document))
return result_list
def save_links_to_validate(self, link: str, mail_address: str, _all_contact_list: list): def save_links_to_validate(self, link: str, mail_address: str, _all_contact_list: list):
collection_to_use = self.db[LINKS_TO_VALIDATE] collection_to_use = self.db[LINKS_TO_VALIDATE]
updated_at = time.strftime("%H:%M:%S", time.localtime()) updated_at = time.strftime("%H:%M:%S", time.localtime())
_ip_country = "FR" _ip_country = "FR"
serial = "requests"
# find ip_country info # find ip_country info
for _contact in _all_contact_list: for _contact in _all_contact_list:
if _contact.mail == mail_address: if _contact.mail == mail_address:
@@ -97,7 +88,6 @@ class MongoDbManager:
collection_to_use.replace_one(filter={'_id': mail_address, }, replacement={ collection_to_use.replace_one(filter={'_id': mail_address, }, replacement={
u'url': link, u'url': link,
u'email': mail_address, u'email': mail_address,
u'serial': serial,
u'ip_country': _ip_country, u'ip_country': _ip_country,
"updated_at": updated_at "updated_at": updated_at
}, },
@@ -105,7 +95,6 @@ class MongoDbManager:
else: else:
collection_to_use.replace_one(filter={'_id': link, }, replacement={ collection_to_use.replace_one(filter={'_id': link, }, replacement={
u'url': link, u'url': link,
u'serial': serial,
u'ip_country': _ip_country, u'ip_country': _ip_country,
"updated_at": updated_at "updated_at": updated_at
}, },
@@ -185,10 +174,7 @@ class MongoDbManager:
validated_by = "Double" validated_by = "Double"
collection.find_one_and_update({'_id': _id}, { collection.find_one_and_update({'_id': _id}, {
"$set": {"url_validated": state, "validated_at": validated_at, "id": _id, "email": linkPojo.email, "$set": {"url_validated": state, "validated_at": validated_at, "id": _id, "email": linkPojo.email,
"url": link, "url": link, "validated_by_ua": ua,
"source_from": linkPojo.model,
"serial": linkPojo.serial,
"validated_by_ua": ua,
"validated_by": validated_by}}, "validated_by": validated_by}},
upsert=True) upsert=True)
# remove the link from db # remove the link from db
docs/4_40_0/jsdata_ch_4_40_0_bitbrowser.json
-130
View File
@@ -1,130 +0,0 @@
{
"ttst": 5.500000000465661,
"ifov": false,
"hc": 8,
"br_oh": 992,
"br_ow": 1920,
"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
"wbd": false,
"tagpu": 1.7174740373540653,
"wdif": false,
"wdifrm": false,
"npmtm": false,
"br_h": 992,
"br_w": 1920,
"isf": true,
"nddc": 1,
"rs_h": 768,
"rs_w": 1024,
"rs_cd": 24,
"phe": false,
"nm": false,
"jsf": false,
"lg": "fr-BE",
"pr": 1,
"ars_h": 743,
"ars_w": 1024,
"tz": -60,
"str_ss": true,
"str_ls": true,
"str_idb": true,
"str_odb": false,
"plgod": false,
"plg": 5,
"plgne": true,
"plgre": true,
"plgof": false,
"plggt": false,
"pltod": false,
"hcovdr": false,
"hcovdr2": false,
"plovdr": false,
"plovdr2": false,
"ftsovdr": false,
"ftsovdr2": false,
"lb": false,
"eva": 33,
"lo": false,
"ts_mtp": 0,
"ts_tec": false,
"ts_tsa": false,
"vnd": "Google Inc.",
"bid": "NA",
"mmt": "application/pdf,text/pdf",
"plu": "PDF Viewer,Chrome PDF Viewer,Chromium PDF Viewer,Microsoft Edge PDF Viewer,WebKit built-in PDF",
"hdn": false,
"awe": false,
"geb": false,
"dat": false,
"med": "defined",
"aco": "probably",
"acots": false,
"acmp": "probably",
"acmpts": true,
"acw": "probably",
"acwts": false,
"acma": "maybe",
"acmats": false,
"acaa": "probably",
"acaats": true,
"ac3": "",
"ac3ts": false,
"acf": "probably",
"acfts": false,
"acmp4": "maybe",
"acmp4ts": false,
"acmp3": "probably",
"acmp3ts": false,
"acwm": "maybe",
"acwmts": false,
"ocpt": false,
"vco": "",
"vcots": false,
"vch": "probably",
"vchts": true,
"vcw": "probably",
"vcwts": true,
"vc3": "maybe",
"vc3ts": false,
"vcmp": "",
"vcmpts": false,
"vcq": "",
"vcqts": false,
"vc1": "probably",
"vc1ts": true,
"dvm": 4,
"sqt": false,
"so": "landscape-primary",
"wdw": true,
"cokys": "bG9hZFRpbWVzY3NpYXBwL=",
"ecpc": false,
"lgs": true,
"lgsod": false,
"psn": true,
"edp": true,
"addt": true,
"wsdc": true,
"ccsr": true,
"nuad": true,
"bcda": true,
"idn": true,
"capi": false,
"svde": false,
"vpbq": true,
"ucdv": false,
"spwn": false,
"emt": false,
"bfr": false,
"dbov": false,
"cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LGk9ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ2MSkpO2lmKGkpeyFmdW5jdGlvbiB0KCl7dHJ5e3ZhciBuPWlbJ1x4NzNceDY4XHg2MVx4NjRceDZmXHg3",
"stcfp": "cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4ODYzNSkKICAgIGF0IE9iamVjdC5kZF9hZCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDc3OTYpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjYzNzkw",
"ckwa": true,
"prm": true,
"cvs": true,
"usb": "defined",
"emd": "k:ai,vi,ao",
"glvd": "Apple Inc.",
"glrd": "Intel HD Graphics 3000 OpenGL Engine",
"wwl": false,
"jset": 1734772335
}
docs/4_40_0/jsdata_ch_4_40_0_mobile.json
-127
View File
@@ -1,127 +0,0 @@
{
"ttst": 22.000000002793968,
"ifov": false,
"hc": 8,
"br_oh": 763,
"br_ow": 393,
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Mobile Safari/537.36",
"wbd": false,
"tagpu": 2.3672133588807087,
"wdif": false,
"wdifrm": false,
"npmtm": false,
"br_h": 763,
"br_w": 393,
"isf": true,
"nddc": 1,
"rs_h": 873,
"rs_w": 393,
"rs_cd": 24,
"phe": false,
"nm": false,
"jsf": false,
"lg": "fr-FR",
"pr": 2.75,
"ars_h": 873,
"ars_w": 393,
"tz": -60,
"str_ss": true,
"str_ls": true,
"str_idb": true,
"str_odb": false,
"plgod": false,
"plg": 0,
"plgne": "NA",
"plgre": "NA",
"plgof": "NA",
"plggt": "NA",
"pltod": false,
"hcovdr": false,
"hcovdr2": false,
"plovdr": false,
"plovdr2": false,
"ftsovdr": false,
"ftsovdr2": false,
"lb": false,
"eva": 33,
"lo": false,
"ts_mtp": 5,
"ts_tec": true,
"ts_tsa": true,
"vnd": "Google Inc.",
"bid": "NA",
"mmt": "empty",
"plu": "empty",
"hdn": false,
"awe": false,
"geb": false,
"dat": false,
"med": "defined",
"aco": "probably",
"acots": false,
"acmp": "probably",
"acmpts": true,
"acw": "probably",
"acwts": false,
"acma": "maybe",
"acmats": false,
"acaa": "probably",
"acaats": true,
"ac3": "",
"ac3ts": false,
"acf": "probably",
"acfts": false,
"acmp4": "maybe",
"acmp4ts": false,
"acmp3": "probably",
"acmp3ts": false,
"acwm": "maybe",
"acwmts": false,
"ocpt": false,
"vco": "",
"vcots": false,
"vch": "probably",
"vchts": true,
"vcw": "probably",
"vcwts": true,
"vc3": "maybe",
"vc3ts": false,
"vcmp": "",
"vcmpts": false,
"vcq": "",
"vcqts": false,
"vc1": "probably",
"vc1ts": true,
"dvm": 8,
"sqt": false,
"so": "portrait-primary",
"wdw": true,
"cokys": "bG9hZFRpbWVzY3NpL=",
"ecpc": false,
"lgs": true,
"lgsod": false,
"psn": true,
"edp": false,
"addt": true,
"wsdc": true,
"ccsr": true,
"nuad": true,
"bcda": true,
"idn": true,
"capi": false,
"svde": false,
"vpbq": true,
"ucdv": false,
"spwn": false,
"emt": false,
"bfr": false,
"dbov": false,
"cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LGk9ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ2MSkpO2lmKGkpeyFmdW5jdGlvbiB0KCl7dHJ5e3ZhciBuPWlbJ1x4NzNceDY4XHg2MVx4NjRceDZmXHg3",
"stcfp": "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4ODYzNSkKICAgIGF0IHQuZXhwb3J0cy5kZF9hZCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDc3OTYpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjYzNzkw",
"ckwa": true,
"emd": "k:ai,vi,ao",
"glvd": "Qualcomm",
"glrd": "Adreno (TM) 642L",
"wwl": false,
"jset": 1734690551
}
docs/4_40_0/jsdata_ch_4_40_0_mobile_generated.json
-127
View File
@@ -1,127 +0,0 @@
{
"ttst": 15.000000002047893,
"ifov": false,
"hc": 5,
"br_oh": 620,
"br_ow": 360,
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36",
"wbd": false,
"tagpu": 9.136966557013412,
"wdif": false,
"wdifrm": false,
"npmtm": false,
"br_h": 620,
"br_w": 360,
"isf": true,
"nddc": 1,
"rs_h": 804,
"rs_w": 360,
"rs_cd": 24,
"phe": false,
"nm": false,
"jsf": false,
"lg": "fr-FR",
"pr": 3,
"ars_h": 804,
"ars_w": 360,
"tz": -120,
"str_ss": true,
"str_ls": true,
"str_idb": true,
"str_odb": true,
"plgod": false,
"plg": 2,
"plgne": "err",
"plgre": "err",
"plgof": "err",
"plggt": "err",
"pltod": false,
"hcovdr": false,
"hcovdr2": false,
"plovdr": false,
"plovdr2": false,
"ftsovdr": false,
"ftsovdr2": false,
"lb": false,
"eva": 33,
"lo": false,
"ts_mtp": 5,
"ts_tec": true,
"ts_tsa": true,
"vnd": "Google Inc.",
"bid": "NA",
"mmt": "empty",
"plu": "eTJjZUp,iZMOHLs1",
"hdn": false,
"awe": false,
"geb": false,
"dat": false,
"med": "defined",
"aco": "probably",
"acots": false,
"acmp": "probably",
"acmpts": true,
"acw": "probably",
"acwts": false,
"acma": "maybe",
"acmats": false,
"acaa": "probably",
"acaats": true,
"ac3": "",
"ac3ts": false,
"acf": "probably",
"acfts": false,
"acmp4": "maybe",
"acmp4ts": false,
"acmp3": "probably",
"acmp3ts": false,
"acwm": "maybe",
"acwmts": false,
"ocpt": false,
"vco": "",
"vcots": false,
"vch": "probably",
"vchts": true,
"vcw": "probably",
"vcwts": true,
"vc3": "maybe",
"vc3ts": false,
"vcmp": "",
"vcmpts": false,
"vcq": "",
"vcqts": false,
"vc1": "probably",
"vc1ts": true,
"dvm": 4,
"sqt": false,
"so": "portrait-primary",
"wdw": true,
"cokys": "bG9hZFRpbWVzY3NpL=",
"ecpc": false,
"lgs": true,
"lgsod": false,
"psn": true,
"edp": false,
"addt": true,
"wsdc": true,
"ccsr": true,
"nuad": true,
"bcda": true,
"idn": true,
"capi": false,
"svde": false,
"vpbq": true,
"ucdv": false,
"spwn": false,
"emt": false,
"bfr": false,
"dbov": false,
"cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LGk9ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ2MSkpO2lmKGkpeyFmdW5jdGlvbiB0KCl7dHJ5e3ZhciBuPWlbJ1x4NzNceDY4XHg2MVx4NjRceDZmXHg3",
"stcfp": "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4ODYzNSkKICAgIGF0IHQuZXhwb3J0cy5kZF9hZCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDc3OTYpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjYzNzkw",
"ckwa": true,
"emd": "k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801",
"glvd": "ARM",
"glrd": "Mali-G57 MC2",
"wwl": false,
"jset": 1734775162
}
docs/4_40_0/jsdata_le_4_40_0_mobile.json
-151
View File
@@ -1,151 +0,0 @@
{
"ttst": 46.8000000026077,
"ifov": false,
"hc": 8,
"br_oh": 763,
"br_ow": 393,
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Mobile Safari/537.36",
"wbd": false,
"tagpu": 7.9067645182878366,
"wdif": false,
"wdifrm": false,
"npmtm": false,
"br_h": 763,
"br_w": 393,
"isf": true,
"nddc": 1,
"rs_h": 873,
"rs_w": 393,
"rs_cd": 24,
"phe": false,
"nm": false,
"jsf": false,
"lg": "fr-FR",
"pr": 2.75,
"ars_h": 873,
"ars_w": 393,
"tz": -60,
"str_ss": true,
"str_ls": true,
"str_idb": true,
"str_odb": false,
"plgod": false,
"plg": 0,
"plgne": "NA",
"plgre": "NA",
"plgof": "NA",
"plggt": "NA",
"pltod": false,
"hcovdr": false,
"hcovdr2": false,
"plovdr": false,
"plovdr2": false,
"ftsovdr": false,
"ftsovdr2": false,
"lb": false,
"eva": 33,
"lo": false,
"ts_mtp": 5,
"ts_tec": true,
"ts_tsa": true,
"vnd": "Google Inc.",
"bid": "NA",
"mmt": "empty",
"plu": "empty",
"hdn": false,
"awe": false,
"geb": false,
"dat": false,
"med": "defined",
"aco": "probably",
"acots": false,
"acmp": "probably",
"acmpts": true,
"acw": "probably",
"acwts": false,
"acma": "maybe",
"acmats": false,
"acaa": "probably",
"acaats": true,
"ac3": "",
"ac3ts": false,
"acf": "probably",
"acfts": false,
"acmp4": "maybe",
"acmp4ts": false,
"acmp3": "probably",
"acmp3ts": false,
"acwm": "maybe",
"acwmts": false,
"ocpt": false,
"vco": "",
"vcots": false,
"vch": "probably",
"vchts": true,
"vcw": "probably",
"vcwts": true,
"vc3": "maybe",
"vc3ts": false,
"vcmp": "",
"vcmpts": false,
"vcq": "",
"vcqts": false,
"vc1": "probably",
"vc1ts": true,
"dvm": 8,
"sqt": false,
"so": "portrait-primary",
"wdw": true,
"cokys": "bG9hZFRpbWVzY3NpL=",
"ecpc": false,
"lgs": true,
"lgsod": false,
"psn": true,
"edp": false,
"addt": true,
"wsdc": true,
"ccsr": true,
"nuad": true,
"bcda": true,
"idn": true,
"capi": false,
"svde": false,
"vpbq": true,
"ucdv": false,
"spwn": false,
"emt": false,
"bfr": false,
"dbov": false,
"cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LGk9ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ2MSkpO2lmKGkpeyFmdW5jdGlvbiB0KCl7dHJ5e3ZhciBuPWlbJ1x4NzNceDY4XHg2MVx4NjRceDZmXHg3",
"stcfp": "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4ODYzNSkKICAgIGF0IHQuZXhwb3J0cy5kZF9hZCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDc3OTYpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjYzNzkw",
"ckwa": true,
"glvd": "Qualcomm",
"glrd": "Adreno (TM) 642L",
"wwl": false,
"tzp": "Europe/Paris",
"jset": 1734690519,
"emd": "k:ai,vi,ao",
"dcok": ".hermes.com",
"m_fmi": false,
"tbce": 0,
"es_sigmdn": null,
"es_mumdn": null,
"es_distmdn": null,
"es_angsmdn": null,
"es_angemdn": null,
"k_hA": 0,
"k_hSD": 0,
"k_pA": 862.6666666666666,
"k_pSD": 1159.6290594649463,
"k_rA": 862.6666666666666,
"k_rSD": 1159.6290594649463,
"k_ikA": 862.6666666666666,
"k_ikSD": 1159.6290594649463,
"k_kdc": 7,
"k_kuc": 7,
"m_s_c": 30,
"m_m_c": 3,
"m_c_c": 3,
"m_cm_r": 1,
"m_ms_r": 0.1
}
docs/4_40_0/jsdata_le_4_40_0_mobile_generated.json
-149
View File
@@ -1,149 +0,0 @@
{
"ttst": 13.000000008223282,
"ifov": false,
"hc": 5,
"br_oh": 620,
"br_ow": 360,
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36",
"wbd": false,
"tagpu": 5.501734809203354,
"wdif": false,
"wdifrm": false,
"npmtm": false,
"br_h": 620,
"br_w": 360,
"isf": true,
"nddc": 1,
"rs_h": 804,
"rs_w": 360,
"rs_cd": 24,
"phe": false,
"nm": false,
"jsf": false,
"lg": "fr-FR",
"pr": 3,
"ars_h": 804,
"ars_w": 360,
"tz": -120,
"str_ss": true,
"str_ls": true,
"str_idb": true,
"str_odb": true,
"plgod": false,
"plg": 2,
"plgne": "err",
"plgre": "err",
"plgof": "err",
"plggt": "err",
"pltod": false,
"hcovdr": false,
"hcovdr2": false,
"plovdr": false,
"plovdr2": false,
"ftsovdr": false,
"ftsovdr2": false,
"lb": false,
"eva": 33,
"lo": false,
"ts_mtp": 5,
"ts_tec": true,
"ts_tsa": true,
"vnd": "Google Inc.",
"bid": "NA",
"mmt": "empty",
"plu": "eTJjZUp,iZMOHLs1",
"hdn": false,
"awe": false,
"geb": false,
"dat": false,
"med": "defined",
"aco": "probably",
"acots": false,
"acmp": "probably",
"acmpts": true,
"acw": "probably",
"acwts": false,
"acma": "maybe",
"acmats": false,
"acaa": "probably",
"acaats": true,
"ac3": "",
"ac3ts": false,
"acf": "probably",
"acfts": false,
"acmp4": "maybe",
"acmp4ts": false,
"acmp3": "probably",
"acmp3ts": false,
"acwm": "maybe",
"acwmts": false,
"ocpt": false,
"vco": "",
"vcots": false,
"vch": "probably",
"vchts": true,
"vcw": "probably",
"vcwts": true,
"vc3": "maybe",
"vc3ts": false,
"vcmp": "",
"vcmpts": false,
"vcq": "",
"vcqts": false,
"vc1": "probably",
"vc1ts": true,
"dvm": 4,
"sqt": false,
"so": "portrait-primary",
"wdw": true,
"cokys": "bG9hZFRpbWVzY3NpL=",
"ecpc": false,
"lgs": true,
"lgsod": false,
"psn": true,
"edp": false,
"addt": true,
"wsdc": true,
"ccsr": true,
"nuad": true,
"bcda": true,
"idn": true,
"capi": false,
"svde": false,
"vpbq": true,
"ucdv": false,
"spwn": false,
"emt": false,
"bfr": false,
"dbov": false,
"cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LGk9ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ2MSkpO2lmKGkpeyFmdW5jdGlvbiB0KCl7dHJ5e3ZhciBuPWlbJ1x4NzNceDY4XHg2MVx4NjRceDZmXHg3",
"stcfp": "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4ODYzNSkKICAgIGF0IHQuZXhwb3J0cy5kZF9hZCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDc3OTYpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjYzNzkw",
"ckwa": true,
"emd": "k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801",
"glvd": "ARM",
"glrd": "Mali-G57 MC2",
"wwl": false,
"jset": 1734775843,
"dcok": ".hermes.com",
"log2": true,
"es_sigmdn": null,
"es_mumdn": null,
"es_distmdn": null,
"es_angsmdn": null,
"es_angemdn": null,
"k_hA": null,
"k_hSD": null,
"k_pA": null,
"k_pSD": null,
"k_rA": null,
"k_rSD": null,
"k_ikA": null,
"k_ikSD": null,
"k_kdc": 0,
"k_kuc": 0,
"m_s_c": 0,
"m_m_c": 3,
"m_c_c": 3,
"m_cm_r": 1,
"m_ms_r": -1
}
link_validator_executor.py
+5 -2
View File
@@ -2,11 +2,14 @@ import time
from workers.link_validator_with_provided_list import validate_all_links from workers.link_validator_with_provided_list import validate_all_links
# if __name__ == '__main__':
# # link_list = MONGO_STORE_MANAGER.get_links_to_validate()
# validate_with_FR_ip(segment_position=2)
if __name__ == '__main__': if __name__ == '__main__':
# generate test data # generate test data
while True: while True:
print("call validate_all_links()") print("call validate_all_links()")
validate_all_links([]) validate_all_links()
print("wait for 30 seconds") print("wait for 30 seconds")
time.sleep(10) time.sleep(30)
mail/mail_constants.py
+18 -36
View File
@@ -86,65 +86,47 @@ def create_imap(login: str):
if DOMAIN_163 in login: if DOMAIN_163 in login:
imap = IMAPClient(IMAP_SERVER_163, use_uid=True) imap = IMAPClient(IMAP_SERVER_163, use_uid=True)
elif DOMAIN_YAHOO in login: elif DOMAIN_YAHOO in login:
# imap = imaplib.IMAP4_SSL(YAHOO_IMAP_SERVER) imap = imaplib.IMAP4_SSL(YAHOO_IMAP_SERVER)
imap = IMAPClient(YAHOO_IMAP_SERVER, use_uid=True)
elif DOMAIN_FIREMAIL_DE in login: elif DOMAIN_FIREMAIL_DE in login:
# imap = imaplib.IMAP4_SSL(SERVER_FIREMAIL_DE) # imap = imaplib.IMAP4_SSL(SERVER_FIREMAIL_DE)
imap = IMAPClient(SERVER_FIREMAIL_DE, use_uid=True) imap = IMAPClient(SERVER_FIREMAIL_DE, use_uid=True)
elif DOMAIN_GMX in login or DOMAIN_GMX_FR in login or DOMAIN_GMX_US in login or DOMAIN_GMX_CH in login or DOMAIN_GMX_PT in login or DOMAIN_GMX_SG in login: elif DOMAIN_GMX in login or DOMAIN_GMX_FR in login or DOMAIN_GMX_US in login or DOMAIN_GMX_CH in login or DOMAIN_GMX_PT in login or DOMAIN_GMX_SG in login:
# imap = imaplib.IMAP4_SSL(SERVER_GMX) imap = imaplib.IMAP4_SSL(SERVER_GMX)
imap = IMAPClient(SERVER_GMX, use_uid=True)
elif DOMAIN_SINA in login: elif DOMAIN_SINA in login:
# imap = imaplib.IMAP4_SSL(IMAP_SERVER_SINA) # imap = imaplib.IMAP4_SSL(IMAP_SERVER_SINA)
imap = IMAPClient(IMAP_SERVER_SINA, use_uid=True) imap = IMAPClient(IMAP_SERVER_SINA, use_uid=True)
elif DOMAIN_HOTMAIL in login or DOMAIN_OUTLOOK_COM in login: elif DOMAIN_HOTMAIL in login or DOMAIN_OUTLOOK_COM in login:
# imap = imaplib.IMAP4_SSL(HOTMAIL_IMAP_SERVER) imap = imaplib.IMAP4_SSL(HOTMAIL_IMAP_SERVER)
imap = IMAPClient(HOTMAIL_IMAP_SERVER, use_uid=True)
elif DOMAIN_RAMBLER_RU in login: elif DOMAIN_RAMBLER_RU in login:
# imap = imaplib.IMAP4_SSL(RAMBLER_IMAP_SERVER) imap = imaplib.IMAP4_SSL(RAMBLER_IMAP_SERVER)
imap = IMAPClient(RAMBLER_IMAP_SERVER, use_uid=True)
elif DOMAN_BTVM_NE_JP in login: elif DOMAN_BTVM_NE_JP in login:
# imap = imaplib.IMAP4_SSL(BTVM_NE_JP) imap = imaplib.IMAP4_SSL(BTVM_NE_JP)
imap = IMAPClient(BTVM_NE_JP, use_uid=True)
elif DOMAN_GMAIL in login: elif DOMAN_GMAIL in login:
# imap = imaplib.IMAP4_SSL(SEREVER_GMAIL, port=993) imap = imaplib.IMAP4_SSL(SEREVER_GMAIL, port=993)
imap = IMAPClient(SEREVER_GMAIL, use_uid=True)
elif DOMAIN_ONET in login: elif DOMAIN_ONET in login:
imap = IMAPClient(SERVER_IMAGE_ONET, use_uid=True) imap = IMAPClient(SERVER_IMAGE_ONET, use_uid=True)
elif DOMAIN_TIM_IT in login: elif DOMAIN_TIM_IT in login:
# imap = imaplib.IMAP4(TIME_IT_SERVER) imap = imaplib.IMAP4(TIME_IT_SERVER)
imap = IMAPClient(TIME_IT_SERVER, use_uid=True)
elif DOMAIN_ALICE_IT in login: elif DOMAIN_ALICE_IT in login:
# imap = imaplib.IMAP4(ALICE_IMAP_SERVER, port=143) imap = imaplib.IMAP4(ALICE_IMAP_SERVER, port=143)
imap = IMAPClient(ALICE_IMAP_SERVER, use_uid=True)
elif DOMAIN_MARS_DTI_NE_JP in login: elif DOMAIN_MARS_DTI_NE_JP in login:
# imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143) imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143)
imap = IMAPClient(MARS_DTI_NE_JP_SERVER, use_uid=True)
elif DOMAN_AURORA_DTI_NE_JP in login: elif DOMAN_AURORA_DTI_NE_JP in login:
# imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143) imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143)
imap = IMAPClient(MARS_DTI_NE_JP_SERVER, use_uid=True)
elif DOMAIN_NAVER in login: elif DOMAIN_NAVER in login:
# imap = imaplib.IMAP4_SSL(NAVER_SERVER, port=993) imap = imaplib.IMAP4_SSL(NAVER_SERVER, port=993)
imap = IMAPClient(NAVER_SERVER, use_uid=True)
elif DOMAIN_GMX_DE in login or DOMAIN_GMX_NET in login: elif DOMAIN_GMX_DE in login or DOMAIN_GMX_NET in login:
# imap = imaplib.IMAP4_SSL(SERVER_GMX_NET, port=993) imap = imaplib.IMAP4_SSL(SERVER_GMX_NET, port=993)
imap = IMAPClient(SERVER_GMX_NET, use_uid=True)
elif DOMAIN_GMX_AT in login: elif DOMAIN_GMX_AT in login:
# imap = imaplib.IMAP4_SSL(SERVER_GMX_AT, port=993) imap = imaplib.IMAP4_SSL(SERVER_GMX_AT, port=993)
imap = IMAPClient(SERVER_GMX_AT, use_uid=True)
elif DOMAIN_GAZETA_PL in login: elif DOMAIN_GAZETA_PL in login:
# imap = imaplib.IMAP4_SSL(IMAP_SERVER_DOMAIN_GAZETA_PL, port=993) imap = imaplib.IMAP4_SSL(IMAP_SERVER_DOMAIN_GAZETA_PL, port=993)
imap = IMAPClient(IMAP_SERVER_DOMAIN_GAZETA_PL, use_uid=True)
elif DOMAIN_INBOX_LV in login: elif DOMAIN_INBOX_LV in login:
# imap = imaplib.IMAP4_SSL(INBOX_LV, port=993) imap = imaplib.IMAP4_SSL(INBOX_LV, port=993)
imap = IMAPClient(INBOX_LV, use_uid=True)
elif DOMAIN_WEB_DE in login: elif DOMAIN_WEB_DE in login:
# imap = imaplib.IMAP4_SSL(SERVER_WEB_DE, port=993) imap = imaplib.IMAP4_SSL(SERVER_WEB_DE, port=993)
imap = IMAPClient(SERVER_WEB_DE, use_uid=True)
elif DOMAIN_PISS_MAIL in login or DOMAIN_CHILD_PIZZA in login or DOMAIN_DMC_CHAT in login or DOMAIN_GENOCIDE_FUN in login or DOMAIN_HATESJE_WS in login or DOMAIN_INCEL_EMAIL in login or DOMAIN_SHITPOSTING_EXPERT in login: elif DOMAIN_PISS_MAIL in login or DOMAIN_CHILD_PIZZA in login or DOMAIN_DMC_CHAT in login or DOMAIN_GENOCIDE_FUN in login or DOMAIN_HATESJE_WS in login or DOMAIN_INCEL_EMAIL in login or DOMAIN_SHITPOSTING_EXPERT in login:
# imap = imaplib.IMAP4_SSL(SERVER_PISS_MAIL, port=993) imap = imaplib.IMAP4_SSL(SERVER_PISS_MAIL, port=993)
imap = IMAPClient(SERVER_PISS_MAIL, use_uid=True)
else: else:
# imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER) imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER)
imap = IMAPClient(AOL_IMAP_SERVER, use_uid=True)
return imap return imap
mail/mail_reader_all_contacts.py
+7 -41
View File
@@ -9,7 +9,6 @@ from email.message import Message
from imapclient import IMAPClient from imapclient import IMAPClient
from db.mongo_manager import MONGO_STORE_MANAGER from db.mongo_manager import MONGO_STORE_MANAGER
from excel_reader import read_contacts
from mail.mail_constants import DOMAIN_HOTMAIL, create_imap from mail.mail_constants import DOMAIN_HOTMAIL, create_imap
from models.mail_pojo import MailPojo, MailAddress from models.mail_pojo import MailPojo, MailAddress
@@ -49,11 +48,6 @@ def find_from_mail(param):
return from_address.strip(" ").strip(">").strip("<") return from_address.strip(" ").strip(">").strip("<")
def extract_email_from_from_address(content: str):
_match = re.search(r'[\w.+-]+@[\w-]+\.[\w.-]+', content)
return _match.group(0)
class MailReader(): class MailReader():
def __init__(self, login, password): def __init__(self, login, password):
self.login = login self.login = login
@@ -176,27 +170,23 @@ class MailReader():
email_message = email.message_from_bytes(message_data[b'RFC822']) email_message = email.message_from_bytes(message_data[b'RFC822'])
from_address = email_message.get('FROM') from_address = email_message.get('FROM')
subject = email_message.get('subject') subject = email_message.get('subject')
# print("{}, {},{}".format(from_address, subject, email_message))
body = "" body = ""
hermes_mail_adress = "no-reply@hermes.com" if "no-reply@hermes.com" in from_address or "appointment2022@aol.com":
if hermes_mail_adress in from_address or "outlook.com" in from_address or "hotmail" in from_address:
for part in email_message.walk(): for part in email_message.walk():
print(part.get_content_type()) print(part.get_content_type())
if part.get_content_type() == "text/html": if part.get_content_type() == "text/html":
body = body + part.get_payload(decode=True).decode("utf-8") body = body + part.get_payload(decode=True).decode("utf-8")
elif part.get_content_type() == "text/plain": elif part.get_content_type() == "text/plain":
body = body + part.get_payload() body = body + part.get_payload()
if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject or "Votre=20demande=20de=20rendez-vous" in subject or "Votre demande de rendez-vous" in body: if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject or "Validation=20de=20votre=20demande=20de=20rendez-vous" in subject:
mail = MailPojo(subject=subject, body=body, from_address=from_address) mail = MailPojo(subject=subject, body=body, from_address=from_address)
mail.isImapClient = True mail.isImapClient = True
print("email is {}".format(self.login)) print("email is {}".format(self.login))
print("body is {}".format(body)) print("body is {}".format(body))
print("subject is {}".format(subject)) print("subject is {}".format(subject))
if len(mail.to_address) == 0: if len(mail.to_address) == 0:
if "outlook.com" in from_address or "hotmail.com" in from_address: mail.to_address = self.login
# it is a transferred email
mail.to_address = extract_email_from_from_address(from_address)
else:
mail.to_address = self.login
mail_messages.append(mail) mail_messages.append(mail)
except Exception as error: except Exception as error:
print(error) print(error)
@@ -272,31 +262,7 @@ def find_links_to_validate_from_mail_list(mail_list: list, logger):
if __name__ == '__main__': if __name__ == '__main__':
# mail_address1 = MailAddress(mail="tinagonzales685585@aol.com", password="yhihvdkrbxnksema") mail_address1 = MailAddress(mail="tzfhygwwjiwnf@hotmail.com", password="rqO4Pjm7NQ")
# mail_list = [mail_address1] mail_list = [mail_address1]
contact_to_book_list = read_contacts(
# file_name="/Users/rdv/Desktop/contact_list_not_used_contacts.xlsx")
# file_name="/Users/lpan/Desktop/contact_list_not_used_contacts.xlsx")
# file_name="/Users/rdv/Desktop/real_name_contacts_100_hotmail.xlsx")
file_name="/Users/rdv/Desktop/contact_list_2025-06-23.xlsx")
# file_name="/Users/rdv/Desktop/contact_list_all_studio.xlsx")
# file_name="/Users/rdv/Desktop/contact_list_all_studo_gmx_us.xlsx")
# file_name="/Users/rdv/Desktop/contact_list_2025-05-24.xlsx")
all_mail_list = MONGO_STORE_MANAGER.get_destination_emails()
mail_list_to_check = []
for contact in contact_to_book_list:
for mail in all_mail_list:
if contact.mail == mail.mail:
mail_list_to_check.append(mail)
logger = logging.getLogger() logger = logging.getLogger()
_all_links = MONGO_STORE_MANAGER.get_links_to_validate() find_links_to_validate_from_mail_list(mail_list, logger)
filter_mail = []
for mail_pojo in mail_list_to_check:
_to_add = True
for _link in _all_links:
if _link.email == mail_pojo.mail:
_to_add = False
if _to_add:
filter_mail.append(mail_pojo)
filter_mail.append(MailAddress("saigecong1990@pissmail.com", "cvExXKOP8oY1D@"))
find_links_to_validate_from_mail_list(filter_mail, logger)
migrate_to_data_cache_2.py
+2 -2
View File
@@ -3,6 +3,6 @@ from queue_message.CookiesPublisher import MORNING_DATA_CACHE_2, MORNING_DATA_CA
from workers.MessagerTransporter import migrate_message_to_queue, empty_message_just_to from workers.MessagerTransporter import migrate_message_to_queue, empty_message_just_to
if __name__ == '__main__': if __name__ == '__main__':
migrate_message_to_queue(from_queue=MORNING_DATA_CACHE, to_queue=MORNING_DATA_CACHE_BAK) # migrate_message_to_queue(from_queue=MORNING_DATA_CACHE_BAK, to_queue=MORNING_DATA_CACHE)
# empty_message_just_to(2000, queue_name=MORNING_DATA_CACHE_BAK) empty_message_just_to(20000, queue_name=MORNING_DATA_CACHE_BAK)
# migrate_message_to_queue(from_queue=MORNING_DATA_CACHE_BAK,to_queue=REQUEST_DATA_QUEUE_DE) # migrate_message_to_queue(from_queue=MORNING_DATA_CACHE_BAK,to_queue=REQUEST_DATA_QUEUE_DE)
models/LinkPojo.py
+2 -9
View File
@@ -1,9 +1,7 @@
class LinkPojo(): class LinkPojo():
def __init__(self, url, email, model, serial, updated_at, ip_country): def __init__(self, url, email, updated_at, ip_country):
self.url = url self.url = url
self.email = email self.email = email
self.model = model
self.serial = serial
self.updated_at = updated_at self.updated_at = updated_at
self.ip_country = ip_country self.ip_country = ip_country
@@ -11,14 +9,9 @@ class LinkPojo():
def from_firestore_dict(source): def from_firestore_dict(source):
updated_at = source['updated_at'] updated_at = source['updated_at']
email = source['email'] email = source['email']
model = ""
if 'model' in source:
model = source['model']
serial = source['serial']
url = source['url'] url = source['url']
ip_country = "FR" ip_country = "FR"
if source.get('ip_country'): if source.get('ip_country'):
ip_country = source['ip_country'] ip_country = source['ip_country']
result = LinkPojo(email=email, url=url, model=model, serial=serial, updated_at=updated_at, result = LinkPojo(email=email, url=url, updated_at=updated_at, ip_country=ip_country)
ip_country=ip_country)
return result return result
models/ReserveResultPojo.py
+2 -2
View File
@@ -29,7 +29,6 @@ class ReserveResultPojo:
slot_position = None slot_position = None
sim_position = None sim_position = None
ccid: str = "" ccid: str = ""
model: str = ""
source_from: str = socket.gethostname() source_from: str = socket.gethostname()
store_type = 0 store_type = 0
url_validated = None url_validated = None
@@ -123,7 +122,8 @@ class ReserveResultPojo:
u'email': self.email, u'email': self.email,
u'passport': self.passport, u'passport': self.passport,
u'url': self.url, u'url': self.url,
u'model': self.model, # u'sim_position': self.sim_position,
# u'slot_position': self.slot_position,
u'source_from': self.source_from, u'source_from': self.source_from,
u'hostName': self.source_from, u'hostName': self.source_from,
u'created_at': self.created_at, u'created_at': self.created_at,
models/contact_pojo.py
-7
View File
@@ -13,7 +13,6 @@ class ContactPojo:
store: str store: str
note: str note: str
ua: str ua: str
serial: str
def __init__(self, phone_number: str, passport_number: str, last_name: str, first_name: str, mail: str, def __init__(self, phone_number: str, passport_number: str, last_name: str, first_name: str, mail: str,
ccid: str = "", ccid: str = "",
@@ -28,7 +27,6 @@ class ContactPojo:
self.note = "" self.note = ""
self.ua = "" self.ua = ""
self.current_ip = "" self.current_ip = ""
self.serial = "reqeusts"
def to_firestore_dict(self): def to_firestore_dict(self):
dest = { dest = {
@@ -40,7 +38,6 @@ class ContactPojo:
u'store': self.store, u'store': self.store,
u'ccid': self.ccid, u'ccid': self.ccid,
u'position': self.position, u'position': self.position,
u'serial': self.serial,
u'current_ip': self.current_ip, u'current_ip': self.current_ip,
u'ua': self.ua u'ua': self.ua
} }
@@ -55,13 +52,9 @@ class ContactPojo:
last_name = source['last_name'] last_name = source['last_name']
first_name = source['first_name'] first_name = source['first_name']
ip_country = "FR" ip_country = "FR"
if source.get('ip_country'): if source.get('ip_country'):
ip_country = source['ip_country'] ip_country = source['ip_country']
result = ContactPojo(phone_number=phone, passport_number=passport, mail=email, result = ContactPojo(phone_number=phone, passport_number=passport, mail=email,
last_name=last_name, first_name=first_name) last_name=last_name, first_name=first_name)
result.ip_country = ip_country result.ip_country = ip_country
if source.get('serial'):
serial = source['serial']
result.serial = serial
return result return result
models/jsdata_le_pojo.py
+2 -2
View File
@@ -50,8 +50,8 @@ class JsDataLeTypePojo(JsDataPojo):
self.m_c_c = m_c_c self.m_c_c = m_c_c
self.m_cm_r = m_cm_r self.m_cm_r = m_cm_r
self.m_ms_r = m_ms_r self.m_ms_r = m_ms_r
self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LGk9ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ2MSkpO2lmKGkpeyFmdW5jdGlvbiB0KCl7dHJ5e3ZhciBuPWlbJ1x4NzNceDY4XHg2MVx4NjRceDZmXHg3" # 4.40.0 self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.35.0
self.stcfp = "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4ODYzNSkKICAgIGF0IHQuZXhwb3J0cy5kZF9hZCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDc3OTYpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjYzNzkw" # 4.40.0 self.stcfp = "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4MjIzMikKICAgIGF0IHQuZXhwb3J0cy5kZF9hYyAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDA3MDEpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU3ODc0" # 4.35.0
test_data_json = """{"glvd":"ARM","glrd":"Mali-G57 MC2","hc":5,"br_oh":620,"br_ow":360,"br_h":620,"br_w":360,"rs_h":804,"rs_w":360,"rs_cd":24,"ars_h":804,"ars_w":360,"plg":2,"eva":33,"vnd":"Google Inc.","plu":["eTJjZUp","iZMOHLs1"],"ua":"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36","dvm":4,"acw":"probably","pr":3,"ts_mtp":5,"tz":-120,"emd":"k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801","cookiesStr":"app.sig=_FqS3_SPN2D40Z4Qlq-_ybxzRvQ;datadome=VSGCNMYkY6L81typRJUpurlyd39dHxdfxIYc6DzRWDT7_rtZ8UodGcbow1okQR2zg_t4d_NcP0rC1ZHNz3XSDAGFZCJhFI49lai~x2Nb2unkhOlY8o4E04Wjmde~auDZ;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiLW9jbktLVXAzMjVTOEFEdFpLUU9kSXlMIn0=;policy=accepted;lang=fr;"}""" test_data_json = """{"glvd":"ARM","glrd":"Mali-G57 MC2","hc":5,"br_oh":620,"br_ow":360,"br_h":620,"br_w":360,"rs_h":804,"rs_w":360,"rs_cd":24,"ars_h":804,"ars_w":360,"plg":2,"eva":33,"vnd":"Google Inc.","plu":["eTJjZUp","iZMOHLs1"],"ua":"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36","dvm":4,"acw":"probably","pr":3,"ts_mtp":5,"tz":-120,"emd":"k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801","cookiesStr":"app.sig=_FqS3_SPN2D40Z4Qlq-_ybxzRvQ;datadome=VSGCNMYkY6L81typRJUpurlyd39dHxdfxIYc6DzRWDT7_rtZ8UodGcbow1okQR2zg_t4d_NcP0rC1ZHNz3XSDAGFZCJhFI49lai~x2Nb2unkhOlY8o4E04Wjmde~auDZ;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiLW9jbktLVXAzMjVTOEFEdFpLUU9kSXlMIn0=;policy=accepted;lang=fr;"}"""
models/jsdata_pojo.py
+3 -3
View File
@@ -138,7 +138,7 @@ class JsDataPojo:
def __init__(self, glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu, def __init__(self, glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu,
vnd, pr, ts_mtp, dvm, emd): vnd, pr, ts_mtp, dvm, emd):
# self.opts = "endpoint,ajaxListenerPath" # self.opts = "endpoint,ajaxListenerPath"
self.ttst = (random.randint(5, 46) * 10 ** 16 + random.randint(1 * 10 ** 7, 9 * 10 ** 7)) / 10 ** 16 self.ttst = random.randint(38, 148)
self.ifov = False self.ifov = False
self.hc = hc self.hc = hc
self.br_oh = br_oh self.br_oh = br_oh
@@ -266,8 +266,8 @@ class JsDataPojo:
self.emt = False self.emt = False
self.bfr = False self.bfr = False
self.dbov = False self.dbov = False
self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LGk9ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ2MSkpO2lmKGkpeyFmdW5jdGlvbiB0KCl7dHJ5e3ZhciBuPWlbJ1x4NzNceDY4XHg2MVx4NjRceDZmXHg3" # 4.40.0 self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.35.0
self.stcfp = "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4ODYzNSkKICAgIGF0IHQuZXhwb3J0cy5kZF9hZCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDc3OTYpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjYzNzkw" # 4.40.0 self.stcfp = "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4MjIzMikKICAgIGF0IHQuZXhwb3J0cy5kZF9hYyAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDA3MDEpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU3ODc0" # 4.35.0
self.ckwa = True self.ckwa = True
def to_url_encoded_json(self): def to_url_encoded_json(self):
proxy_manager/proxy_manager.py
+20 -46
View File
@@ -1,8 +1,5 @@
import logging
import random import random
import requests
FR_ASOCKS_MOBILE_PROXY = { FR_ASOCKS_MOBILE_PROXY = {
'http': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046', 'http': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046',
'https': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046', 'https': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046',
@@ -12,15 +9,10 @@ FR_PROXY_RES_OXY = {
'http': 'http://customer-rendezvous-cc-FR:Rdv202220212023@pr.oxylabs.io:7777', 'http': 'http://customer-rendezvous-cc-FR:Rdv202220212023@pr.oxylabs.io:7777',
'https': 'http://customer-rendezvous-cc-FR:Rdv202220212023@pr.oxylabs.io:7777' 'https': 'http://customer-rendezvous-cc-FR:Rdv202220212023@pr.oxylabs.io:7777'
} }
FR_PROXY_RES_PARIS_OXY = {
# customer-rendezvous-cc-fr-city-paris:PASSWORD@pr.oxylabs.io:7777
'http': 'http://customer-rendezvous-cc-fr-city-paris:Rdv202220212023@pr.oxylabs.io:7777',
'https': 'http://customer-rendezvous-cc-fr-city-paris:Rdv202220212023@pr.oxylabs.io:7777'
}
FR_PROXY_MOB_OXY = { FR_PROXY_MOB_OXY = {
'http': 'http://customer-rendezvousmob-cc-FR:Rdv+20222021@pr.oxylabs.io:7777', 'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@pr.oxylabs.io:7777',
'https': 'http://customer-rendezvousmob-cc-FR:Rdv+20222021@pr.oxylabs.io:7777' 'https': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@pr.oxylabs.io:7777'
} }
FR_PROXY_ASOCK_RES_2 = { FR_PROXY_ASOCK_RES_2 = {
@@ -28,8 +20,8 @@ FR_PROXY_ASOCK_RES_2 = {
'https': 'http://zd6fbrujot-res-country-FR-hold-query:8k8avNlnLHQaMsWg@217.23.6.161:9999' 'https': 'http://zd6fbrujot-res-country-FR-hold-query:8k8avNlnLHQaMsWg@217.23.6.161:9999'
} }
FR_PROXY_MOB_OXY_STICKY = { FR_PROXY_MOB_OXY_STICKY = {
'http': 'http://customer-rendezvousmob-cc-fr-sessid-0{}-sesstime-2:Rdv+202220212023@pr.oxylabs.io:7777', 'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}',
'https': 'http://customer-rendezvousmob-cc-fr-sessid-0{}-sesstime-2:Rdv+202220212023@pr.oxylabs.io:7777' 'https': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}'
} }
FR_PROXY_DATA_IMPULSE_STICKY = { FR_PROXY_DATA_IMPULSE_STICKY = {
@@ -50,21 +42,17 @@ FR_MOBILE_ANY_IP_STICKY = {
'http': 'http://user_6a7f21,type_mobile,country_FR,session_{}:d5c051@portal.anyip.io:1080', 'http': 'http://user_6a7f21,type_mobile,country_FR,session_{}:d5c051@portal.anyip.io:1080',
'https': 'http://user_6a7f21,type_mobile,country_FR,session_{}:d5c051@portal.anyip.io:1080', 'https': 'http://user_6a7f21,type_mobile,country_FR,session_{}:d5c051@portal.anyip.io:1080',
} }
FR_MOBILE_ANY_IP_ROTATING = {
'http': 'http://user_6a7f21,type_residential,country_FR:d5c051@portal.anyip.io:1080',
'https': 'http://user_6a7f21,type_residential,country_FR:d5c051@portal.anyip.io:1080',
}
# 八分之一用data_impulse # 八分之一用data_impulse
# MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY,
# FR_PROXY_MOB_OXY_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY,
# FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY,
# FR_PROXY_RES_OXY_STICKY, FR_PROXY_RES_OXY_STICKY,
# FR_PROXY_DATA_IMPULSE_STICKY] FR_PROXY_DATA_IMPULSE_STICKY]
# MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY]
# MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY, FR_PROXY_MOB_OXY_STICKY] # MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY]
# MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY]
MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY]
class ProxyManager: class ProxyManager:
@@ -73,16 +61,13 @@ class ProxyManager:
self.logger = logger self.logger = logger
def get_link_validate_proxy(self, links_to_validate: list) -> list: def get_link_validate_proxy(self, links_to_validate: list) -> list:
# return [FR_PROXY_RES_PARIS_OXY] if len(links_to_validate) > 15:
# return [FR_MOBILE_ANY_IP_ROTATING] return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
return [FR_PROXY_RES_OXY] # return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY]
# if len(links_to_validate) > 15: else:
# return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
# # return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY] return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
# else: # return [FR_PROXY_RES_OXY]
#
# return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
# return [FR_PROXY_RES_OXY]
def get_result_link_proxy(self): def get_result_link_proxy(self):
return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY] return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY]
@@ -91,7 +76,7 @@ class ProxyManager:
_chosen_proxy = random.choice(MOBILE_PROXY_LIST) _chosen_proxy = random.choice(MOBILE_PROXY_LIST)
if "oxylabs" in _chosen_proxy["http"]: if "oxylabs" in _chosen_proxy["http"]:
self.logger.info("use oxylabs proxy") self.logger.info("use oxylabs proxy")
_port = random.randint(900000000, 995869818) _port = random.randint(40001, 49999)
elif "anyip" in _chosen_proxy["http"]: elif "anyip" in _chosen_proxy["http"]:
self.logger.info("use anyip proxy") self.logger.info("use anyip proxy")
_port = random.randint(40001, 49999) _port = random.randint(40001, 49999)
@@ -103,14 +88,3 @@ class ProxyManager:
_proxy_to_use["http"] = _chosen_proxy["http"].format(_port) _proxy_to_use["http"] = _chosen_proxy["http"].format(_port)
_proxy_to_use["https"] = _chosen_proxy["https"].format(_port) _proxy_to_use["https"] = _chosen_proxy["https"].format(_port)
return _proxy_to_use return _proxy_to_use
if __name__ == '__main__':
_logger = logging.getLogger()
_proxy = ProxyManager(logger=_logger)
_proxise = _proxy.get_proxy_for_appointment_request()
print(_proxise)
response = requests.get(
"https://ip.oxylabs.io/location",
proxies=_proxise)
print(response.content)
queue_message/CookiesPublisher.py
+1 -1
View File
@@ -31,7 +31,7 @@ class CookiesPublisher:
self.queue_method = self.channel.queue_declare(queue=self.to_queue, durable=True) self.queue_method = self.channel.queue_declare(queue=self.to_queue, durable=True)
def publish_body(self, body: str): def publish_body(self, body: str):
print("will push to queue {}".format(body)) print("will push to queue {}".format(self.to_queue))
self.channel.basic_publish(exchange='', routing_key=self.to_queue, body=body, properties=pika.BasicProperties( self.channel.basic_publish(exchange='', routing_key=self.to_queue, body=body, properties=pika.BasicProperties(
delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE
)) ))
queue_message/appointmentrequestsender.py
+4 -5
View File
@@ -101,8 +101,8 @@ class AppointmentRequestSender(threading.Thread):
_received_object = body.decode("UTF-8") _received_object = body.decode("UTF-8")
self.logger.info(f" [x] Received {_received_object}") self.logger.info(f" [x] Received {_received_object}")
# 同一个COOKIES保持用同一个IP # 同一个COOKIES保持用同一个IP
# _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
# print("proxy to use is {}".format(_proxy_to_use)) print("proxy to use is {}".format(_proxy_to_use))
if "glrd" in _received_object: if "glrd" in _received_object:
_received_dict = json.loads(_received_object) _received_dict = json.loads(_received_object)
js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'], js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'],
@@ -124,8 +124,8 @@ class AppointmentRequestSender(threading.Thread):
# self.contact_list = filter_contacts(self.contact_list) # self.contact_list = filter_contacts(self.contact_list)
can_continue = None can_continue = None
for con in self.contact_list: for con in self.contact_list:
_proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() # _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
print(_proxy_to_use) # print(_proxy_to_use)
sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher,
received_dict=_received_dict, received_dict=_received_dict,
proxy_to_use=_proxy_to_use, logger=self.logger) proxy_to_use=_proxy_to_use, logger=self.logger)
@@ -239,7 +239,6 @@ class AppointmentRequestSender(threading.Thread):
self.channel.start_consuming() self.channel.start_consuming()
def retrieve_invalidate_urls(self): def retrieve_invalidate_urls(self):
self.list_to_retrieve_mails = self.initial_contact_list
# 如果没有已读邮件，而且需要读邮件的联系人表不为空，就读取未读邮件 # 如果没有已读邮件，而且需要读邮件的联系人表不为空，就读取未读邮件
if not self.already_read_emails and len(self.list_to_retrieve_mails) > 0: if not self.already_read_emails and len(self.list_to_retrieve_mails) > 0:
self.logger.info("will retrieve validate urls") self.logger.info("will retrieve validate urls")
request_sender.py
+3 -3
View File
@@ -71,10 +71,10 @@ def send_appointment_request(message_queue_name, _contact_list):
def start_send_requests(): def start_send_requests():
print("start send requests") print("start send requests")
contacts_file_path = '~/Desktop/06_05_to_test.xlsx' contacts_file_path = '~/Desktop/11_05_to_test.xlsx'
_contact_list = read_contacts(contacts_file_path)[:1] _contact_list = read_contacts(contacts_file_path)[:-1]
_contact_list_to_book = filter_contacts(_contact_list) _contact_list_to_book = filter_contacts(_contact_list)
_segment_number = 1 _segment_number = 5
logger.info("{} contacts to book".format(len(_contact_list_to_book))) logger.info("{} contacts to book".format(len(_contact_list_to_book)))
last_thread = None last_thread = None
for i in range(0, _segment_number): for i in range(0, _segment_number):
request_sender_test.py
+3 -3
View File
@@ -103,6 +103,6 @@ if __name__ == '__main__':
# '~/Desktop/contact_list_2024-05-21.xlsx', # '~/Desktop/contact_list_2024-05-21.xlsx',
# '~/Desktop/15_05_to_test.xlsx'] # '~/Desktop/15_05_to_test.xlsx']
# file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx'] # file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx']
file_list = ['~/Desktop/contact_list_2025-06-09_2.xlsx'] file_list = ['~/Desktop/contact_list_all.xlsx']
send_request_for_file_list(file_list=file_list, thread_number=10, send_request_for_file_list(file_list=file_list, thread_number=1,
data_queue_name=MORNING_DATA_CACHE, stop_at_hour=19, stop_at_mins=50) data_queue_name=MORNING_DATA_CACHE, stop_at_hour=18, stop_at_mins=30)
scheduler_test.py
+4 -2
View File
@@ -6,8 +6,10 @@ from request_sender_test import send_request_for_file_list
def start_book_appointment(): def start_book_appointment():
file_list = ['~/Desktop/contact_list_2025-06-09_2.xlsx'] file_list = ['~/Desktop/contact_list_2024-09-06.xlsx',
send_request_for_file_list(file_list=file_list, thread_number=30, '~/Desktop/contact_list_all.xlsx',
'~/Desktop/contact_list_2024-05-27.xlsx']
send_request_for_file_list(file_list=file_list, thread_number=80,
data_queue_name=MORNING_DATA_CACHE, stop_at_hour=11, stop_at_mins=20) data_queue_name=MORNING_DATA_CACHE, stop_at_hour=11, stop_at_mins=20)
utils/address_ip.py
+12 -12
View File
@@ -6,18 +6,18 @@ from proxy_manager.proxy_manager import ProxyManager
def get_address_ip(proxy_to_use): def get_address_ip(proxy_to_use):
# return "" return ""
try: # try:
response = requests.get(url="https://api.ipify.org", proxies=proxy_to_use, verify=False, # response = requests.get(url="https://api.ipify.org", proxies=proxy_to_use, verify=False,
timeout=15) # timeout=15)
print(response.status_code) # print(response.status_code)
if response.status_code == 200: # if response.status_code == 200:
return response.text # return response.text
else: # else:
return "" # return ""
except Exception as Error: # except Exception as Error:
print(Error) # print(Error)
return "error" # return "error"
if __name__ == '__main__': if __name__ == '__main__':
utils/tls_ja3.py
-29
View File
@@ -1,29 +0,0 @@
# import requests
from curl_cffi import requests
# chrome99
# chrome100
# chrome101
# chrome104
# chrome107
# chrome110
# chrome116 [1]
# chrome119 [1]
# chrome120 [1]
# chrome123 [3]
# chrome124 [3]
# chrome99_android
# edge99
# edge101
# safari15_3 [2]
# safari15_5 [2]
# safari17_0 [1]
# safari17_2_ios [1]
# safari18_0 [4]
# safari18_0_ios [4]
TLS_JA3_CHECK_SERVER = "https://tls.browserleaks.com/json"
if __name__ == '__main__':
_res = requests.get(TLS_JA3_CHECK_SERVER, impersonate="chrome99_android")
print(_res.text)
utils/user_agent_helper.py
-57
View File
@@ -1,57 +0,0 @@
import re
from http.cookies import SimpleCookie
from pprint import pprint
def get_chrome_version_from_ua(ua: str) -> str:
_regex = "Chrome\/([0-9]+)"
_match = re.search(_regex, ua)
if _match:
return _match.group(1)
def generate_headers_from_request_message(_received_dict, cookie_str):
_ua = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36'
cookie = SimpleCookie()
_chrome_version = '125'
if _received_dict is not None:
_ua = _received_dict['ua']
_chrome_version = get_chrome_version_from_ua(_ua)
_model = _received_dict['model']
# _ua = "Mozilla/5.0 (Linux; Android 9; {}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.58 Mobile Safari/537.36".format(
# _model)
print("model:" + _model)
cookie.load(cookie_str)
_referer = "android-app://com.google.android.gm/"
headers = {
'User-Agent': _ua,
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'max-age=0',
'Referer': _referer,
'Cookie': cookie_str,
'Sec-Fetch-Mode': 'navigate',
'Host': 'rendezvousparis.hermes.com',
'Sec-Fetch-Site': 'same-origin',
'sec-ch-ua': '"Google Chrome";v="{}", "Chromium";v="{}", "Not.A/Brand";v="24"'.format(_chrome_version,
_chrome_version),
'sec-ch-ua-platform': '"Android"',
'sec-ch-ua-model': '"{}"'.format(_model),
'Sec-Fetch-Dest': 'document',
'Accept-Language': 'fr-FR,fr;q=0.6'}
return headers
if __name__ == '__main__':
# _ua_to_test = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Mobile Safari/537.36"
# _ua_to_test = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Mobile Safari/537.36"
# print(get_chrome_version_from_ua(_ua_to_test)) # Output: 129.0.0.0
_received_dict = {"glvd": "Google Inc. (ARM)", "glrd": "ANGLE (ARM, Mali-G57 MC2, OpenGL ES 3.2)", "hc": 8,
"br_oh": 745, "br_ow": 393, "br_h": 745, "br_w": 393, "rs_h": 876, "rs_w": 393, "rs_cd": 24,
"ars_h": 876, "ars_w": 393, "plg": 0, "eva": 33, "vnd": "Google Inc.", "plu": [],
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Mobile Safari/537.36",
"dvm": 4, "acw": "probably", "pr": 2.75, "ts_mtp": 5, "tz": -60, "emd": "k:ai,vi,ao",
"cookiesStr": "datadome=cVhteVJiGoU3N1T61hX4dF5vNG4~p5JpkbsKLH674kzQiuT1G5sDohLD33nFx2QFqlPiJfY~yfsRNIpqM8xZ1JfXTgT7p8v5QfkmmOl~BIhLOVCBHe_6Al6CM2QsSq7g;__cf_bm=J9g80A6wlNO03BLqyhiuf5ZRkx_8Ig7QIOLhHoEh1i8-1734010016-1.0.1.1-fMIRmq.K8K093kcSzJdFlcIEoCH9XwWhDlsXFoDmvNnerYvpyG0eC9vIdqj5xACF28YYAYGXwuag5f33JoDiBg;",
"model": "22041219PG", "serial": "fmiz5pa6rsx4u4ts"}
pprint(generate_headers_from_request_message(_received_dict, _received_dict["cookiesStr"]))
workers/captcha_result_getter.py
+5 -4
View File
@@ -112,16 +112,17 @@ class CaptchaResultGetter:
return _raw_data return _raw_data
def get_valid_ch_cookie(self, proxy_to_use, js_data: JsDataPojo, old_valid_cookie: str = None, def get_valid_ch_cookie(self, proxy_to_use, js_data: JsDataPojo, old_valid_cookie: str = None,
referrer: str = "https://rendezvousparis.hermes.com/", referrer: str = None) -> Union[str, None]:
origin="https://rendezvousparis.hermes.com/") -> Union[str, None]:
# extract datadome cookie # extract datadome cookie
# get new cookie # get new cookie
_cookies_to_use = get_datadome_cookies(old_valid_cookie) _cookies_to_use = get_datadome_cookies(old_valid_cookie)
_app_cookies = get_app_cookies(old_valid_cookie) _app_cookies = get_app_cookies(old_valid_cookie)
_lang_cookies = get_lang_cookies(old_valid_cookie) _lang_cookies = get_lang_cookies(old_valid_cookie)
_origin = origin _origin = "https://rendezvousparis.hermes.com/"
# _origin = "https://www.hermes.com" # _origin = "https://www.hermes.com"
_referer = referrer _referer = "https://rendezvousparis.hermes.com/"
if referrer is not None:
_referer = referrer
# _referer = "https://www.hermes.com" # _referer = "https://www.hermes.com"
headers = {'content-Type': 'application/x-www-form-urlencoded', headers = {'content-Type': 'application/x-www-form-urlencoded',
workers/cookie_generator.py
+35 -83
View File
@@ -1,5 +1,3 @@
import json
import logging
import random import random
import threading import threading
import time import time
@@ -7,11 +5,8 @@ from http.cookies import SimpleCookie
import pika import pika
from models.jsdata_le_pojo import JsDataLeTypePojo
from models.jsdata_pojo import JsDataPojo
from proxy_manager.proxy_manager import ProxyManager
from queue_message.CookiesPublisher import CookiesPublisher, QUEUE_HOST, credentials, REQUEST_DATA_QUEUE, \ from queue_message.CookiesPublisher import CookiesPublisher, QUEUE_HOST, credentials, REQUEST_DATA_QUEUE, \
REQUEST_DATA_QUEUE_TEST, MORNING_DATA_CACHE_BAK REQUEST_DATA_QUEUE_TEST
from workers.captcha_result_getter import CaptchaResultGetter from workers.captcha_result_getter import CaptchaResultGetter
from workers.proxies_constants import PROXY_LIST_FR from workers.proxies_constants import PROXY_LIST_FR
@@ -19,16 +14,13 @@ DVM_LIST = [2, 3, 4, 6]
class CookiesGenerator(threading.Thread): class CookiesGenerator(threading.Thread):
def __init__(self, proxy_manager: ProxyManager, cookiesPublisher: CookiesPublisher, def __init__(self, proxy_to_use_list: list, cookiesPublisher: CookiesPublisher):
logger=logging.getLogger(__name__)):
super().__init__() super().__init__()
self.connection = None self.connection = None
self.cookiesPublisher = cookiesPublisher self.cookiesPublisher = cookiesPublisher
self.channel = None self.channel = None
self.valid_csrf = None self.valid_csrf = None
self.proxy_manager = proxy_manager self.proxy_to_use_list = proxy_to_use_list
self.queue_name = MORNING_DATA_CACHE_BAK
self.logger = logger
def set_up_connection(self): def set_up_connection(self):
self.connection = pika.BlockingConnection( self.connection = pika.BlockingConnection(
@@ -37,90 +29,50 @@ class CookiesGenerator(threading.Thread):
def listen_to_queue(self, callback): def listen_to_queue(self, callback):
self.channel.basic_qos(prefetch_count=1) self.channel.basic_qos(prefetch_count=1)
self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback) self.channel.basic_consume(queue=REQUEST_DATA_QUEUE, auto_ack=False, on_message_callback=callback)
self.channel.start_consuming() self.channel.start_consuming()
def message_count(self): def message_count(self):
return self.channel.queue_declare(queue=self.queue_name, durable=True).method.message_count return self.channel.queue_declare(queue=REQUEST_DATA_QUEUE, durable=True).method.message_count
def on_message(self, ch, method, properties, body): def on_message(self, ch, method, properties, body):
print(f" [x] Received {body}") print(f" [x] Received {body}")
_received_object = body.decode("UTF-8") _cached_message_count = self.cookiesPublisher.message_count()
_received_dict = json.loads(_received_object) _valid_message_count = self.message_count()
_received_cookies = _received_dict["cookiesStr"] print("message count in cache is {}".format(_cached_message_count))
print("message count in current queue is {}".format(_valid_message_count))
# generate 10 cookies to new_queue # generate 10 cookies to new_queue
captchaResultGetter = CaptchaResultGetter() captchaResultGetter = CaptchaResultGetter()
_cookies_count = random.randint(8, 10) _cookies_count = random.randint(8, 10)
_received_cookies = body.decode("UTF-8") _received_cookies = body.decode("UTF-8")
dvm = random.choice(DVM_LIST) dvm = random.choice(DVM_LIST)
hc = random.choice(DVM_LIST) hc = random.choice(DVM_LIST)
_proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() if _cached_message_count < 10 and _valid_message_count > 40:
# new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, dvm=dvm, hc=hc) _proxy_to_use = random.choice(self.proxy_to_use_list)
_app_sig = get_app_and_app_sig(_received_cookies) for i in range(1, _cookies_count):
# if new_cookie is not None and _app_sig is not None: # new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, dvm=dvm, hc=hc)
# new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie _app_sig = get_app_and_app_sig(_received_cookies)
# new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace( # if new_cookie is not None and _app_sig is not None:
# "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "") # new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
# print("new_cookie is " + new_cookie) # new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
# _received_cookies = new_cookie # "Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "")
# self.cookiesPublisher.publish_body(new_cookie) # print("new_cookie is " + new_cookie)
js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'], # _received_cookies = new_cookie
ua=_received_dict['ua'], br_oh=_received_dict['br_oh'], br_ow=_received_dict['br_ow'], # self.cookiesPublisher.publish_body(new_cookie)
ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'], pr=_received_dict['pr'], new_cookie = captchaResultGetter.get_valid_cookie(proxy_to_use=_proxy_to_use,
plg=_received_dict['plg'], br_h=_received_dict['br_h'], br_w=_received_dict['br_w'], old_valid_cookie=_received_cookies, dvm=dvm,
plu=_received_dict['plu'], vnd=_received_dict['vnd'], dvm=_received_dict['dvm'], hc=hc)
ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'], if new_cookie is not None and _app_sig is not None:
rs_h=_received_dict['rs_h'], new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'], emd=_received_dict['emd']) new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
"Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "")
new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, js_data=js_data, _received_cookies = new_cookie
old_valid_cookie=_received_cookies, self.cookiesPublisher.publish_body(new_cookie)
origin="https://www.hermes.com", ch.basic_ack(delivery_tag=method.delivery_tag)
referrer="https://www.hermes.com") else:
if new_cookie is not None: time.sleep(30)
new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
new_cookie = new_cookie.replace("Path=/;", "").replace(
"Secure; SameSite=None", "").replace(" ", "")
_received_cookies = new_cookie
# _received_dict["cookiesStr"] = new_cookie
# self.cookiesPublisher.publish_body(json.dumps(_received_dict))
if _received_cookies is not None:
self.logger.info("new cookie is " + _received_cookies)
time.sleep(random.randint(1, 3))
# m_s_c = f.scroll
m_s_c = random.randint(0, 3)
m_c_c = random.randint(3, 5) # click count
m_m_c = random.randint(3, 5) # move count
m_cm_r = m_c_c / m_m_c
m_ms_r = -1 # move scroll ratio
if m_s_c == 0:
m_ms_r = -1
else:
m_ms_r = m_m_c / m_s_c
js_le_data = JsDataLeTypePojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'],
hc=_received_dict['hc'],
ua=_received_dict['ua'], br_oh=_received_dict['br_oh'],
br_ow=_received_dict['br_ow'],
ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'],
pr=_received_dict['pr'],
plg=_received_dict['plg'], br_h=_received_dict['br_h'],
br_w=_received_dict['br_w'],
plu=_received_dict['plu'], vnd=_received_dict['vnd'],
dvm=_received_dict['dvm'],
ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'],
rs_h=_received_dict['rs_h'],
rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'],
m_s_c=m_s_c, m_m_c=m_m_c, m_c_c=m_c_c,
m_cm_r=m_cm_r, m_ms_r=m_ms_r, emd=_received_dict['emd'])
time.sleep(random.randint(1, 4))
_new_le_cookies = captchaResultGetter.get_le_valid_cookie(proxy_to_use=_proxy_to_use,
js_le_type_data=js_le_data,
old_valid_cookie=_received_cookies)
_received_dict["cookiesStr"] = new_cookie
self.cookiesPublisher.publish_body(json.dumps(_received_dict))
# self.cookiesPublisher.publish_body(new_cookie)
ch.basic_ack(delivery_tag=method.delivery_tag)
def run(self): def run(self):
print(threading.currentThread().name + " starts") print(threading.currentThread().name + " starts")
workers/de_cookie_generator.py
+4 -12
View File
@@ -1,17 +1,9 @@
import logging from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher
import sys
from proxy_manager.proxy_manager import ProxyManager
from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher, MORNING_DATA_CACHE
from utils.AppLogging import init_logger
from workers.cookie_generator import CookiesGenerator from workers.cookie_generator import CookiesGenerator
from workers.proxies_constants import PROXY_LIST_DE
if __name__ == '__main__': if __name__ == '__main__':
init_logger() cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_DE)
logger = logging.getLogger()
logger.addHandler(logging.StreamHandler(stream=sys.stdout))
cookiesPublisher = CookiesPublisher(queue_name=MORNING_DATA_CACHE)
cookiesPublisher.set_up_connection() cookiesPublisher.set_up_connection()
cookieGenerator = CookiesGenerator(proxy_manager=ProxyManager(logger=logger), cookiesPublisher=cookiesPublisher, cookieGenerator = CookiesGenerator(cookiesPublisher=cookiesPublisher, proxy_to_use_list=PROXY_LIST_DE)
logger=logger)
cookieGenerator.run() cookieGenerator.run()
workers/link_validator_with_provided_list.py
+76 -95
View File
@@ -7,43 +7,32 @@ from datetime import datetime
from http.cookies import SimpleCookie from http.cookies import SimpleCookie
import pika import pika
from curl_cffi import requests import requests
from db.mongo_manager import MONGO_STORE_MANAGER from db.mongo_manager import MONGO_STORE_MANAGER
from models.LinkPojo import LinkPojo from models.LinkPojo import LinkPojo
from models.result_pojo import RequestResult from models.result_pojo import RequestResult
from proxy_manager.proxy_manager import ProxyManager from proxy_manager.proxy_manager import ProxyManager
from queue_message.CookiesPublisher import CookiesPublisher, MORNING_DATA_CACHE, MORNING_DATA_CACHE_BAK from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST, TEST_QUEUE, SHARED_OBJECT, \
MORNING_DATA_CACHE_BAK, MORNING_DATA_CACHE, MORNING_DATA_CACHE_2
from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
from utils.AppLogging import init_logger from utils.AppLogging import init_logger
from utils.user_agent_helper import generate_headers_from_request_message from workers.proxies_constants import PROXY_LIST_FR
DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ces coordonnées" DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ces coordonnées"
INVALID = "nos conseillers vous accueillent sur rendez-vous dans nos magasins" INVALID = "Depuis plus de 130 ans,"
init_logger() init_logger()
logger = logging.getLogger() logger = logging.getLogger()
def filter_link_pojo_list_with_serial(_received_dict, link_to_validate_list):
_serial = _received_dict["serial"]
_model = _received_dict["model"]
_bit_browser_list = list(filter(lambda link_pojo: link_pojo.serial == "requests", link_to_validate_list))
if len(_bit_browser_list) > 0:
return _bit_browser_list
else:
_to_return = filter(lambda link_pojo: link_pojo.serial == _serial, link_to_validate_list)
return list(_to_return)
class LinkValidatorWithProvidedList(threading.Thread): class LinkValidatorWithProvidedList(threading.Thread):
def __init__(self, cookiesPublisher: CookiesPublisher, link_list: list, _contact_serial_list, def __init__(self, cookiesPublisher: CookiesPublisher, link_list: list,
queue_to_listen=REQUEST_DATA_QUEUE, queue_to_listen=REQUEST_DATA_QUEUE,
ip_country="FR", ip_country="FR",
limit=40): limit=40):
super().__init__() super().__init__()
self.contact_serial_list = _contact_serial_list
self.link_to_validate_list = link_list self.link_to_validate_list = link_list
self.cookie = SimpleCookie() self.cookie = SimpleCookie()
self.cookiesPublisher = cookiesPublisher self.cookiesPublisher = cookiesPublisher
@@ -67,22 +56,34 @@ class LinkValidatorWithProvidedList(threading.Thread):
_ua = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36' _ua = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36'
if _received_dict is not None: if _received_dict is not None:
_ua = _received_dict['ua'] _ua = _received_dict['ua']
_model = _received_dict['model']
logger.info("model: %s", _model)
self.cookie.load(self.cookie_str) self.cookie.load(self.cookie_str)
headers = generate_headers_from_request_message(_received_dict, self.cookie_str) headers = {
'User-Agent': _ua,
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'max-age=0',
'Referer': linkPojo.url,
'Cookie': self.cookie_str,
'Sec-Fetch-Mode': 'navigate',
'Host': 'rendezvousparis.hermes.com',
'Sec-Fetch-Site': 'same-origin',
'sec-ch-ua': '"Brave";v="125", "Chromium";v="125", "Not.A/Brand";v="24"',
'sec-ch-ua-platform': '"Android"',
'sec-ch-ua-model': '""',
'Sec-Fetch-Dest': 'document',
'Accept-Language': 'fr-FR,fr;q=0.6'}
print("received cookie is " + str(self.cookie_str)) print("received cookie is " + str(self.cookie_str))
print("send request for link: " + linkPojo.url) print("send request for link: " + linkPojo.url)
try: try:
print("will send request with ua {}".format(_ua)) print("will send request with ua {}".format(_ua))
print("will send request with cookie {}".format(self.cookie_str)) print("will send request with cookie {}".format(self.cookie_str))
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use, response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
timeout=60, impersonate="chrome99_android") timeout=60)
print(response.status_code) print(response.status_code)
if response.status_code == 200: if response.status_code == 200:
_content = response.text _content = response.text
print(response.text) print(response.text)
if "Votre demande de rendez-vous est enregistrée" in _content: if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content:
print(response.url) print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo,
segement_position=threading.currentThread().name, segement_position=threading.currentThread().name,
@@ -105,6 +106,7 @@ class LinkValidatorWithProvidedList(threading.Thread):
self.cookie_str = new_coolies_str self.cookie_str = new_coolies_str
return RequestResult.SUCCESS return RequestResult.SUCCESS
elif INVALID in _content: elif INVALID in _content:
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True, ua=_ua) MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True, ua=_ua)
# set new cookies # set new cookies
_cookies_to_set = response.headers['set-cookie'] _cookies_to_set = response.headers['set-cookie']
@@ -149,51 +151,45 @@ class LinkValidatorWithProvidedList(threading.Thread):
_message_in_queue_count = self.cookiesPublisher.message_count() _message_in_queue_count = self.cookiesPublisher.message_count()
print("message count in queue is {}".format(_message_in_queue_count)) print("message count in queue is {}".format(_message_in_queue_count))
_received_object = body.decode("UTF-8") _received_object = body.decode("UTF-8")
try: _received_dict = None
if "glrd" in _received_object:
_received_dict = json.loads(_received_object) _received_dict = json.loads(_received_object)
_received_cookies = _received_dict["cookiesStr"] _received_cookies = _received_dict["cookiesStr"]
self.cookie_str = _received_cookies else:
_links_to_validate = filter_link_pojo_list_with_serial(_received_dict, self.link_to_validate_list) _received_cookies = _received_object
random.shuffle(_links_to_validate) self.cookie_str = _received_cookies
if len(_links_to_validate) > 0 and _message_in_queue_count >= self.limit: random.shuffle(self.link_to_validate_list)
print("{}:links number is {}".format(threading.currentThread().name, len(_links_to_validate))) if len(self.link_to_validate_list) > 0 and _message_in_queue_count >= self.limit:
can_continue = None print("{}:links number is {}".format(threading.currentThread().name, len(self.link_to_validate_list)))
for link_to_validate in _links_to_validate: can_continue = None
print(link_to_validate) for link_to_validate in self.link_to_validate_list:
self.proxy_to_use = random.choice( print(link_to_validate)
self.proxy_manager.get_link_validate_proxy(_links_to_validate)) self.proxy_to_use = random.choice(
print("proxy to use is {}".format(self.proxy_to_use)) self.proxy_manager.get_link_validate_proxy(self.link_to_validate_list))
can_continue = self.send_request(link_to_validate, _received_dict) print("proxy to use is {}".format(self.proxy_to_use))
# remove the tested link from link list can_continue = self.send_request(link_to_validate, _received_dict)
self.link_to_validate_list.remove(link_to_validate) # remove the tested link from link list
if can_continue == RequestResult.BLOCKED: self.link_to_validate_list.remove(link_to_validate)
print("cannot continue, blocked, then skip") if can_continue == RequestResult.BLOCKED:
break print("cannot continue, blocked, then skip")
else: break
if can_continue == RequestResult.BAD_GATEWAY:
time.sleep(30)
break
time.sleep(random.randint(2, 5))
print("can continue, continue")
if can_continue == RequestResult.BAD_GATEWAY or can_continue == RequestResult.PROXY_ERROR or can_continue == RequestResult.SUCCESS:
print("will requeue the message")
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
else: else:
print("will ack") if can_continue == RequestResult.BAD_GATEWAY:
print("will wait for 40s") time.sleep(30)
time.sleep(20) break
ch.basic_ack(delivery_tag=method.delivery_tag) time.sleep(random.randint(2, 5))
print("can continue, continue")
if can_continue == RequestResult.BAD_GATEWAY or can_continue == RequestResult.PROXY_ERROR or can_continue == RequestResult.SUCCESS:
print("will requeue the message")
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
else: else:
print("empty list, will republish message") print("will ack")
time.sleep(0)
print("body in json:{}".format(json.dumps(_received_dict)))
self.cookiesPublisher.publish_body(json.dumps(_received_dict))
ch.basic_ack(delivery_tag=method.delivery_tag) ch.basic_ack(delivery_tag=method.delivery_tag)
self.channel.stop_consuming() else:
except Exception as error: print("empty list, no need to ack")
print(error) time.sleep(5)
print("not json format will ack") ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
ch.basic_ack(delivery_tag=method.delivery_tag) self.channel.stop_consuming()
def filter_with_ip_country(self): def filter_with_ip_country(self):
_link_list_to_click = [] _link_list_to_click = []
@@ -203,10 +199,9 @@ class LinkValidatorWithProvidedList(threading.Thread):
self.link_to_validate_list = _link_list_to_click self.link_to_validate_list = _link_list_to_click
def validate_links(cookiesPublisher, queue_name: str, link_list: list, _contact_serial_list): def validate_links(cookiesPublisher, queue_name: str, link_list: list):
receiver = LinkValidatorWithProvidedList(cookiesPublisher=cookiesPublisher, link_list=link_list, receiver = LinkValidatorWithProvidedList(cookiesPublisher=cookiesPublisher, link_list=link_list,
queue_to_listen=queue_name, _contact_serial_list=_contact_serial_list, queue_to_listen=queue_name, ip_country="FR", limit=0)
ip_country="FR", limit=0)
print("{} set_up_connection".format(threading.currentThread().name)) print("{} set_up_connection".format(threading.currentThread().name))
receiver.set_up_connection() receiver.set_up_connection()
receiver.listen_to_queue(receiver.on_message) receiver.listen_to_queue(receiver.on_message)
@@ -214,37 +209,26 @@ def validate_links(cookiesPublisher, queue_name: str, link_list: list, _contact_
# default_segment_number 并发数，决定速度 # default_segment_number 并发数，决定速度
# divided = 4，越小，一次处理得越多 # divided = 4，越小，一次处理得越多
def validate_all_links(_contact_serial_list): def validate_all_links():
print("will get all links") print("will get all links")
all_link_list = MONGO_STORE_MANAGER.get_links_to_validate() all_link_list = MONGO_STORE_MANAGER.get_links_to_validate()
link_to_validated = []
for _link in all_link_list:
# print("serial is "+_link.serial)
print("email is " + _link.email)
if _link.serial == "requests":
link_to_validated.append(_link)
# get the first 50 links # get the first 50 links
if len(link_to_validated) == 0: if len(all_link_list) == 0:
return return
divided = 1 divided = 3
# default_segment_number = 20 default_segment_number = 20
_first_25_percent_links = link_to_validated[0:(int(len(all_link_list) / divided))] _first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))]
_first_25_percent_links = all_link_list
_queue_name = MORNING_DATA_CACHE _queue_name = MORNING_DATA_CACHE
# _queue_name = MORNING_DATA_CACHE_BAK # _queue_name = MORNING_DATA_CACHE_BAK
# if len(all_link_list) > divided * default_segment_number: if len(all_link_list) > divided * default_segment_number:
# _segment_number = default_segment_number _segment_number = default_segment_number
# else:
# _first_25_percent_links = all_link_list
# if len(_first_25_percent_links) > divided:
# _segment_number = int(len(_first_25_percent_links) / divided)
# else:
# _segment_number = 1
_thread_list = []
if len(_first_25_percent_links) >= 10:
_segment_number = 10
else: else:
_segment_number = len(_first_25_percent_links) _first_25_percent_links = all_link_list
if len(_first_25_percent_links) > divided:
_segment_number = int(len(_first_25_percent_links) / divided)
else:
_segment_number = 1
_thread_list = []
for i in range(0, _segment_number): for i in range(0, _segment_number):
logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_25_percent_links))) logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_25_percent_links)))
logger.info("segment is {}".format(i)) logger.info("segment is {}".format(i))
@@ -252,8 +236,7 @@ def validate_all_links(_contact_serial_list):
_cookiesPublisher.set_up_connection() _cookiesPublisher.set_up_connection()
_step = int(len(_first_25_percent_links) / _segment_number) _step = int(len(_first_25_percent_links) / _segment_number)
_sublist = _first_25_percent_links[i * _step:_step * (i + 1)] _sublist = _first_25_percent_links[i * _step:_step * (i + 1)]
_thread1 = threading.Thread(target=validate_links, _thread1 = threading.Thread(target=validate_links, args=(_cookiesPublisher, _queue_name, _sublist))
args=(_cookiesPublisher, _queue_name, _sublist, _contact_serial_list))
_thread_list.append(_thread1) _thread_list.append(_thread1)
_thread1.start() _thread1.start()
for _thread in _thread_list: for _thread in _thread_list:
@@ -262,12 +245,10 @@ def validate_all_links(_contact_serial_list):
if __name__ == '__main__': if __name__ == '__main__':
# generate test data # generate test data
# contact_serial_map_list = MONGO_STORE_MANAGER.get_all_contact_serial_list()
while True: while True:
print("call validate_all_links()") print("call validate_all_links()")
validate_all_links([]) validate_all_links()
# delay = random.randint(10, 30) delay = random.randint(10, 30)
delay = random.randint(1, 10)
current_time = datetime.now() current_time = datetime.now()
current_hour = current_time.hour current_hour = current_time.hour
print("Current hour ", current_time.hour) print("Current hour ", current_time.hour)
workers/sender.py
+14 -17
View File
@@ -1,10 +1,12 @@
import datetime import datetime
import json import json
import logging
import random
import re import re
import time import time
from http.cookies import SimpleCookie from http.cookies import SimpleCookie
from models.contact_pojo import ContactPojo from models.contact_pojo import ContactPojo
from curl_cffi import requests import requests
from db.mongo_manager import MONGO_STORE_MANAGER from db.mongo_manager import MONGO_STORE_MANAGER
from models.ReserveResultPojo import ReserveResultPojo, PublishType from models.ReserveResultPojo import ReserveResultPojo, PublishType
@@ -38,8 +40,7 @@ class Sender:
self.proxy_to_use = proxy_to_use self.proxy_to_use = proxy_to_use
self.cookie.load(self.cookie_str) self.cookie.load(self.cookie_str)
def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str, store_type: str, def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str, store_type: str):
model: str = ""):
# create the message # create the message
if url == "https://rendezvousparis.hermes.com/client/welcome": if url == "https://rendezvousparis.hermes.com/client/welcome":
return return
@@ -60,12 +61,11 @@ class Sender:
result.proxy = "data_impulse" result.proxy = "data_impulse"
result.id = id result.id = id
result.store_type = store_type result.store_type = store_type
result.model = model
result.created_at = time.strftime("%H:%M:%S", time.localtime()) result.created_at = time.strftime("%H:%M:%S", time.localtime())
collection_name = str(datetime.date.today()) collection_name = str(datetime.date.today())
MONGO_STORE_MANAGER.insert_reserve_result(collection_name=collection_name, reserve=result) MONGO_STORE_MANAGER.insert_reserve_result(collection_name=collection_name, reserve=result)
def apply_redirect(self, response, old_headers, contact, js_data, selected_store, model=""): def apply_redirect(self, response, old_headers, contact, js_data, selected_store):
# /client/register/5XD2E2 # /client/register/5XD2E2
_res_headers = response.headers _res_headers = response.headers
_location = _res_headers['location'] _location = _res_headers['location']
@@ -89,7 +89,7 @@ class Sender:
contact.current_ip = get_address_ip(proxy_to_use=self.proxy_to_use) contact.current_ip = get_address_ip(proxy_to_use=self.proxy_to_use)
_appointment_url = _redirect_url _appointment_url = _redirect_url
self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=_appointment_url, self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=_appointment_url,
store_type=selected_store, model=model) store_type=selected_store)
self.cookie.load(_cookies_to_set) self.cookie.load(_cookies_to_set)
new_cookies = {k: v.value for k, v in self.cookie.items()} new_cookies = {k: v.value for k, v in self.cookie.items()}
new_coolies_str = "" new_coolies_str = ""
@@ -118,8 +118,6 @@ class Sender:
self._csrf = csrf self._csrf = csrf
_csrf = csrf _csrf = csrf
self.logger.info("model is:{}".format(model)) self.logger.info("model is:{}".format(model))
js_data.ua = "Mozilla/5.0 (Linux; Android 9; {}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.58 Mobile Safari/537.36".format(
model)
_chrome_code = get_chrome_version_from_ua(js_data.ua) _chrome_code = get_chrome_version_from_ua(js_data.ua)
_chrome_version = None _chrome_version = None
if _chrome_code == '128': if _chrome_code == '128':
@@ -130,12 +128,11 @@ class Sender:
_chrome_version = "116.0.5845.114" _chrome_version = "116.0.5845.114"
elif _chrome_code == '114': elif _chrome_code == '114':
_chrome_version = "114.0.5735.53" _chrome_version = "114.0.5735.53"
elif _chrome_code == '99':
_chrome_version = "99.0.4844.58" _chrome_ua_version = '"Chromium";v="{}", "Not;A=Brand";v="24.0.0.0", "Google Chrome";v="{}"'.format(
# 99: " Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99" _chrome_version, _chrome_version)
# " Not A;Brand";v="99.0.0.0", "Chromium";v="99.0.4844.58", "Google Chrome";v="99.0.4844.58" _chrome_ua_header = '"Chromium";v="{}", "Not;A=Brand";v="24", "Google Chrome";v="{}"'.format(_chrome_code,
_chrome_ua_version = 'Not A;Brand";v="99.0.0.0", "Chromium";v="99.0.4844.58", "Google Chrome";v="99.0.4844.58"' _chrome_code)
_chrome_ua_header = 'Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"'
self.logger.info("chrome_ua_header is:{}".format(_chrome_ua_header)) self.logger.info("chrome_ua_header is:{}".format(_chrome_ua_header))
self.logger.info("_chrome_ua_version is:{}".format(_chrome_ua_version)) self.logger.info("_chrome_ua_version is:{}".format(_chrome_ua_version))
# 默认为空，也就是随机 # 默认为空，也就是随机
@@ -162,7 +159,7 @@ class Sender:
headers['sec-ch-ua-full-version-list'] = _chrome_ua_version headers['sec-ch-ua-full-version-list'] = _chrome_ua_version
data = {'check': '', '_csrf': _csrf, 'prefer': _selected_store, data = {'check': '', '_csrf': _csrf, 'prefer': _selected_store,
'surname': contact.last_name.capitalize(), 'name': contact.first_name.capitalize(), 'surname': contact.last_name.capitalize(), 'name': contact.first_name.capitalize(),
'phone_country': "FR", 'phone_number': "0" + contact.phone, 'email': contact.mail, 'phone_country': "FR", 'phone_number': "+33 " + contact.phone, 'email': contact.mail,
'passport_id': contact.passport, 'processing': 'on', 'cgu': 'on'} 'passport_id': contact.passport, 'processing': 'on', 'cgu': 'on'}
self.logger.info(data) self.logger.info(data)
self.logger.info("try to request for mail:{}".format(contact.mail)) self.logger.info("try to request for mail:{}".format(contact.mail))
@@ -171,13 +168,13 @@ class Sender:
proxy_to_use = self.proxy_to_use proxy_to_use = self.proxy_to_use
# print(proxy_to_use) # print(proxy_to_use)
response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data, response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data,
timeout=15, allow_redirects=False, impersonate="chrome99_android") timeout=15, allow_redirects=False)
self.logger.info(response.status_code) self.logger.info(response.status_code)
if response.status_code == 302: if response.status_code == 302:
# add to mongodb # add to mongodb
self.logger.info(response.text) self.logger.info(response.text)
self.apply_redirect(response=response, old_headers=headers, contact=contact, js_data=js_data, self.apply_redirect(response=response, old_headers=headers, contact=contact, js_data=js_data,
selected_store=_selected_store, model=model) selected_store=_selected_store)
# self.logger.info("{}:{}".format(contact.mail, response.url)) # self.logger.info("{}:{}".format(contact.mail, response.url))
# contact.ua = js_data.ua # contact.ua = js_data.ua
# contact.current_ip = get_address_ip(proxy_to_use=proxy_to_use) # contact.current_ip = get_address_ip(proxy_to_use=proxy_to_use)