panleicim/appointment_request
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: panleicim/appointment_request:feature/test
Branches Tags
panleicim/appointment_request:master panleicim/appointment_request:feature/read_mail_issue panleicim/appointment_request:feature/password_in_env panleicim/appointment_request:api/app_api panleicim/appointment_request:feature/get_ip_geo_info panleicim/appointment_request:feature/links_filter_with_model panleicim/appointment_request:feature/v_5_1_15 panleicim/appointment_request:feature/with_lan panleicim/appointment_request:feature/token_from_cookies panleicim/appointment_request:feature/5_0_2 panleicim/appointment_request:feature/4_40_0 panleicim/appointment_request:feature/test panleicim/appointment_request:develop panleicim/appointment_request:feature/account_creation
...
pull from: panleicim/appointment_request:feature/4_40_0
Branches Tags
panleicim/appointment_request:feature/read_mail_issue panleicim/appointment_request:master panleicim/appointment_request:feature/password_in_env panleicim/appointment_request:api/app_api panleicim/appointment_request:feature/get_ip_geo_info panleicim/appointment_request:feature/links_filter_with_model panleicim/appointment_request:feature/v_5_1_15 panleicim/appointment_request:feature/with_lan panleicim/appointment_request:feature/token_from_cookies panleicim/appointment_request:feature/5_0_2 panleicim/appointment_request:feature/4_40_0 panleicim/appointment_request:feature/test panleicim/appointment_request:develop panleicim/appointment_request:feature/account_creation

14 Commits
feature/test ... feature/4_40_0

Author SHA1 Message Date
panleicim 2dded5e310 use gmail referer 2025-03-22 09:26:05 +01:00
panleicim 8dfee7b79f use anyip proxy 2024-12-26 15:26:50 +01:00
panleicim 293947fda8 support bitbrowser 2024-12-26 11:25:31 +01:00
panleicim 32d2a0b52b support 4.40.0 2024-12-21 11:19:35 +01:00
panleicim bda446f2d9 valid the links with paris ip 2024-12-14 11:51:05 +01:00
panleicim 4b17fa27c8 update .gitignore 2024-12-13 09:34:40 +01:00
panleicim adbabca2dc Merge branch 'feature/tls_ja3' of bitbucket.org:panleicim/appointment_request into feature/tls_ja3 2024-12-03 18:49:02 +01:00
panleicim 629a18aaf9 new confirmed msg 2024-12-03 18:48:51 +01:00
panleicim 8da4b5d530 Merge branch 'feature/test' into feature/tls_ja3 2024-11-11 10:15:41 +01:00
panleicim d72f793900 remove unused check 2024-11-11 10:15:16 +01:00
panleicim 686b472a41 Cookie generator not work 2024-10-20 21:42:35 +02:00
panleicim 1e8d7f7c83 try to use curl_cffi 2024-10-05 11:21:23 +02:00
panleicim f9ce820b9f try to use curl_cffi 2024-10-05 11:21:20 +02:00
panleicim c2db1035c4 try to use curl_cffi 2024-10-05 00:51:51 +02:00
24 changed files with 1008 additions and 141 deletions
Expand all files Collapse all files
.gitignore
+1
View File
@@ -4,3 +4,4 @@ __pycache__/
.idea
db/__pycache__/*
*.iml
venv
db/mongo_manager.py
+12 -1
View File
@@ -20,6 +20,7 @@ EMAIL_LIST = "EMAIL_LIST"
DESTINATION_EMAIL_LIST = "DESTINATION_EMAIL_LIST"
LINKS_TO_VALIDATE = "LINKS_TO_VALIDATE"
INVALID_EMAIL_LIST = "INVALID_EMAIL_LIST"
CONTACT_LIST_SERIAL_MAP = "CONTACT_LIST_SERIAL_MAP"
class MongoDbManager:
@@ -75,6 +76,13 @@ class MongoDbManager:
result_list.append(ContactPojo.from_firestore_dict(document))
return result_list
def get_all_contact_serial_list(self) -> list:
result_list = []
cursor = self.db[CONTACT_LIST_SERIAL_MAP]
for document in cursor.find():
result_list.append(ContactPojo.from_firestore_dict(document))
return result_list
def save_links_to_validate(self, link: str, mail_address: str, _all_contact_list: list):
collection_to_use = self.db[LINKS_TO_VALIDATE]
updated_at = time.strftime("%H:%M:%S", time.localtime())
@@ -174,7 +182,10 @@ class MongoDbManager:
validated_by = "Double"
collection.find_one_and_update({'_id': _id}, {
"$set": {"url_validated": state, "validated_at": validated_at, "id": _id, "email": linkPojo.email,
"url": link, "validated_by_ua": ua,
"url": link,
"source_from": linkPojo.model,
"serial": linkPojo.serial,
"validated_by_ua": ua,
"validated_by": validated_by}},
upsert=True)
# remove the link from db
docs/4_40_0/jsdata_ch_4_40_0_bitbrowser.json
+130
View File
@@ -0,0 +1,130 @@
{
"ttst": 5.500000000465661,
"ifov": false,
"hc": 8,
"br_oh": 992,
"br_ow": 1920,
"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
"wbd": false,
"tagpu": 1.7174740373540653,
"wdif": false,
"wdifrm": false,
"npmtm": false,
"br_h": 992,
"br_w": 1920,
"isf": true,
"nddc": 1,
"rs_h": 768,
"rs_w": 1024,
"rs_cd": 24,
"phe": false,
"nm": false,
"jsf": false,
"lg": "fr-BE",
"pr": 1,
"ars_h": 743,
"ars_w": 1024,
"tz": -60,
"str_ss": true,
"str_ls": true,
"str_idb": true,
"str_odb": false,
"plgod": false,
"plg": 5,
"plgne": true,
"plgre": true,
"plgof": false,
"plggt": false,
"pltod": false,
"hcovdr": false,
"hcovdr2": false,
"plovdr": false,
"plovdr2": false,
"ftsovdr": false,
"ftsovdr2": false,
"lb": false,
"eva": 33,
"lo": false,
"ts_mtp": 0,
"ts_tec": false,
"ts_tsa": false,
"vnd": "Google Inc.",
"bid": "NA",
"mmt": "application/pdf,text/pdf",
"plu": "PDF Viewer,Chrome PDF Viewer,Chromium PDF Viewer,Microsoft Edge PDF Viewer,WebKit built-in PDF",
"hdn": false,
"awe": false,
"geb": false,
"dat": false,
"med": "defined",
"aco": "probably",
"acots": false,
"acmp": "probably",
"acmpts": true,
"acw": "probably",
"acwts": false,
"acma": "maybe",
"acmats": false,
"acaa": "probably",
"acaats": true,
"ac3": "",
"ac3ts": false,
"acf": "probably",
"acfts": false,
"acmp4": "maybe",
"acmp4ts": false,
"acmp3": "probably",
"acmp3ts": false,
"acwm": "maybe",
"acwmts": false,
"ocpt": false,
"vco": "",
"vcots": false,
"vch": "probably",
"vchts": true,
"vcw": "probably",
"vcwts": true,
"vc3": "maybe",
"vc3ts": false,
"vcmp": "",
"vcmpts": false,
"vcq": "",
"vcqts": false,
"vc1": "probably",
"vc1ts": true,
"dvm": 4,
"sqt": false,
"so": "landscape-primary",
"wdw": true,
"cokys": "bG9hZFRpbWVzY3NpYXBwL=",
"ecpc": false,
"lgs": true,
"lgsod": false,
"psn": true,
"edp": true,
"addt": true,
"wsdc": true,
"ccsr": true,
"nuad": true,
"bcda": true,
"idn": true,
"capi": false,
"svde": false,
"vpbq": true,
"ucdv": false,
"spwn": false,
"emt": false,
"bfr": false,
"dbov": false,
"cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LGk9ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ2MSkpO2lmKGkpeyFmdW5jdGlvbiB0KCl7dHJ5e3ZhciBuPWlbJ1x4NzNceDY4XHg2MVx4NjRceDZmXHg3",
"stcfp": "cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4ODYzNSkKICAgIGF0IE9iamVjdC5kZF9hZCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDc3OTYpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjYzNzkw",
"ckwa": true,
"prm": true,
"cvs": true,
"usb": "defined",
"emd": "k:ai,vi,ao",
"glvd": "Apple Inc.",
"glrd": "Intel HD Graphics 3000 OpenGL Engine",
"wwl": false,
"jset": 1734772335
}
docs/4_40_0/jsdata_ch_4_40_0_mobile.json
+127
View File
@@ -0,0 +1,127 @@
{
"ttst": 22.000000002793968,
"ifov": false,
"hc": 8,
"br_oh": 763,
"br_ow": 393,
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Mobile Safari/537.36",
"wbd": false,
"tagpu": 2.3672133588807087,
"wdif": false,
"wdifrm": false,
"npmtm": false,
"br_h": 763,
"br_w": 393,
"isf": true,
"nddc": 1,
"rs_h": 873,
"rs_w": 393,
"rs_cd": 24,
"phe": false,
"nm": false,
"jsf": false,
"lg": "fr-FR",
"pr": 2.75,
"ars_h": 873,
"ars_w": 393,
"tz": -60,
"str_ss": true,
"str_ls": true,
"str_idb": true,
"str_odb": false,
"plgod": false,
"plg": 0,
"plgne": "NA",
"plgre": "NA",
"plgof": "NA",
"plggt": "NA",
"pltod": false,
"hcovdr": false,
"hcovdr2": false,
"plovdr": false,
"plovdr2": false,
"ftsovdr": false,
"ftsovdr2": false,
"lb": false,
"eva": 33,
"lo": false,
"ts_mtp": 5,
"ts_tec": true,
"ts_tsa": true,
"vnd": "Google Inc.",
"bid": "NA",
"mmt": "empty",
"plu": "empty",
"hdn": false,
"awe": false,
"geb": false,
"dat": false,
"med": "defined",
"aco": "probably",
"acots": false,
"acmp": "probably",
"acmpts": true,
"acw": "probably",
"acwts": false,
"acma": "maybe",
"acmats": false,
"acaa": "probably",
"acaats": true,
"ac3": "",
"ac3ts": false,
"acf": "probably",
"acfts": false,
"acmp4": "maybe",
"acmp4ts": false,
"acmp3": "probably",
"acmp3ts": false,
"acwm": "maybe",
"acwmts": false,
"ocpt": false,
"vco": "",
"vcots": false,
"vch": "probably",
"vchts": true,
"vcw": "probably",
"vcwts": true,
"vc3": "maybe",
"vc3ts": false,
"vcmp": "",
"vcmpts": false,
"vcq": "",
"vcqts": false,
"vc1": "probably",
"vc1ts": true,
"dvm": 8,
"sqt": false,
"so": "portrait-primary",
"wdw": true,
"cokys": "bG9hZFRpbWVzY3NpL=",
"ecpc": false,
"lgs": true,
"lgsod": false,
"psn": true,
"edp": false,
"addt": true,
"wsdc": true,
"ccsr": true,
"nuad": true,
"bcda": true,
"idn": true,
"capi": false,
"svde": false,
"vpbq": true,
"ucdv": false,
"spwn": false,
"emt": false,
"bfr": false,
"dbov": false,
"cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LGk9ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ2MSkpO2lmKGkpeyFmdW5jdGlvbiB0KCl7dHJ5e3ZhciBuPWlbJ1x4NzNceDY4XHg2MVx4NjRceDZmXHg3",
"stcfp": "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4ODYzNSkKICAgIGF0IHQuZXhwb3J0cy5kZF9hZCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDc3OTYpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjYzNzkw",
"ckwa": true,
"emd": "k:ai,vi,ao",
"glvd": "Qualcomm",
"glrd": "Adreno (TM) 642L",
"wwl": false,
"jset": 1734690551
}
docs/4_40_0/jsdata_ch_4_40_0_mobile_generated.json
+127
View File
@@ -0,0 +1,127 @@
{
"ttst": 15.000000002047893,
"ifov": false,
"hc": 5,
"br_oh": 620,
"br_ow": 360,
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36",
"wbd": false,
"tagpu": 9.136966557013412,
"wdif": false,
"wdifrm": false,
"npmtm": false,
"br_h": 620,
"br_w": 360,
"isf": true,
"nddc": 1,
"rs_h": 804,
"rs_w": 360,
"rs_cd": 24,
"phe": false,
"nm": false,
"jsf": false,
"lg": "fr-FR",
"pr": 3,
"ars_h": 804,
"ars_w": 360,
"tz": -120,
"str_ss": true,
"str_ls": true,
"str_idb": true,
"str_odb": true,
"plgod": false,
"plg": 2,
"plgne": "err",
"plgre": "err",
"plgof": "err",
"plggt": "err",
"pltod": false,
"hcovdr": false,
"hcovdr2": false,
"plovdr": false,
"plovdr2": false,
"ftsovdr": false,
"ftsovdr2": false,
"lb": false,
"eva": 33,
"lo": false,
"ts_mtp": 5,
"ts_tec": true,
"ts_tsa": true,
"vnd": "Google Inc.",
"bid": "NA",
"mmt": "empty",
"plu": "eTJjZUp,iZMOHLs1",
"hdn": false,
"awe": false,
"geb": false,
"dat": false,
"med": "defined",
"aco": "probably",
"acots": false,
"acmp": "probably",
"acmpts": true,
"acw": "probably",
"acwts": false,
"acma": "maybe",
"acmats": false,
"acaa": "probably",
"acaats": true,
"ac3": "",
"ac3ts": false,
"acf": "probably",
"acfts": false,
"acmp4": "maybe",
"acmp4ts": false,
"acmp3": "probably",
"acmp3ts": false,
"acwm": "maybe",
"acwmts": false,
"ocpt": false,
"vco": "",
"vcots": false,
"vch": "probably",
"vchts": true,
"vcw": "probably",
"vcwts": true,
"vc3": "maybe",
"vc3ts": false,
"vcmp": "",
"vcmpts": false,
"vcq": "",
"vcqts": false,
"vc1": "probably",
"vc1ts": true,
"dvm": 4,
"sqt": false,
"so": "portrait-primary",
"wdw": true,
"cokys": "bG9hZFRpbWVzY3NpL=",
"ecpc": false,
"lgs": true,
"lgsod": false,
"psn": true,
"edp": false,
"addt": true,
"wsdc": true,
"ccsr": true,
"nuad": true,
"bcda": true,
"idn": true,
"capi": false,
"svde": false,
"vpbq": true,
"ucdv": false,
"spwn": false,
"emt": false,
"bfr": false,
"dbov": false,
"cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LGk9ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ2MSkpO2lmKGkpeyFmdW5jdGlvbiB0KCl7dHJ5e3ZhciBuPWlbJ1x4NzNceDY4XHg2MVx4NjRceDZmXHg3",
"stcfp": "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4ODYzNSkKICAgIGF0IHQuZXhwb3J0cy5kZF9hZCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDc3OTYpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjYzNzkw",
"ckwa": true,
"emd": "k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801",
"glvd": "ARM",
"glrd": "Mali-G57 MC2",
"wwl": false,
"jset": 1734775162
}
docs/4_40_0/jsdata_le_4_40_0_mobile.json
+151
View File
@@ -0,0 +1,151 @@
{
"ttst": 46.8000000026077,
"ifov": false,
"hc": 8,
"br_oh": 763,
"br_ow": 393,
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Mobile Safari/537.36",
"wbd": false,
"tagpu": 7.9067645182878366,
"wdif": false,
"wdifrm": false,
"npmtm": false,
"br_h": 763,
"br_w": 393,
"isf": true,
"nddc": 1,
"rs_h": 873,
"rs_w": 393,
"rs_cd": 24,
"phe": false,
"nm": false,
"jsf": false,
"lg": "fr-FR",
"pr": 2.75,
"ars_h": 873,
"ars_w": 393,
"tz": -60,
"str_ss": true,
"str_ls": true,
"str_idb": true,
"str_odb": false,
"plgod": false,
"plg": 0,
"plgne": "NA",
"plgre": "NA",
"plgof": "NA",
"plggt": "NA",
"pltod": false,
"hcovdr": false,
"hcovdr2": false,
"plovdr": false,
"plovdr2": false,
"ftsovdr": false,
"ftsovdr2": false,
"lb": false,
"eva": 33,
"lo": false,
"ts_mtp": 5,
"ts_tec": true,
"ts_tsa": true,
"vnd": "Google Inc.",
"bid": "NA",
"mmt": "empty",
"plu": "empty",
"hdn": false,
"awe": false,
"geb": false,
"dat": false,
"med": "defined",
"aco": "probably",
"acots": false,
"acmp": "probably",
"acmpts": true,
"acw": "probably",
"acwts": false,
"acma": "maybe",
"acmats": false,
"acaa": "probably",
"acaats": true,
"ac3": "",
"ac3ts": false,
"acf": "probably",
"acfts": false,
"acmp4": "maybe",
"acmp4ts": false,
"acmp3": "probably",
"acmp3ts": false,
"acwm": "maybe",
"acwmts": false,
"ocpt": false,
"vco": "",
"vcots": false,
"vch": "probably",
"vchts": true,
"vcw": "probably",
"vcwts": true,
"vc3": "maybe",
"vc3ts": false,
"vcmp": "",
"vcmpts": false,
"vcq": "",
"vcqts": false,
"vc1": "probably",
"vc1ts": true,
"dvm": 8,
"sqt": false,
"so": "portrait-primary",
"wdw": true,
"cokys": "bG9hZFRpbWVzY3NpL=",
"ecpc": false,
"lgs": true,
"lgsod": false,
"psn": true,
"edp": false,
"addt": true,
"wsdc": true,
"ccsr": true,
"nuad": true,
"bcda": true,
"idn": true,
"capi": false,
"svde": false,
"vpbq": true,
"ucdv": false,
"spwn": false,
"emt": false,
"bfr": false,
"dbov": false,
"cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LGk9ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ2MSkpO2lmKGkpeyFmdW5jdGlvbiB0KCl7dHJ5e3ZhciBuPWlbJ1x4NzNceDY4XHg2MVx4NjRceDZmXHg3",
"stcfp": "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4ODYzNSkKICAgIGF0IHQuZXhwb3J0cy5kZF9hZCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDc3OTYpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjYzNzkw",
"ckwa": true,
"glvd": "Qualcomm",
"glrd": "Adreno (TM) 642L",
"wwl": false,
"tzp": "Europe/Paris",
"jset": 1734690519,
"emd": "k:ai,vi,ao",
"dcok": ".hermes.com",
"m_fmi": false,
"tbce": 0,
"es_sigmdn": null,
"es_mumdn": null,
"es_distmdn": null,
"es_angsmdn": null,
"es_angemdn": null,
"k_hA": 0,
"k_hSD": 0,
"k_pA": 862.6666666666666,
"k_pSD": 1159.6290594649463,
"k_rA": 862.6666666666666,
"k_rSD": 1159.6290594649463,
"k_ikA": 862.6666666666666,
"k_ikSD": 1159.6290594649463,
"k_kdc": 7,
"k_kuc": 7,
"m_s_c": 30,
"m_m_c": 3,
"m_c_c": 3,
"m_cm_r": 1,
"m_ms_r": 0.1
}
docs/4_40_0/jsdata_le_4_40_0_mobile_generated.json
+149
View File
@@ -0,0 +1,149 @@
{
"ttst": 13.000000008223282,
"ifov": false,
"hc": 5,
"br_oh": 620,
"br_ow": 360,
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36",
"wbd": false,
"tagpu": 5.501734809203354,
"wdif": false,
"wdifrm": false,
"npmtm": false,
"br_h": 620,
"br_w": 360,
"isf": true,
"nddc": 1,
"rs_h": 804,
"rs_w": 360,
"rs_cd": 24,
"phe": false,
"nm": false,
"jsf": false,
"lg": "fr-FR",
"pr": 3,
"ars_h": 804,
"ars_w": 360,
"tz": -120,
"str_ss": true,
"str_ls": true,
"str_idb": true,
"str_odb": true,
"plgod": false,
"plg": 2,
"plgne": "err",
"plgre": "err",
"plgof": "err",
"plggt": "err",
"pltod": false,
"hcovdr": false,
"hcovdr2": false,
"plovdr": false,
"plovdr2": false,
"ftsovdr": false,
"ftsovdr2": false,
"lb": false,
"eva": 33,
"lo": false,
"ts_mtp": 5,
"ts_tec": true,
"ts_tsa": true,
"vnd": "Google Inc.",
"bid": "NA",
"mmt": "empty",
"plu": "eTJjZUp,iZMOHLs1",
"hdn": false,
"awe": false,
"geb": false,
"dat": false,
"med": "defined",
"aco": "probably",
"acots": false,
"acmp": "probably",
"acmpts": true,
"acw": "probably",
"acwts": false,
"acma": "maybe",
"acmats": false,
"acaa": "probably",
"acaats": true,
"ac3": "",
"ac3ts": false,
"acf": "probably",
"acfts": false,
"acmp4": "maybe",
"acmp4ts": false,
"acmp3": "probably",
"acmp3ts": false,
"acwm": "maybe",
"acwmts": false,
"ocpt": false,
"vco": "",
"vcots": false,
"vch": "probably",
"vchts": true,
"vcw": "probably",
"vcwts": true,
"vc3": "maybe",
"vc3ts": false,
"vcmp": "",
"vcmpts": false,
"vcq": "",
"vcqts": false,
"vc1": "probably",
"vc1ts": true,
"dvm": 4,
"sqt": false,
"so": "portrait-primary",
"wdw": true,
"cokys": "bG9hZFRpbWVzY3NpL=",
"ecpc": false,
"lgs": true,
"lgsod": false,
"psn": true,
"edp": false,
"addt": true,
"wsdc": true,
"ccsr": true,
"nuad": true,
"bcda": true,
"idn": true,
"capi": false,
"svde": false,
"vpbq": true,
"ucdv": false,
"spwn": false,
"emt": false,
"bfr": false,
"dbov": false,
"cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LGk9ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ2MSkpO2lmKGkpeyFmdW5jdGlvbiB0KCl7dHJ5e3ZhciBuPWlbJ1x4NzNceDY4XHg2MVx4NjRceDZmXHg3",
"stcfp": "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4ODYzNSkKICAgIGF0IHQuZXhwb3J0cy5kZF9hZCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDc3OTYpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjYzNzkw",
"ckwa": true,
"emd": "k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801",
"glvd": "ARM",
"glrd": "Mali-G57 MC2",
"wwl": false,
"jset": 1734775843,
"dcok": ".hermes.com",
"log2": true,
"es_sigmdn": null,
"es_mumdn": null,
"es_distmdn": null,
"es_angsmdn": null,
"es_angemdn": null,
"k_hA": null,
"k_hSD": null,
"k_pA": null,
"k_pSD": null,
"k_rA": null,
"k_rSD": null,
"k_ikA": null,
"k_ikSD": null,
"k_kdc": 0,
"k_kuc": 0,
"m_s_c": 0,
"m_m_c": 3,
"m_c_c": 3,
"m_cm_r": 1,
"m_ms_r": -1
}
link_validator_executor.py
+1 -1
View File
@@ -10,6 +10,6 @@ if __name__ == '__main__':
# generate test data
while True:
print("call validate_all_links()")
validate_all_links()
validate_all_links([])
print("wait for 30 seconds")
time.sleep(30)
models/LinkPojo.py
+7 -2
View File
@@ -1,7 +1,9 @@
class LinkPojo():
def __init__(self, url, email, updated_at, ip_country):
def __init__(self, url, email, model, serial, updated_at, ip_country):
self.url = url
self.email = email
self.model = model
self.serial = serial
self.updated_at = updated_at
self.ip_country = ip_country
@@ -9,9 +11,12 @@ class LinkPojo():
def from_firestore_dict(source):
updated_at = source['updated_at']
email = source['email']
model = source['model']
serial = source['serial']
url = source['url']
ip_country = "FR"
if source.get('ip_country'):
ip_country = source['ip_country']
result = LinkPojo(email=email, url=url, updated_at=updated_at, ip_country=ip_country)
result = LinkPojo(email=email, url=url, model=model, serial=serial, updated_at=updated_at,
ip_country=ip_country)
return result
models/contact_pojo.py
+7
View File
@@ -13,6 +13,7 @@ class ContactPojo:
store: str
note: str
ua: str
serial: str
def __init__(self, phone_number: str, passport_number: str, last_name: str, first_name: str, mail: str,
ccid: str = "",
@@ -27,6 +28,7 @@ class ContactPojo:
self.note = ""
self.ua = ""
self.current_ip = ""
self.serial = "reqeusts"
def to_firestore_dict(self):
dest = {
@@ -38,6 +40,7 @@ class ContactPojo:
u'store': self.store,
u'ccid': self.ccid,
u'position': self.position,
u'serial': self.serial,
u'current_ip': self.current_ip,
u'ua': self.ua
}
@@ -52,9 +55,13 @@ class ContactPojo:
last_name = source['last_name']
first_name = source['first_name']
ip_country = "FR"
if source.get('ip_country'):
ip_country = source['ip_country']
result = ContactPojo(phone_number=phone, passport_number=passport, mail=email,
last_name=last_name, first_name=first_name)
result.ip_country = ip_country
if source.get('serial'):
serial = source['serial']
result.serial = serial
return result
models/jsdata_le_pojo.py
+2 -2
View File
@@ -50,8 +50,8 @@ class JsDataLeTypePojo(JsDataPojo):
self.m_c_c = m_c_c
self.m_cm_r = m_cm_r
self.m_ms_r = m_ms_r
self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.35.0
self.stcfp = "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4MjIzMikKICAgIGF0IHQuZXhwb3J0cy5kZF9hYyAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDA3MDEpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU3ODc0" # 4.35.0
self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LGk9ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ2MSkpO2lmKGkpeyFmdW5jdGlvbiB0KCl7dHJ5e3ZhciBuPWlbJ1x4NzNceDY4XHg2MVx4NjRceDZmXHg3" # 4.40.0
self.stcfp = "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4ODYzNSkKICAgIGF0IHQuZXhwb3J0cy5kZF9hZCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDc3OTYpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjYzNzkw" # 4.40.0
test_data_json = """{"glvd":"ARM","glrd":"Mali-G57 MC2","hc":5,"br_oh":620,"br_ow":360,"br_h":620,"br_w":360,"rs_h":804,"rs_w":360,"rs_cd":24,"ars_h":804,"ars_w":360,"plg":2,"eva":33,"vnd":"Google Inc.","plu":["eTJjZUp","iZMOHLs1"],"ua":"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36","dvm":4,"acw":"probably","pr":3,"ts_mtp":5,"tz":-120,"emd":"k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801","cookiesStr":"app.sig=_FqS3_SPN2D40Z4Qlq-_ybxzRvQ;datadome=VSGCNMYkY6L81typRJUpurlyd39dHxdfxIYc6DzRWDT7_rtZ8UodGcbow1okQR2zg_t4d_NcP0rC1ZHNz3XSDAGFZCJhFI49lai~x2Nb2unkhOlY8o4E04Wjmde~auDZ;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiLW9jbktLVXAzMjVTOEFEdFpLUU9kSXlMIn0=;policy=accepted;lang=fr;"}"""
models/jsdata_pojo.py
+3 -3
View File
@@ -138,7 +138,7 @@ class JsDataPojo:
def __init__(self, glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu,
vnd, pr, ts_mtp, dvm, emd):
# self.opts = "endpoint,ajaxListenerPath"
self.ttst = random.randint(38, 148)
self.ttst = (random.randint(5, 46) * 10 ** 16 + random.randint(1 * 10 ** 7, 9 * 10 ** 7)) / 10 ** 16
self.ifov = False
self.hc = hc
self.br_oh = br_oh
@@ -266,8 +266,8 @@ class JsDataPojo:
self.emt = False
self.bfr = False
self.dbov = False
self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.35.0
self.stcfp = "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4MjIzMikKICAgIGF0IHQuZXhwb3J0cy5kZF9hYyAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDA3MDEpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjU3ODc0" # 4.35.0
self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LGk9ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ2MSkpO2lmKGkpeyFmdW5jdGlvbiB0KCl7dHJ5e3ZhciBuPWlbJ1x4NzNceDY4XHg2MVx4NjRceDZmXHg3" # 4.40.0
self.stcfp = "Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo4ODYzNSkKICAgIGF0IHQuZXhwb3J0cy5kZF9hZCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6MjoxMDc3OTYpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjYzNzkw" # 4.40.0
self.ckwa = True
def to_url_encoded_json(self):
proxy_manager/proxy_manager.py
+18 -6
View File
@@ -9,6 +9,11 @@ FR_PROXY_RES_OXY = {
'http': 'http://customer-rendezvous-cc-FR:Rdv202220212023@pr.oxylabs.io:7777',
'https': 'http://customer-rendezvous-cc-FR:Rdv202220212023@pr.oxylabs.io:7777'
}
FR_PROXY_RES_PARIS_OXY = {
# customer-rendezvous-cc-fr-city-paris:PASSWORD@pr.oxylabs.io:7777
'http': 'http://customer-rendezvous-cc-fr-city-paris:Rdv202220212023@pr.oxylabs.io:7777',
'https': 'http://customer-rendezvous-cc-fr-city-paris:Rdv202220212023@pr.oxylabs.io:7777'
}
FR_PROXY_MOB_OXY = {
'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@pr.oxylabs.io:7777',
@@ -42,6 +47,10 @@ FR_MOBILE_ANY_IP_STICKY = {
'http': 'http://user_6a7f21,type_mobile,country_FR,session_{}:d5c051@portal.anyip.io:1080',
'https': 'http://user_6a7f21,type_mobile,country_FR,session_{}:d5c051@portal.anyip.io:1080',
}
FR_MOBILE_ANY_IP_ROTATING = {
'http': 'http://user_6a7f21,type_residential,country_FR:d5c051@portal.anyip.io:1080',
'https': 'http://user_6a7f21,type_residential,country_FR:d5c051@portal.anyip.io:1080',
}
# 八分之一用data_impulse
MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY,
FR_PROXY_MOB_OXY_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY, FR_MOBILE_ANY_IP_STICKY,
@@ -49,6 +58,7 @@ MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_
FR_PROXY_RES_OXY_STICKY,
FR_PROXY_DATA_IMPULSE_STICKY]
# MOBILE_PROXY_LIST = [FR_MOBILE_ANY_IP_STICKY]
@@ -61,12 +71,14 @@ class ProxyManager:
self.logger = logger
def get_link_validate_proxy(self, links_to_validate: list) -> list:
if len(links_to_validate) > 15:
return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
# return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY]
else:
return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
# return [FR_PROXY_RES_PARIS_OXY]
return [FR_MOBILE_ANY_IP_ROTATING]
# if len(links_to_validate) > 15:
# return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
# # return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY]
# else:
#
# return [FR_PROXY_RES_OXY, FR_PROXY_MOB_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES]
# return [FR_PROXY_RES_OXY]
def get_result_link_proxy(self):
queue_message/CookiesPublisher.py
+1 -1
View File
@@ -31,7 +31,7 @@ class CookiesPublisher:
self.queue_method = self.channel.queue_declare(queue=self.to_queue, durable=True)
def publish_body(self, body: str):
print("will push to queue {}".format(self.to_queue))
print("will push to queue {}".format(body))
self.channel.basic_publish(exchange='', routing_key=self.to_queue, body=body, properties=pika.BasicProperties(
delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE
))
queue_message/appointmentrequestsender.py
+4 -4
View File
@@ -101,8 +101,8 @@ class AppointmentRequestSender(threading.Thread):
_received_object = body.decode("UTF-8")
self.logger.info(f" [x] Received {_received_object}")
# 同一个COOKIES保持用同一个IP
_proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
print("proxy to use is {}".format(_proxy_to_use))
# _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
# print("proxy to use is {}".format(_proxy_to_use))
if "glrd" in _received_object:
_received_dict = json.loads(_received_object)
js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'],
@@ -124,8 +124,8 @@ class AppointmentRequestSender(threading.Thread):
# self.contact_list = filter_contacts(self.contact_list)
can_continue = None
for con in self.contact_list:
# _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
# print(_proxy_to_use)
_proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
print(_proxy_to_use)
sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher,
received_dict=_received_dict,
proxy_to_use=_proxy_to_use, logger=self.logger)
request_sender_test.py
+2 -2
View File
@@ -103,6 +103,6 @@ if __name__ == '__main__':
# '~/Desktop/contact_list_2024-05-21.xlsx',
# '~/Desktop/15_05_to_test.xlsx']
# file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx']
file_list = ['~/Desktop/contact_list_all.xlsx']
file_list = ['~/Desktop/contact_list_2024-10-05_gmx_net_100.xlsx']
send_request_for_file_list(file_list=file_list, thread_number=1,
data_queue_name=MORNING_DATA_CACHE, stop_at_hour=18, stop_at_mins=30)
data_queue_name=MORNING_DATA_CACHE, stop_at_hour=19, stop_at_mins=50)
utils/address_ip.py
+11 -11
View File
@@ -6,18 +6,18 @@ from proxy_manager.proxy_manager import ProxyManager
def get_address_ip(proxy_to_use):
return ""
# try:
# response = requests.get(url="https://api.ipify.org", proxies=proxy_to_use, verify=False,
# timeout=15)
# print(response.status_code)
# if response.status_code == 200:
# return response.text
# else:
# return ""
# except Exception as Error:
# print(Error)
# return "error"
try:
response = requests.get(url="https://api.ipify.org", proxies=proxy_to_use, verify=False,
timeout=15)
print(response.status_code)
if response.status_code == 200:
return response.text
else:
return ""
except Exception as Error:
print(Error)
return "error"
if __name__ == '__main__':
utils/tls_ja3.py
+29
View File
@@ -0,0 +1,29 @@
# import requests
from curl_cffi import requests
# chrome99
# chrome100
# chrome101
# chrome104
# chrome107
# chrome110
# chrome116 [1]
# chrome119 [1]
# chrome120 [1]
# chrome123 [3]
# chrome124 [3]
# chrome99_android
# edge99
# edge101
# safari15_3 [2]
# safari15_5 [2]
# safari17_0 [1]
# safari17_2_ios [1]
# safari18_0 [4]
# safari18_0_ios [4]
TLS_JA3_CHECK_SERVER = "https://tls.browserleaks.com/json"
if __name__ == '__main__':
_res = requests.get(TLS_JA3_CHECK_SERVER, impersonate="chrome99_android")
print(_res.text)
utils/user_agent_helper.py
+57
View File
@@ -0,0 +1,57 @@
import re
from http.cookies import SimpleCookie
from pprint import pprint
def get_chrome_version_from_ua(ua: str) -> str:
_regex = "Chrome\/([0-9]+)"
_match = re.search(_regex, ua)
if _match:
return _match.group(1)
def generate_headers_from_request_message(_received_dict, cookie_str):
_ua = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36'
cookie = SimpleCookie()
_chrome_version = '125'
if _received_dict is not None:
_ua = _received_dict['ua']
_chrome_version = get_chrome_version_from_ua(_ua)
_model = _received_dict['model']
# _ua = "Mozilla/5.0 (Linux; Android 9; {}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.58 Mobile Safari/537.36".format(
# _model)
print("model:" + _model)
cookie.load(cookie_str)
_referer = "android-app://com.google.android.gm/"
headers = {
'User-Agent': _ua,
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'max-age=0',
'Referer': _referer,
'Cookie': cookie_str,
'Sec-Fetch-Mode': 'navigate',
'Host': 'rendezvousparis.hermes.com',
'Sec-Fetch-Site': 'same-origin',
'sec-ch-ua': '"Google Chrome";v="{}", "Chromium";v="{}", "Not.A/Brand";v="24"'.format(_chrome_version,
_chrome_version),
'sec-ch-ua-platform': '"Android"',
'sec-ch-ua-model': '"{}"'.format(_model),
'Sec-Fetch-Dest': 'document',
'Accept-Language': 'fr-FR,fr;q=0.6'}
return headers
if __name__ == '__main__':
# _ua_to_test = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Mobile Safari/537.36"
# _ua_to_test = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Mobile Safari/537.36"
# print(get_chrome_version_from_ua(_ua_to_test)) # Output: 129.0.0.0
_received_dict = {"glvd": "Google Inc. (ARM)", "glrd": "ANGLE (ARM, Mali-G57 MC2, OpenGL ES 3.2)", "hc": 8,
"br_oh": 745, "br_ow": 393, "br_h": 745, "br_w": 393, "rs_h": 876, "rs_w": 393, "rs_cd": 24,
"ars_h": 876, "ars_w": 393, "plg": 0, "eva": 33, "vnd": "Google Inc.", "plu": [],
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Mobile Safari/537.36",
"dvm": 4, "acw": "probably", "pr": 2.75, "ts_mtp": 5, "tz": -60, "emd": "k:ai,vi,ao",
"cookiesStr": "datadome=cVhteVJiGoU3N1T61hX4dF5vNG4~p5JpkbsKLH674kzQiuT1G5sDohLD33nFx2QFqlPiJfY~yfsRNIpqM8xZ1JfXTgT7p8v5QfkmmOl~BIhLOVCBHe_6Al6CM2QsSq7g;__cf_bm=J9g80A6wlNO03BLqyhiuf5ZRkx_8Ig7QIOLhHoEh1i8-1734010016-1.0.1.1-fMIRmq.K8K093kcSzJdFlcIEoCH9XwWhDlsXFoDmvNnerYvpyG0eC9vIdqj5xACF28YYAYGXwuag5f33JoDiBg;",
"model": "22041219PG", "serial": "fmiz5pa6rsx4u4ts"}
pprint(generate_headers_from_request_message(_received_dict, _received_dict["cookiesStr"]))
workers/captcha_result_getter.py
+3 -4
View File
@@ -112,16 +112,15 @@ class CaptchaResultGetter:
return _raw_data
def get_valid_ch_cookie(self, proxy_to_use, js_data: JsDataPojo, old_valid_cookie: str = None,
referrer: str = None) -> Union[str, None]:
referrer: str = "https://rendezvousparis.hermes.com/",
origin="https://rendezvousparis.hermes.com/") -> Union[str, None]:
# extract datadome cookie
# get new cookie
_cookies_to_use = get_datadome_cookies(old_valid_cookie)
_app_cookies = get_app_cookies(old_valid_cookie)
_lang_cookies = get_lang_cookies(old_valid_cookie)
_origin = "https://rendezvousparis.hermes.com/"
_origin = origin
# _origin = "https://www.hermes.com"
_referer = "https://rendezvousparis.hermes.com/"
if referrer is not None:
_referer = referrer
# _referer = "https://www.hermes.com"
workers/cookie_generator.py
+71 -23
View File
@@ -1,3 +1,5 @@
import json
import logging
import random
import threading
import time
@@ -5,8 +7,11 @@ from http.cookies import SimpleCookie
import pika
from models.jsdata_le_pojo import JsDataLeTypePojo
from models.jsdata_pojo import JsDataPojo
from proxy_manager.proxy_manager import ProxyManager
from queue_message.CookiesPublisher import CookiesPublisher, QUEUE_HOST, credentials, REQUEST_DATA_QUEUE, \
REQUEST_DATA_QUEUE_TEST
REQUEST_DATA_QUEUE_TEST, MORNING_DATA_CACHE_BAK
from workers.captcha_result_getter import CaptchaResultGetter
from workers.proxies_constants import PROXY_LIST_FR
@@ -14,13 +19,16 @@ DVM_LIST = [2, 3, 4, 6]
class CookiesGenerator(threading.Thread):
def __init__(self, proxy_to_use_list: list, cookiesPublisher: CookiesPublisher):
def __init__(self, proxy_manager: ProxyManager, cookiesPublisher: CookiesPublisher,
logger=logging.getLogger(__name__)):
super().__init__()
self.connection = None
self.cookiesPublisher = cookiesPublisher
self.channel = None
self.valid_csrf = None
self.proxy_to_use_list = proxy_to_use_list
self.proxy_manager = proxy_manager
self.queue_name = MORNING_DATA_CACHE_BAK
self.logger = logger
def set_up_connection(self):
self.connection = pika.BlockingConnection(
@@ -29,28 +37,24 @@ class CookiesGenerator(threading.Thread):
def listen_to_queue(self, callback):
self.channel.basic_qos(prefetch_count=1)
self.channel.basic_consume(queue=REQUEST_DATA_QUEUE, auto_ack=False, on_message_callback=callback)
self.channel.basic_consume(queue=self.queue_name, auto_ack=False, on_message_callback=callback)
self.channel.start_consuming()
def message_count(self):
return self.channel.queue_declare(queue=REQUEST_DATA_QUEUE, durable=True).method.message_count
return self.channel.queue_declare(queue=self.queue_name, durable=True).method.message_count
def on_message(self, ch, method, properties, body):
print(f" [x] Received {body}")
_cached_message_count = self.cookiesPublisher.message_count()
_valid_message_count = self.message_count()
print("message count in cache is {}".format(_cached_message_count))
print("message count in current queue is {}".format(_valid_message_count))
_received_object = body.decode("UTF-8")
_received_dict = json.loads(_received_object)
_received_cookies = _received_dict["cookiesStr"]
# generate 10 cookies to new_queue
captchaResultGetter = CaptchaResultGetter()
_cookies_count = random.randint(8, 10)
_received_cookies = body.decode("UTF-8")
dvm = random.choice(DVM_LIST)
hc = random.choice(DVM_LIST)
if _cached_message_count < 10 and _valid_message_count > 40:
_proxy_to_use = random.choice(self.proxy_to_use_list)
for i in range(1, _cookies_count):
_proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request()
# new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, dvm=dvm, hc=hc)
_app_sig = get_app_and_app_sig(_received_cookies)
# if new_cookie is not None and _app_sig is not None:
@@ -60,19 +64,63 @@ class CookiesGenerator(threading.Thread):
# print("new_cookie is " + new_cookie)
# _received_cookies = new_cookie
# self.cookiesPublisher.publish_body(new_cookie)
new_cookie = captchaResultGetter.get_valid_cookie(proxy_to_use=_proxy_to_use,
old_valid_cookie=_received_cookies, dvm=dvm,
hc=hc)
if new_cookie is not None and _app_sig is not None:
js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'],
ua=_received_dict['ua'], br_oh=_received_dict['br_oh'], br_ow=_received_dict['br_ow'],
ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'], pr=_received_dict['pr'],
plg=_received_dict['plg'], br_h=_received_dict['br_h'], br_w=_received_dict['br_w'],
plu=_received_dict['plu'], vnd=_received_dict['vnd'], dvm=_received_dict['dvm'],
ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'],
rs_h=_received_dict['rs_h'],
rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'], emd=_received_dict['emd'])
new_cookie = captchaResultGetter.get_valid_ch_cookie(proxy_to_use=_proxy_to_use, js_data=js_data,
old_valid_cookie=_received_cookies,
origin="https://www.hermes.com",
referrer="https://www.hermes.com")
if new_cookie is not None:
new_cookie = _app_sig + "policy=accepted;lang=fr;" + new_cookie
new_cookie = new_cookie.replace("Domain=.hermes.com;", "").replace("Path=/;", "").replace(
"Secure; SameSite=None", "").replace("Max-Age=31536000;", "").replace(" ", "")
new_cookie = new_cookie.replace("Path=/;", "").replace(
"Secure; SameSite=None", "").replace(" ", "")
_received_cookies = new_cookie
self.cookiesPublisher.publish_body(new_cookie)
ch.basic_ack(delivery_tag=method.delivery_tag)
# _received_dict["cookiesStr"] = new_cookie
# self.cookiesPublisher.publish_body(json.dumps(_received_dict))
if _received_cookies is not None:
self.logger.info("new cookie is " + _received_cookies)
time.sleep(random.randint(1, 3))
# m_s_c = f.scroll
m_s_c = random.randint(0, 3)
m_c_c = random.randint(3, 5) # click count
m_m_c = random.randint(3, 5) # move count
m_cm_r = m_c_c / m_m_c
m_ms_r = -1 # move scroll ratio
if m_s_c == 0:
m_ms_r = -1
else:
time.sleep(30)
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
m_ms_r = m_m_c / m_s_c
js_le_data = JsDataLeTypePojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'],
hc=_received_dict['hc'],
ua=_received_dict['ua'], br_oh=_received_dict['br_oh'],
br_ow=_received_dict['br_ow'],
ars_h=_received_dict['ars_h'], ars_w=_received_dict['ars_w'],
pr=_received_dict['pr'],
plg=_received_dict['plg'], br_h=_received_dict['br_h'],
br_w=_received_dict['br_w'],
plu=_received_dict['plu'], vnd=_received_dict['vnd'],
dvm=_received_dict['dvm'],
ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'],
rs_h=_received_dict['rs_h'],
rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'],
m_s_c=m_s_c, m_m_c=m_m_c, m_c_c=m_c_c,
m_cm_r=m_cm_r, m_ms_r=m_ms_r, emd=_received_dict['emd'])
time.sleep(random.randint(1, 4))
_new_le_cookies = captchaResultGetter.get_le_valid_cookie(proxy_to_use=_proxy_to_use,
js_le_type_data=js_le_data,
old_valid_cookie=_received_cookies)
_received_dict["cookiesStr"] = new_cookie
self.cookiesPublisher.publish_body(json.dumps(_received_dict))
# self.cookiesPublisher.publish_body(new_cookie)
ch.basic_ack(delivery_tag=method.delivery_tag)
def run(self):
print(threading.currentThread().name + " starts")
workers/de_cookie_generator.py
+12 -4
View File
@@ -1,9 +1,17 @@
from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher
import logging
import sys
from proxy_manager.proxy_manager import ProxyManager
from queue_message.CookiesPublisher import REQUEST_DATA_QUEUE_DE, CookiesPublisher, MORNING_DATA_CACHE
from utils.AppLogging import init_logger
from workers.cookie_generator import CookiesGenerator
from workers.proxies_constants import PROXY_LIST_DE
if __name__ == '__main__':
cookiesPublisher = CookiesPublisher(queue_name=REQUEST_DATA_QUEUE_DE)
init_logger()
logger = logging.getLogger()
logger.addHandler(logging.StreamHandler(stream=sys.stdout))
cookiesPublisher = CookiesPublisher(queue_name=MORNING_DATA_CACHE)
cookiesPublisher.set_up_connection()
cookieGenerator = CookiesGenerator(cookiesPublisher=cookiesPublisher, proxy_to_use_list=PROXY_LIST_DE)
cookieGenerator = CookiesGenerator(proxy_manager=ProxyManager(logger=logger), cookiesPublisher=cookiesPublisher,
logger=logger)
cookieGenerator.run()
workers/link_validator_with_provided_list.py
+55 -52
View File
@@ -7,17 +7,16 @@ from datetime import datetime
from http.cookies import SimpleCookie
import pika
import requests
from curl_cffi import requests
from db.mongo_manager import MONGO_STORE_MANAGER
from models.LinkPojo import LinkPojo
from models.result_pojo import RequestResult
from proxy_manager.proxy_manager import ProxyManager
from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST, TEST_QUEUE, SHARED_OBJECT, \
MORNING_DATA_CACHE_BAK, MORNING_DATA_CACHE, MORNING_DATA_CACHE_2
from queue_message.CookiesPublisher import CookiesPublisher, MORNING_DATA_CACHE, MORNING_DATA_CACHE_BAK
from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials
from utils.AppLogging import init_logger
from workers.proxies_constants import PROXY_LIST_FR
from utils.user_agent_helper import generate_headers_from_request_message
DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ces coordonnées"
INVALID = "Depuis plus de 130 ans,"
@@ -26,13 +25,25 @@ init_logger()
logger = logging.getLogger()
def filter_link_pojo_list_with_serial(_received_dict, link_to_validate_list):
_serial = _received_dict["serial"]
_model = _received_dict["model"]
_bit_browser_list = list(filter(lambda link_pojo: link_pojo.serial == "bitbrowser", link_to_validate_list))
if len(_bit_browser_list) > 0:
return _bit_browser_list
else:
_to_return = filter(lambda link_pojo: link_pojo.serial == _serial, link_to_validate_list)
return list(_to_return)
class LinkValidatorWithProvidedList(threading.Thread):
def __init__(self, cookiesPublisher: CookiesPublisher, link_list: list,
def __init__(self, cookiesPublisher: CookiesPublisher, link_list: list, _contact_serial_list,
queue_to_listen=REQUEST_DATA_QUEUE,
ip_country="FR",
limit=40):
super().__init__()
self.contact_serial_list = _contact_serial_list
self.link_to_validate_list = link_list
self.cookie = SimpleCookie()
self.cookiesPublisher = cookiesPublisher
@@ -56,34 +67,22 @@ class LinkValidatorWithProvidedList(threading.Thread):
_ua = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36'
if _received_dict is not None:
_ua = _received_dict['ua']
_model = _received_dict['model']
logger.info("model: %s", _model)
self.cookie.load(self.cookie_str)
headers = {
'User-Agent': _ua,
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'max-age=0',
'Referer': linkPojo.url,
'Cookie': self.cookie_str,
'Sec-Fetch-Mode': 'navigate',
'Host': 'rendezvousparis.hermes.com',
'Sec-Fetch-Site': 'same-origin',
'sec-ch-ua': '"Brave";v="125", "Chromium";v="125", "Not.A/Brand";v="24"',
'sec-ch-ua-platform': '"Android"',
'sec-ch-ua-model': '""',
'Sec-Fetch-Dest': 'document',
'Accept-Language': 'fr-FR,fr;q=0.6'}
headers = generate_headers_from_request_message(_received_dict, self.cookie_str)
print("received cookie is " + str(self.cookie_str))
print("send request for link: " + linkPojo.url)
try:
print("will send request with ua {}".format(_ua))
print("will send request with cookie {}".format(self.cookie_str))
response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use,
timeout=60)
timeout=60, impersonate="chrome99_android")
print(response.status_code)
if response.status_code == 200:
_content = response.text
print(response.text)
if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content:
if "Votre demande de rendez-vous est enregistrée" in _content:
print(response.url)
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo,
segement_position=threading.currentThread().name,
@@ -106,7 +105,6 @@ class LinkValidatorWithProvidedList(threading.Thread):
self.cookie_str = new_coolies_str
return RequestResult.SUCCESS
elif INVALID in _content:
MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True, ua=_ua)
# set new cookies
_cookies_to_set = response.headers['set-cookie']
@@ -151,21 +149,18 @@ class LinkValidatorWithProvidedList(threading.Thread):
_message_in_queue_count = self.cookiesPublisher.message_count()
print("message count in queue is {}".format(_message_in_queue_count))
_received_object = body.decode("UTF-8")
_received_dict = None
if "glrd" in _received_object:
_received_dict = json.loads(_received_object)
_received_cookies = _received_dict["cookiesStr"]
else:
_received_cookies = _received_object
self.cookie_str = _received_cookies
random.shuffle(self.link_to_validate_list)
if len(self.link_to_validate_list) > 0 and _message_in_queue_count >= self.limit:
print("{}:links number is {}".format(threading.currentThread().name, len(self.link_to_validate_list)))
_links_to_validate = filter_link_pojo_list_with_serial(_received_dict, self.link_to_validate_list)
random.shuffle(_links_to_validate)
if len(_links_to_validate) > 0 and _message_in_queue_count >= self.limit:
print("{}:links number is {}".format(threading.currentThread().name, len(_links_to_validate)))
can_continue = None
for link_to_validate in self.link_to_validate_list:
for link_to_validate in _links_to_validate:
print(link_to_validate)
self.proxy_to_use = random.choice(
self.proxy_manager.get_link_validate_proxy(self.link_to_validate_list))
self.proxy_manager.get_link_validate_proxy(_links_to_validate))
print("proxy to use is {}".format(self.proxy_to_use))
can_continue = self.send_request(link_to_validate, _received_dict)
# remove the tested link from link list
@@ -186,9 +181,11 @@ class LinkValidatorWithProvidedList(threading.Thread):
print("will ack")
ch.basic_ack(delivery_tag=method.delivery_tag)
else:
print("empty list, no need to ack")
time.sleep(5)
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True)
print("empty list, will republish message")
time.sleep(0)
print("body in json:{}".format(json.dumps(_received_dict)))
self.cookiesPublisher.publish_body(json.dumps(_received_dict))
ch.basic_ack(delivery_tag=method.delivery_tag)
self.channel.stop_consuming()
def filter_with_ip_country(self):
@@ -199,9 +196,10 @@ class LinkValidatorWithProvidedList(threading.Thread):
self.link_to_validate_list = _link_list_to_click
def validate_links(cookiesPublisher, queue_name: str, link_list: list):
def validate_links(cookiesPublisher, queue_name: str, link_list: list, _contact_serial_list):
receiver = LinkValidatorWithProvidedList(cookiesPublisher=cookiesPublisher, link_list=link_list,
queue_to_listen=queue_name, ip_country="FR", limit=0)
queue_to_listen=queue_name, _contact_serial_list=_contact_serial_list,
ip_country="FR", limit=0)
print("{} set_up_connection".format(threading.currentThread().name))
receiver.set_up_connection()
receiver.listen_to_queue(receiver.on_message)
@@ -209,26 +207,28 @@ def validate_links(cookiesPublisher, queue_name: str, link_list: list):
# default_segment_number 并发数，决定速度
# divided = 4，越小，一次处理得越多
def validate_all_links():
def validate_all_links(_contact_serial_list):
print("will get all links")
all_link_list = MONGO_STORE_MANAGER.get_links_to_validate()
# get the first 50 links
if len(all_link_list) == 0:
return
divided = 3
default_segment_number = 20
_first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))]
# divided = 1
# default_segment_number = 20
# _first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))]
_first_25_percent_links = all_link_list
_queue_name = MORNING_DATA_CACHE
# _queue_name = MORNING_DATA_CACHE_BAK
if len(all_link_list) > divided * default_segment_number:
_segment_number = default_segment_number
else:
_first_25_percent_links = all_link_list
if len(_first_25_percent_links) > divided:
_segment_number = int(len(_first_25_percent_links) / divided)
else:
_segment_number = 1
# if len(all_link_list) > divided * default_segment_number:
# _segment_number = default_segment_number
# else:
# _first_25_percent_links = all_link_list
# if len(_first_25_percent_links) > divided:
# _segment_number = int(len(_first_25_percent_links) / divided)
# else:
# _segment_number = 1
_thread_list = []
_segment_number = 1
for i in range(0, _segment_number):
logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_25_percent_links)))
logger.info("segment is {}".format(i))
@@ -236,7 +236,8 @@ def validate_all_links():
_cookiesPublisher.set_up_connection()
_step = int(len(_first_25_percent_links) / _segment_number)
_sublist = _first_25_percent_links[i * _step:_step * (i + 1)]
_thread1 = threading.Thread(target=validate_links, args=(_cookiesPublisher, _queue_name, _sublist))
_thread1 = threading.Thread(target=validate_links,
args=(_cookiesPublisher, _queue_name, _sublist, _contact_serial_list))
_thread_list.append(_thread1)
_thread1.start()
for _thread in _thread_list:
@@ -245,10 +246,12 @@ def validate_all_links():
if __name__ == '__main__':
# generate test data
# contact_serial_map_list = MONGO_STORE_MANAGER.get_all_contact_serial_list()
while True:
print("call validate_all_links()")
validate_all_links()
delay = random.randint(10, 30)
validate_all_links([])
# delay = random.randint(10, 30)
delay = random.randint(1, 10)
current_time = datetime.now()
current_hour = current_time.hour
print("Current hour ", current_time.hour)
workers/sender.py
+10 -7
View File
@@ -6,7 +6,7 @@ import re
import time
from http.cookies import SimpleCookie
from models.contact_pojo import ContactPojo
import requests
from curl_cffi import requests
from db.mongo_manager import MONGO_STORE_MANAGER
from models.ReserveResultPojo import ReserveResultPojo, PublishType
@@ -118,6 +118,8 @@ class Sender:
self._csrf = csrf
_csrf = csrf
self.logger.info("model is:{}".format(model))
js_data.ua = "Mozilla/5.0 (Linux; Android 9; {}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.58 Mobile Safari/537.36".format(
model)
_chrome_code = get_chrome_version_from_ua(js_data.ua)
_chrome_version = None
if _chrome_code == '128':
@@ -128,11 +130,12 @@ class Sender:
_chrome_version = "116.0.5845.114"
elif _chrome_code == '114':
_chrome_version = "114.0.5735.53"
_chrome_ua_version = '"Chromium";v="{}", "Not;A=Brand";v="24.0.0.0", "Google Chrome";v="{}"'.format(
_chrome_version, _chrome_version)
_chrome_ua_header = '"Chromium";v="{}", "Not;A=Brand";v="24", "Google Chrome";v="{}"'.format(_chrome_code,
_chrome_code)
elif _chrome_code == '99':
_chrome_version = "99.0.4844.58"
# 99: " Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"
# " Not A;Brand";v="99.0.0.0", "Chromium";v="99.0.4844.58", "Google Chrome";v="99.0.4844.58"
_chrome_ua_version = 'Not A;Brand";v="99.0.0.0", "Chromium";v="99.0.4844.58", "Google Chrome";v="99.0.4844.58"'
_chrome_ua_header = 'Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"'
self.logger.info("chrome_ua_header is:{}".format(_chrome_ua_header))
self.logger.info("_chrome_ua_version is:{}".format(_chrome_ua_version))
# 默认为空，也就是随机
@@ -168,7 +171,7 @@ class Sender:
proxy_to_use = self.proxy_to_use
# print(proxy_to_use)
response = requests.post(url=url, proxies=proxy_to_use, verify=False, headers=headers, data=data,
timeout=15, allow_redirects=False)
timeout=15, allow_redirects=False, impersonate="chrome99_android")
self.logger.info(response.status_code)
if response.status_code == 302:
# add to mongodb