From 1f4de00259cefc9f6c108333f3a794d9fa6c7445 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 22 Apr 2024 13:47:17 +0200 Subject: [PATCH 01/45] support 4.26.0 --- docs/js_data_ch_4.26.0.json | 126 ++++++++++++++++ docs/js_data_ch_4.26.0_generated.json | 126 ++++++++++++++++ ...data_23_01.json => js_data_le_4.26.0.json} | 53 +++---- docs/js_data_le_4.26.0_generated.json | 141 ++++++++++++++++++ models/jsdata_le_pojo.py | 7 +- models/jsdata_pojo.py | 2 +- queue_message/appointmentrequestsender.py | 6 +- workers/captcha_result_getter.py | 4 +- 8 files changed, 427 insertions(+), 38 deletions(-) create mode 100644 docs/js_data_ch_4.26.0.json create mode 100644 docs/js_data_ch_4.26.0_generated.json rename docs/{js_data_23_01.json => js_data_le_4.26.0.json} (72%) create mode 100644 docs/js_data_le_4.26.0_generated.json diff --git a/docs/js_data_ch_4.26.0.json b/docs/js_data_ch_4.26.0.json new file mode 100644 index 0000000..965e61f --- /dev/null +++ b/docs/js_data_ch_4.26.0.json @@ -0,0 +1,126 @@ +{ + "ttst": 120, + "ifov": false, + "hc": 4, + "br_oh": 674, + "br_ow": 393, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 4.8077554980786275, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 674, + "br_w": 393, + "isf": true, + "nddc": 1, + "rs_h": 851, + "rs_w": 393, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.75, + "ars_h": 851, + "ars_w": 393, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "eXyhYz4c,v6GLs9m", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 0.5, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzU2MzEpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hYl0gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODg2NjkpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMzY3", + "ckwa": true, + "glvd": "Google Inc. (Qualcomm)", + "glrd": "ANGLE (Qualcomm, Adreno (TM) 618, OpenGL ES 3.2)", + "wwl": false, + "jset": 1713782722 +} \ No newline at end of file diff --git a/docs/js_data_ch_4.26.0_generated.json b/docs/js_data_ch_4.26.0_generated.json new file mode 100644 index 0000000..16df28f --- /dev/null +++ b/docs/js_data_ch_4.26.0_generated.json @@ -0,0 +1,126 @@ +{ + "ttst": 128, + "ifov": false, + "hc": 4, + "br_oh": 786, + "br_ow": 432, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 9.994786173935164, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 786, + "br_w": 432, + "isf": true, + "nddc": 1, + "rs_h": 984, + "rs_w": 432, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.5, + "ars_h": 984, + "ars_w": 432, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "8DJEKFh,2j4FKFh3", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 4, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzU2MzEpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hYl0gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODg2NjkpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMzY3", + "ckwa": true, + "glvd": "Qualcomm", + "glrd": "Adreno (TM) 619", + "wwl": false, + "jset": 1713783336 +} \ No newline at end of file diff --git a/docs/js_data_23_01.json b/docs/js_data_le_4.26.0.json similarity index 72% rename from docs/js_data_23_01.json rename to docs/js_data_le_4.26.0.json index 03f0064..c28d87a 100644 --- a/docs/js_data_23_01.json +++ b/docs/js_data_le_4.26.0.json @@ -1,31 +1,30 @@ { - "log1": false, - "opts": "endpoint,ajaxListenerPath", - "ttst": 41, + "ttst": 120, "ifov": false, - "hc": 3, - "br_oh": 646, - "br_ow": 360, + "hc": 4, + "br_oh": 674, + "br_ow": 393, "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", "wbd": false, - "tagpu": 3.9694857617239587, + "tagpu": 4.8077554980786275, "wdif": false, "wdifrm": false, "npmtm": false, - "br_h": 646, - "br_w": 360, + "br_h": 674, + "br_w": 393, + "isf": true, "nddc": 1, - "rs_h": 800, - "rs_w": 360, + "rs_h": 851, + "rs_w": 393, "rs_cd": 24, "phe": false, "nm": false, "jsf": false, "lg": "fr-FR", - "pr": 3, - "ars_h": 800, - "ars_w": 360, - "tz": -60, + "pr": 2.75, + "ars_h": 851, + "ars_w": 393, + "tz": -120, "str_ss": true, "str_ls": true, "str_idb": true, @@ -52,7 +51,7 @@ "vnd": "Google Inc.", "bid": "NA", "mmt": "empty", - "plu": "WyZrdOm,EtePPm6d", + "plu": "eXyhYz4c,v6GLs9m", "hdn": false, "awe": false, "geb": false, @@ -93,7 +92,7 @@ "vcqts": false, "vc1": "probably", "vc1ts": true, - "dvm": 8, + "dvm": 0.5, "sqt": false, "so": "portrait-primary", "wdw": true, @@ -117,31 +116,25 @@ "emt": false, "bfr": false, "dbov": false, - "cfpfe": "RXJyb3I6IENhbm5vdCByZWFkIHByb3BlcnRpZXMgb2YgbnVsbCAocmVhZGluZyAndG9TdHJpbmcnKQ==", - "stcfp": "ckFsbCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo3NTIzNykKICAgIGF0IEhUTUxEb2N1bWVudC5yIChodHRwczovL3JlbmRlenZvdXNwYXJpcy5oZXJtZXMuY29tL2Rpc3QvdmVuZG9yLWM2Mjk1ZTljMjg3ZGExNzYwODllLmpzOjI6ODg2Nzkp", + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzU2MzEpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hYl0gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODg2NjkpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMzY3", "ckwa": true, "glvd": "Google Inc. (Qualcomm)", - "glrd": "ANGLE (Qualcomm, Adreno (TM) 650, OpenGL ES 3.2)", + "glrd": "ANGLE (Qualcomm, Adreno (TM) 618, OpenGL ES 3.2)", "wwl": false, - "jset": 1706032747, + "jset": 1713782742, "dcok": ".hermes.com", "log2": true, - "mp_cx": 36, - "mp_cy": 405, - "mp_tr": true, - "mp_mx": 0, - "mp_my": 0, - "mp_sx": 36, - "mp_sy": 495, + "m_fmi": false, "tbce": 0, "es_sigmdn": null, "es_mumdn": null, "es_distmdn": null, "es_angsmdn": null, "es_angemdn": null, - "m_s_c": 8, + "m_s_c": 0, "m_m_c": 4, "m_c_c": 4, "m_cm_r": 1, - "m_ms_r": 0.5 + "m_ms_r": -1 } \ No newline at end of file diff --git a/docs/js_data_le_4.26.0_generated.json b/docs/js_data_le_4.26.0_generated.json new file mode 100644 index 0000000..60f2b1b --- /dev/null +++ b/docs/js_data_le_4.26.0_generated.json @@ -0,0 +1,141 @@ +{ + "ttst": 63, + "ifov": false, + "hc": 4, + "br_oh": 786, + "br_ow": 432, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 9.795249415480846, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 786, + "br_w": 432, + "isf": true, + "nddc": 1, + "rs_h": 984, + "rs_w": 432, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.5, + "ars_h": 984, + "ars_w": 432, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "8DJEKFh,2j4FKFh3", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 4, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzU2MzEpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hYl0gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODg2NjkpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMzY3", + "ckwa": true, + "glvd": "Qualcomm", + "glrd": "Adreno (TM) 619", + "wwl": false, + "jset": 1713783341, + "dcok": ".hermes.com", + "log2": true, + "m_fmi": false, + "tbce": 0, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "m_s_c": 3, + "m_m_c": 5, + "m_c_c": 4, + "m_cm_r": 0.8, + "m_ms_r": 1 +}&eventCounters={"mousemove": 5, "click": 4,"scroll": 3, "touchstart": 5, "touchend":5, "touchmove": 7, "keydown": 8,"keyup": 8 +} \ No newline at end of file diff --git a/models/jsdata_le_pojo.py b/models/jsdata_le_pojo.py index 1f36255..5c33440 100644 --- a/models/jsdata_le_pojo.py +++ b/models/jsdata_le_pojo.py @@ -23,8 +23,8 @@ class JsDataLeTypePojo(JsDataPojo): vnd, pr, ts_mtp, dvm, m_s_c, m_m_c, m_c_c, m_cm_r, m_ms_r): super().__init__(glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu, vnd, pr, ts_mtp, dvm) - self.log2 = True self.dcok = ".hermes.com" + self.log2 = True self.m_fmi = False self.tbce = 0 self.es_sigmdn = None @@ -37,9 +37,8 @@ class JsDataLeTypePojo(JsDataPojo): self.m_c_c = m_c_c self.m_cm_r = m_cm_r self.m_ms_r = m_ms_r - self.cfpfe = "RXJyb3I6IENhbm5vdCByZWFkIHByb3BlcnRpZXMgb2YgbnVsbA==" # 4.25.1 - self.stcfp = "ckFsbCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo3NjQ3MykKICAgIGF0IEhUTUxEb2N1bWVudC5yIChodHRwczovL3JlbmRlenZvdXNwYXJpcy5oZXJtZXMuY29tL2Rpc3QvdmVuZG9yLWM2Mjk1ZTljMjg3ZGExNzYwODllLmpzOjI6ODg2Nzkp" # 4.25.1 - + self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.26.0 + self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzU2MzEpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hYl0gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODg2NjkpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMzY3" # 4.26.0 test_data_json = """{"glvd": "Qualcomm", "glrd": "Adreno (TM) 630", "hc": 2, "br_oh": 663, "br_ow": 384, "br_h": 663, diff --git a/models/jsdata_pojo.py b/models/jsdata_pojo.py index 294dfd0..1070d5c 100644 --- a/models/jsdata_pojo.py +++ b/models/jsdata_pojo.py @@ -266,7 +266,7 @@ class JsDataPojo: self.bfr = False self.dbov = False self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.25.1 - self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzY0NzMpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hYl0gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6OTAxMTgpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUyMjY2" # 4.25.1 + self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzU2MzEpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hYl0gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODg2NjkpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMzY3" # 4.26.0 self.ckwa = True def to_url_encoded_json(self): diff --git a/queue_message/appointmentrequestsender.py b/queue_message/appointmentrequestsender.py index 15e25e5..20557dc 100644 --- a/queue_message/appointmentrequestsender.py +++ b/queue_message/appointmentrequestsender.py @@ -149,7 +149,11 @@ class AppointmentRequestSender(threading.Thread): m_c_c = random.randint(3, 5) # click count m_m_c = random.randint(3, 5) # move count m_cm_r = m_c_c / m_m_c - m_ms_r = random.randint(-1, 1) + m_ms_r = -1 # move scroll ratio + if m_s_c == 0: + m_ms_r = -1 + else: + m_ms_r = m_m_c / m_s_c js_le_data = JsDataLeTypePojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'], diff --git a/workers/captcha_result_getter.py b/workers/captcha_result_getter.py index 7581828..f0464fc 100644 --- a/workers/captcha_result_getter.py +++ b/workers/captcha_result_getter.py @@ -105,7 +105,7 @@ class CaptchaResultGetter: return None def get_ch_raw_data_from_js_data(self, js_data: JsDataPojo, old_valid_cookie) -> str: - _tag_version = "4.25.1" + _tag_version = "4.26.0" _raw_data = "jsData={}&eventCounters=%5B%5D&jsType=ch&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv={}".format( js_data.to_url_encoded_json(), old_valid_cookie, _tag_version) print("raw ch data is " + _raw_data) @@ -172,7 +172,7 @@ class CaptchaResultGetter: # _le_js_raw_data = self.get_le_raw_data_from_js_data(js_le_type_data=js_le_type_data, # old_valid_cookie=old_valid_cookie) _cid = get_datadome_cookies(old_valid_cookie) - _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.25.1".format( + _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.26.0".format( js_le_type_data.to_url_encoded_json(), mousemove_count, click_count, scroll_count, touch_count, touch_count, touch_move, key_count, From 09cd5007da87afd8ae4ba3c3b23a4f9999cbb407 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Wed, 24 Apr 2024 13:48:55 +0200 Subject: [PATCH 02/45] create thread_number and file_path params --- request_sender_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/request_sender_test.py b/request_sender_test.py index 96160d7..9734030 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -69,12 +69,12 @@ def send_appointment_request(message_queue_name, _contact_list): receiver.run() -def start_send_requests(): +def start_send_requests(thread_number=5, file_path='~/Desktop/24_04_to_test.xlsx'): print("start send requests") - contacts_file_path = '~/Desktop/contact_list_merged.xlsx' + contacts_file_path = file_path _contact_list = read_contacts(contacts_file_path) _contact_list_to_book = filter_contacts(_contact_list) - _segment_number = 20 + _segment_number = thread_number logger.info("{} contacts to book".format(len(_contact_list_to_book))) last_thread = None for i in range(0, _segment_number): @@ -88,4 +88,4 @@ def start_send_requests(): if __name__ == '__main__': - start_send_requests() + start_send_requests(thread_number=5, file_path='~/Desktop/24_04_to_test.xlsx') From 0c6d276f8277fb955bd5f290767c958c844fdcf8 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Thu, 25 Apr 2024 20:49:23 +0200 Subject: [PATCH 03/45] use 25 segments --- request_sender.py | 4 ++-- request_sender_test.py | 7 ++----- scheduler_test.py | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/request_sender.py b/request_sender.py index cb36878..14ea7ab 100644 --- a/request_sender.py +++ b/request_sender.py @@ -71,10 +71,10 @@ def send_appointment_request(message_queue_name, _contact_list): def start_send_requests(): print("start send requests") - contacts_file_path = '~/Desktop/contact_list_2024-04-16_2.xlsx' + contacts_file_path = '~/Desktop/contact_list_2024-04-19.xlsx' _contact_list = read_contacts(contacts_file_path) _contact_list_to_book = filter_contacts(_contact_list) - _segment_number = 10 + _segment_number = 1 logger.info("{} contacts to book".format(len(_contact_list_to_book))) last_thread = None for i in range(0, _segment_number): diff --git a/request_sender_test.py b/request_sender_test.py index 9734030..227b894 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -12,9 +12,6 @@ from utiles import is_time_between from utils.AppLogging import init_logger from workers.proxies_constants import MOBILE_PROXY_LIST_FR -IPFIY = 'http://api.ipify.org' -NGROK_TEST = "https://bcc6-193-164-156-53.ngrok-free.app" - def is_already_sent(contact: ContactPojo) -> bool: already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day() @@ -69,7 +66,7 @@ def send_appointment_request(message_queue_name, _contact_list): receiver.run() -def start_send_requests(thread_number=5, file_path='~/Desktop/24_04_to_test.xlsx'): +def start_send_requests(thread_number, file_path): print("start send requests") contacts_file_path = file_path _contact_list = read_contacts(contacts_file_path) @@ -88,4 +85,4 @@ def start_send_requests(thread_number=5, file_path='~/Desktop/24_04_to_test.xlsx if __name__ == '__main__': - start_send_requests(thread_number=5, file_path='~/Desktop/24_04_to_test.xlsx') + start_send_requests(thread_number=25, file_path='~/Desktop/24_04_to_test.xlsx') diff --git a/scheduler_test.py b/scheduler_test.py index 28144d7..47e5c3d 100644 --- a/scheduler_test.py +++ b/scheduler_test.py @@ -5,7 +5,7 @@ from request_sender_test import start_send_requests def start_book_appointment(): - start_send_requests() + start_send_requests(thread_number=25, file_path='~/Desktop/contact_list_2024-04-24.xlsx') def start_check_results_job(sched): From d74904666c7e49a07438ad7a9241a049df3ae434 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 26 Apr 2024 12:27:13 +0200 Subject: [PATCH 04/45] use 25 segments --- queue_message/CookiesPublisher.py | 1 + request_sender_test.py | 7 ++++--- scheduler_test.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/queue_message/CookiesPublisher.py b/queue_message/CookiesPublisher.py index 2dea9fd..5561f7d 100644 --- a/queue_message/CookiesPublisher.py +++ b/queue_message/CookiesPublisher.py @@ -7,6 +7,7 @@ REQUEST_DATA_QUEUE_TEST = 'REQUEST_DATA_TEST' TEST_QUEUE = 'TEST_QUEUE' SHARED_OBJECT = 'SHARED_OBJECT' MORNING_DATA_CACHE = 'MORNING_DATA_CACHE' +MORNING_DATA_CACHE_2 = 'MORNING_DATA_CACHE_2' MORNING_DATA_CACHE_BAK = 'MORNING_DATA_CACHE_BAK' REQUEST_DATA_QUEUE_DE = 'REQUEST_DATA_DE' REQUEST_DATA_OBJECT = 'REQUEST_DATA_OBJECT' diff --git a/request_sender_test.py b/request_sender_test.py index 227b894..824d281 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -6,7 +6,8 @@ from threading import Thread from db.mongo_manager import MONGO_STORE_MANAGER from excel_reader import read_contacts from models.contact_pojo import ContactPojo -from queue_message.CookiesPublisher import CookiesPublisher, SHARED_OBJECT, TEST_QUEUE, MORNING_DATA_CACHE +from queue_message.CookiesPublisher import CookiesPublisher, SHARED_OBJECT, TEST_QUEUE, MORNING_DATA_CACHE, \ + MORNING_DATA_CACHE_2 from queue_message.appointmentrequestsender import AppointmentRequestSender from utiles import is_time_between from utils.AppLogging import init_logger @@ -78,11 +79,11 @@ def start_send_requests(thread_number, file_path): logger.info("segment is {}".format(i)) _step = int(len(_contact_list_to_book) / _segment_number) _sublist = _contact_list_to_book[i * _step:_step * (i + 1)] - _thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE, _sublist)) + _thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE_2, _sublist)) last_thread = _thread1 _thread1.start() last_thread.join() if __name__ == '__main__': - start_send_requests(thread_number=25, file_path='~/Desktop/24_04_to_test.xlsx') + start_send_requests(thread_number=25, file_path='~/Desktop/contact_list_2024-04-17.xlsx') diff --git a/scheduler_test.py b/scheduler_test.py index 47e5c3d..c2c3ede 100644 --- a/scheduler_test.py +++ b/scheduler_test.py @@ -5,7 +5,7 @@ from request_sender_test import start_send_requests def start_book_appointment(): - start_send_requests(thread_number=25, file_path='~/Desktop/contact_list_2024-04-24.xlsx') + start_send_requests(thread_number=25, file_path='~/Desktop/contact_list_2024-04-25_2.xlsx') def start_check_results_job(sched): From b737058997fde855484d5154d34623a83887f276 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Tue, 30 Apr 2024 19:42:09 +0200 Subject: [PATCH 05/45] quit programme after all messages processed --- workers/MessagerTransporter.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/workers/MessagerTransporter.py b/workers/MessagerTransporter.py index 7f4624c..b824136 100644 --- a/workers/MessagerTransporter.py +++ b/workers/MessagerTransporter.py @@ -2,7 +2,7 @@ import threading import pika -from queue_message.CookiesPublisher import QUEUE_HOST, CookiesPublisher, MORNING_DATA_CACHE +from queue_message.CookiesPublisher import QUEUE_HOST, CookiesPublisher, MORNING_DATA_CACHE, MORNING_DATA_CACHE_2 credentials = pika.PlainCredentials('appointment', 'ZyuhJZ2xEYWhElhpJjy7YEpZGZwNYJz2fHIu') @@ -24,6 +24,9 @@ class MessageTransporter(threading.Thread): self.channel.basic_consume(queue=self.queue_to_listen, auto_ack=False, on_message_callback=self.on_message) self.channel.start_consuming() + def message_count(self): + return self.channel.queue_declare(queue=self.queue_to_listen, durable=True).method.message_count + def on_message(self, ch, method, properties, body): print(f" [x] Received {body}") _received_object = body.decode("UTF-8") @@ -31,6 +34,9 @@ class MessageTransporter(threading.Thread): print("message count in queue is {}".format(_message_in_queue_count)) self.cookiesPublisher.publish_body(_received_object) ch.basic_ack(delivery_tag=method.delivery_tag) + if self.message_count() == 0: + print("all messages are processed") + exit(0) if __name__ == '__main__': @@ -38,4 +44,4 @@ if __name__ == '__main__': cookiesPublisher.set_up_connection() message_transporter = MessageTransporter(cookiesPublisher=cookiesPublisher, queue_to_listen=MORNING_DATA_CACHE) message_transporter.set_up_connection() - message_transporter.listen_to_queue() \ No newline at end of file + message_transporter.listen_to_queue() From 5db5aadb12ddd11f61ac9c30a7449bed2c68c06d Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Thu, 2 May 2024 17:32:18 +0200 Subject: [PATCH 06/45] try to programme message migration --- migrate_cached_request_data.py | 6 ++++++ workers/MessagerTransporter.py | 15 ++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 migrate_cached_request_data.py diff --git a/migrate_cached_request_data.py b/migrate_cached_request_data.py new file mode 100644 index 0000000..5870aff --- /dev/null +++ b/migrate_cached_request_data.py @@ -0,0 +1,6 @@ +from queue_message.CookiesPublisher import MORNING_DATA_CACHE_2, MORNING_DATA_CACHE +from workers.MessagerTransporter import migrate_message_to_queue + +if __name__ == '__main__': + migrate_message_to_queue(from_queue=MORNING_DATA_CACHE_2) + migrate_message_to_queue(from_queue=MORNING_DATA_CACHE) diff --git a/workers/MessagerTransporter.py b/workers/MessagerTransporter.py index b824136..8292136 100644 --- a/workers/MessagerTransporter.py +++ b/workers/MessagerTransporter.py @@ -39,9 +39,18 @@ class MessageTransporter(threading.Thread): exit(0) -if __name__ == '__main__': - cookiesPublisher = CookiesPublisher(queue_name="MORNING_DATA_CACHE_BAK") +def migrate_message_to_queue(from_queue, to_queue="MORNING_DATA_CACHE_BAK"): + cookiesPublisher = CookiesPublisher(queue_name=to_queue) cookiesPublisher.set_up_connection() - message_transporter = MessageTransporter(cookiesPublisher=cookiesPublisher, queue_to_listen=MORNING_DATA_CACHE) + message_transporter = MessageTransporter(cookiesPublisher=cookiesPublisher, queue_to_listen=from_queue) message_transporter.set_up_connection() message_transporter.listen_to_queue() + + +if __name__ == '__main__': + migrate_message_to_queue(from_queue=MORNING_DATA_CACHE_2) + # cookiesPublisher = CookiesPublisher(queue_name="MORNING_DATA_CACHE_BAK") + # cookiesPublisher.set_up_connection() + # message_transporter = MessageTransporter(cookiesPublisher=cookiesPublisher, queue_to_listen=MORNING_DATA_CACHE) + # message_transporter.set_up_connection() + # message_transporter.listen_to_queue() From 95db3be6378837788cbd0b76eee1a49c149b7315 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Thu, 2 May 2024 22:26:45 +0200 Subject: [PATCH 07/45] use separate process to migrate messages --- migrate_cached_request_data.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/migrate_cached_request_data.py b/migrate_cached_request_data.py index 5870aff..5882901 100644 --- a/migrate_cached_request_data.py +++ b/migrate_cached_request_data.py @@ -1,6 +1,13 @@ -from queue_message.CookiesPublisher import MORNING_DATA_CACHE_2, MORNING_DATA_CACHE +from multiprocessing import Process + +from queue_message.CookiesPublisher import MORNING_DATA_CACHE_2, MORNING_DATA_CACHE, MORNING_DATA_CACHE_BAK from workers.MessagerTransporter import migrate_message_to_queue if __name__ == '__main__': - migrate_message_to_queue(from_queue=MORNING_DATA_CACHE_2) - migrate_message_to_queue(from_queue=MORNING_DATA_CACHE) + p1 = Process(target=migrate_message_to_queue, args=(MORNING_DATA_CACHE_2, MORNING_DATA_CACHE_BAK)) + p1.start() + p2 = Process(target=migrate_message_to_queue, args=(MORNING_DATA_CACHE, MORNING_DATA_CACHE_BAK)) + p2.start() + p2.join() + # migrate_message_to_queue(from_queue=MORNING_DATA_CACHE_2) + # migrate_message_to_queue(from_queue=MORNING_DATA_CACHE) From bb1a01e2041e5336e161afe71603df7f06d43db8 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 3 May 2024 16:12:19 +0200 Subject: [PATCH 08/45] set default queue to MORNING_DATA_CACHE --- request_sender_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request_sender_test.py b/request_sender_test.py index 824d281..05d19a1 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -79,7 +79,7 @@ def start_send_requests(thread_number, file_path): logger.info("segment is {}".format(i)) _step = int(len(_contact_list_to_book) / _segment_number) _sublist = _contact_list_to_book[i * _step:_step * (i + 1)] - _thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE_2, _sublist)) + _thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE, _sublist)) last_thread = _thread1 _thread1.start() last_thread.join() From b9f390dc8349f3c88778a3f28bdca3bc6d73c7d3 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Sat, 4 May 2024 14:50:26 +0200 Subject: [PATCH 09/45] can use multi thread to validate links --- db/mongo_manager.py | 2 +- queue_message/CookiesPublisher.py | 1 - scheduler_test.py | 4 +- workers/link_validator_with_provided_list.py | 224 +++++++++++++++++++ 4 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 workers/link_validator_with_provided_list.py diff --git a/db/mongo_manager.py b/db/mongo_manager.py index 03fc204..229d407 100755 --- a/db/mongo_manager.py +++ b/db/mongo_manager.py @@ -137,7 +137,7 @@ class MongoDbManager: collection = self.db[collection_name] validated_at = time.strftime("%H:%M:%S", time.localtime()) - validated_by = "requests" + str(segement_position) + validated_by = "requests:" + str(segement_position) if is_invalid: validated_by = "Invalid" if is_duplicated: diff --git a/queue_message/CookiesPublisher.py b/queue_message/CookiesPublisher.py index 5561f7d..99295bb 100644 --- a/queue_message/CookiesPublisher.py +++ b/queue_message/CookiesPublisher.py @@ -37,4 +37,3 @@ class CookiesPublisher: def message_count(self): return self.channel.queue_declare(queue=self.to_queue, durable=True).method.message_count - # return self.queue_method.method.message_count diff --git a/scheduler_test.py b/scheduler_test.py index c2c3ede..2ae5022 100644 --- a/scheduler_test.py +++ b/scheduler_test.py @@ -5,14 +5,14 @@ from request_sender_test import start_send_requests def start_book_appointment(): - start_send_requests(thread_number=25, file_path='~/Desktop/contact_list_2024-04-25_2.xlsx') + start_send_requests(thread_number=25, file_path='~/Desktop/contact_list_2024-05-04.xlsx') def start_check_results_job(sched): sched.add_job(start_book_appointment, 'cron', day_of_week='mon-sat', hour='10', minute='30', misfire_grace_time=10, - second='0', timezone='Europe/Paris', max_instances=1, args=[]) + second='20', timezone='Europe/Paris', max_instances=1, args=[]) def config_and_start_jobs(): diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py new file mode 100644 index 0000000..63cbbb6 --- /dev/null +++ b/workers/link_validator_with_provided_list.py @@ -0,0 +1,224 @@ +import json +import logging +import random +import threading +import time +from http.cookies import SimpleCookie + +import pika +import requests + +from db.mongo_manager import MONGO_STORE_MANAGER +from models.LinkPojo import LinkPojo +from models.result_pojo import RequestResult +from proxy_manager.proxy_manager import ProxyManager +from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST, TEST_QUEUE, SHARED_OBJECT +from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials +from utils.AppLogging import init_logger +from workers.proxies_constants import PROXY_LIST_FR + +DOUBLE_MESSAGE = "Une demande de rendez-vous a déjà été enregistrée avec ces coordonnées" +INVALID = "Depuis plus de 130 ans," + +init_logger() +logger = logging.getLogger() + + +class LinkValidatorWithProvidedList(threading.Thread): + + def __init__(self, cookiesPublisher: CookiesPublisher, link_list: list, + queue_to_listen=REQUEST_DATA_QUEUE, + ip_country="FR", + limit=40): + super().__init__() + self.link_to_validate_list = link_list + self.cookie = SimpleCookie() + self.cookiesPublisher = cookiesPublisher + self.queue_to_listen = queue_to_listen + self.ip_country = ip_country + self.filter_with_ip_country() + self.proxy_manager = ProxyManager() + self.limit = limit + + def set_up_connection(self): + self.connection = pika.BlockingConnection( + pika.ConnectionParameters(host=QUEUE_HOST, port=5672, credentials=credentials)) + self.channel = self.connection.channel() + + def listen_to_queue(self, callback): + self.channel.basic_qos(prefetch_count=1) + self.channel.basic_consume(queue=self.queue_to_listen, auto_ack=False, on_message_callback=callback) + self.channel.start_consuming() + + def send_request(self, linkPojo: LinkPojo, _received_dict=None) -> RequestResult: + _ua = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36' + if _received_dict is not None: + _ua = _received_dict['ua'] + self.cookie.load(self.cookie_str) + headers = { + 'User-Agent': _ua, + 'Accept': '*/*', + 'Accept-Encoding': 'gzip, deflate, br', + 'Cache-Control': 'max-age=0', + 'Referer': linkPojo.url, + 'Cookie': self.cookie_str, + 'Sec-Fetch-Mode': 'navigate', + 'Host': 'rendezvousparis.hermes.com', + 'Sec-Fetch-Site': 'same-origin', + 'Sec-Fetch-Dest': 'document', + 'Accept-Language': 'fr-FR,fr;q=0.6'} + _proxy_to_use = random.choice(self.proxy_manager.get_link_validate_proxy(self.link_to_validate_list)) + print(_proxy_to_use) + print("received cookie is " + str(self.cookie_str)) + print("send request for link: " + linkPojo.url) + try: + print("will send request with ua {}".format(_ua)) + print("will send request with cookie {}".format(self.cookie_str)) + response = requests.get(url=linkPojo.url, headers=headers, verify=False, proxies=self.proxy_to_use, + timeout=60) + print(response.status_code) + if response.status_code == 200: + _content = response.text + print(response.text) + if "Votre demande de rendez-vous Maroquinerie a bien été enregistrée" in _content: + print(response.url) + MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, + segement_position=threading.currentThread().name) + # set new cookies + _cookies_to_set = response.headers['set-cookie'] + self.cookie.load(_cookies_to_set) + new_cookies = {k: v.value for k, v in self.cookie.items()} + new_coolies_str = "" + for key in new_cookies: + new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";" + print("will publish to queue {}".format(new_coolies_str)) + # upload the cookie to queue + if _received_dict is not None: + _received_dict['cookiesStr'] = new_coolies_str + print("body in json:{}".format(json.dumps(_received_dict))) + self.cookiesPublisher.publish_body(json.dumps(_received_dict)) + else: + self.cookiesPublisher.publish_body(new_coolies_str) + self.cookie_str = new_coolies_str + return RequestResult.SUCCESS + elif INVALID in _content: + + MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_invalid=True) + # set new cookies + _cookies_to_set = response.headers['set-cookie'] + self.cookie.load(_cookies_to_set) + new_cookies = {k: v.value for k, v in self.cookie.items()} + new_coolies_str = "" + for key in new_cookies: + new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";" + print("will publish to queue {}".format(new_coolies_str)) + # upload the cookie to queue + self.cookiesPublisher.publish_body(new_coolies_str) + self.cookie_str = new_coolies_str + return RequestResult.SUCCESS + + elif DOUBLE_MESSAGE in _content: + print(response.url) + MONGO_STORE_MANAGER.link_validated_for_result(response.url, linkPojo, is_duplicated=True) + # set new cookies + _cookies_to_set = response.headers['set-cookie'] + self.cookie.load(_cookies_to_set) + new_cookies = {k: v.value for k, v in self.cookie.items()} + new_coolies_str = "" + for key in new_cookies: + new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";" + print("will publish to queue {}".format(new_coolies_str)) + # upload the cookie to queue + self.cookiesPublisher.publish_body(new_coolies_str) + self.cookie_str = new_coolies_str + return RequestResult.SUCCESS + else: + return RequestResult.UNKNOWN + elif response.status_code == 502: + return RequestResult.BAD_GATEWAY + else: + return RequestResult.BLOCKED + except Exception as error: + print(error) + return RequestResult.PROXY_ERROR + + def on_message(self, ch, method, properties, body): + print(f" [x] Received {body}") + _message_in_queue_count = self.cookiesPublisher.message_count() + print("message count in queue is {}".format(_message_in_queue_count)) + _received_object = body.decode("UTF-8") + _received_dict = None + if "glrd" in _received_object: + _received_dict = json.loads(_received_object) + _received_cookies = _received_dict["cookiesStr"] + else: + _received_cookies = _received_object + self.cookie_str = _received_cookies + random.shuffle(self.link_to_validate_list) + if len(self.link_to_validate_list) > 0 and _message_in_queue_count >= self.limit: + print("{}:links number is {}".format(threading.currentThread().name, len(self.link_to_validate_list))) + can_continue = None + for link_to_validate in self.link_to_validate_list: + print(link_to_validate) + self.proxy_to_use = random.choice(PROXY_LIST_FR) + can_continue = self.send_request(link_to_validate, _received_dict) + # remove the tested link from link list + self.link_to_validate_list.remove(link_to_validate) + if can_continue == RequestResult.BLOCKED: + print("cannot continue, blocked, then skip") + break + else: + if can_continue == RequestResult.BAD_GATEWAY: + time.sleep(30) + break + time.sleep(random.randint(2, 5)) + print("can continue, continue") + if can_continue == RequestResult.BAD_GATEWAY or can_continue == RequestResult.PROXY_ERROR or can_continue == RequestResult.SUCCESS: + print("will requeue the message") + ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True) + else: + print("will ack") + ch.basic_ack(delivery_tag=method.delivery_tag) + else: + print("empty list, no need to ack") + time.sleep(5) + ch.basic_reject(delivery_tag=method.delivery_tag, requeue=True) + self.channel.stop_consuming() + + def filter_with_ip_country(self): + _link_list_to_click = [] + for _link in self.link_to_validate_list: + if _link.ip_country == self.ip_country: + _link_list_to_click.append(_link) + self.link_to_validate_list = _link_list_to_click + + +def validate_links(cookiesPublisher, queue_name: str, link_list: list): + receiver = LinkValidatorWithProvidedList(cookiesPublisher=cookiesPublisher, link_list=link_list, + queue_to_listen=queue_name, ip_country="FR", limit=0) + print("{} set_up_connection".format(threading.currentThread().name)) + receiver.set_up_connection() + receiver.listen_to_queue(receiver.on_message) + + +def validate_all_links(): + all_link_list = MONGO_STORE_MANAGER.get_links_to_validate() + _queue_name = TEST_QUEUE + _segment_number = 4 + last_thread = None + for i in range(0, _segment_number): + logger.info("{}:{} links to validate".format(threading.currentThread().name, len(all_link_list))) + logger.info("segment is {}".format(i)) + _cookiesPublisher = CookiesPublisher(queue_name=TEST_QUEUE) + _cookiesPublisher.set_up_connection() + _step = int(len(all_link_list) / _segment_number) + _sublist = all_link_list[i * _step:_step * (i + 1)] + _thread1 = threading.Thread(target=validate_links, args=(_cookiesPublisher, SHARED_OBJECT, _sublist)) + last_thread = _thread1 + _thread1.start() + last_thread.join() + + +if __name__ == '__main__': + # generate test data + validate_all_links() From 9400aee3dce3f1c5d2c745c2c0686ae594424ba5 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 6 May 2024 11:55:09 +0200 Subject: [PATCH 10/45] support 4.26.1 --- docs/4_26_1/ch_jsdata_generated.json | 126 +++++++++++++++++++++++++++ docs/4_26_1/ch_jsdata_original.json | 126 +++++++++++++++++++++++++++ models/jsdata_le_pojo.py | 4 +- models/jsdata_pojo.py | 4 +- workers/captcha_result_getter.py | 4 +- 5 files changed, 258 insertions(+), 6 deletions(-) create mode 100644 docs/4_26_1/ch_jsdata_generated.json create mode 100644 docs/4_26_1/ch_jsdata_original.json diff --git a/docs/4_26_1/ch_jsdata_generated.json b/docs/4_26_1/ch_jsdata_generated.json new file mode 100644 index 0000000..02e640d --- /dev/null +++ b/docs/4_26_1/ch_jsdata_generated.json @@ -0,0 +1,126 @@ +{ + "ttst": 104, + "ifov": false, + "hc": 8, + "br_oh": 786, + "br_ow": 432, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 2.637976599008111, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 786, + "br_w": 432, + "isf": true, + "nddc": 1, + "rs_h": 984, + "rs_w": 432, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.5, + "ars_h": 984, + "ars_w": 432, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "my4cWTwB,8Ls15FK", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 0.5, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyMDcpXSgnXHg2Mlx4NzJceDZmXHg3N1x4NzNceDY1XHg3Mlx4NjZceDZjXHg2Zlx4NzdceDJkXHg2M1x4NmZceDZlXHg3NFx4NjFceDY5XHg2ZVx4NjVceDcyJyk7aWYobyl7IWZ1bmN0aW9uIHQoKXt2YXIg", + "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzUwNTIpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWIgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODkyMjMpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMTE0", + "ckwa": true, + "glvd": "Qualcomm", + "glrd": "Adreno (TM) 619", + "wwl": false, + "jset": 1714989181 +} \ No newline at end of file diff --git a/docs/4_26_1/ch_jsdata_original.json b/docs/4_26_1/ch_jsdata_original.json new file mode 100644 index 0000000..dcfa7f6 --- /dev/null +++ b/docs/4_26_1/ch_jsdata_original.json @@ -0,0 +1,126 @@ +{ + "ttst": 51, + "ifov": false, + "hc": 8, + "br_oh": 684, + "br_ow": 393, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 2.5476085485280664, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 684, + "br_w": 393, + "isf": true, + "nddc": 1, + "rs_h": 873, + "rs_w": 393, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.75, + "ars_h": 873, + "ars_w": 393, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": false, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "a05FKFh,Rnb0Do7G", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 1, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyMDcpXSgnXHg2Mlx4NzJceDZmXHg3N1x4NzNceDY1XHg3Mlx4NjZceDZjXHg2Zlx4NzdceDJkXHg2M1x4NmZceDZlXHg3NFx4NjFceDY5XHg2ZVx4NjVceDcyJyk7aWYobyl7IWZ1bmN0aW9uIHQoKXt2YXIg", + "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzUwNTIpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWIgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODkyMjMpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMTE0", + "ckwa": true, + "glvd": "Google Inc. (ARM)", + "glrd": "ANGLE (ARM, Mali-G57 MC2, OpenGL ES 3.2)", + "wwl": false, + "jset": 1714988585 +} \ No newline at end of file diff --git a/models/jsdata_le_pojo.py b/models/jsdata_le_pojo.py index 5c33440..b139f18 100644 --- a/models/jsdata_le_pojo.py +++ b/models/jsdata_le_pojo.py @@ -37,8 +37,8 @@ class JsDataLeTypePojo(JsDataPojo): self.m_c_c = m_c_c self.m_cm_r = m_cm_r self.m_ms_r = m_ms_r - self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.26.0 - self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzU2MzEpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hYl0gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODg2NjkpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMzY3" # 4.26.0 + self.cfpfe = "RXJyb3I6IENhbm5vdCByZWFkIHByb3BlcnRpZXMgb2YgbnVsbA==" # 4.26.1 + self.stcfp = "ckFsbCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo3NTA1MikKICAgIGF0IEhUTUxEb2N1bWVudC5yIChodHRwczovL3JlbmRlenZvdXNwYXJpcy5oZXJtZXMuY29tL2Rpc3QvdmVuZG9yLWM2Mjk1ZTljMjg3ZGExNzYwODllLmpzOjI6ODg2Nzkp" # 4.26.1 test_data_json = """{"glvd": "Qualcomm", "glrd": "Adreno (TM) 630", "hc": 2, "br_oh": 663, "br_ow": 384, "br_h": 663, diff --git a/models/jsdata_pojo.py b/models/jsdata_pojo.py index 1070d5c..05b802a 100644 --- a/models/jsdata_pojo.py +++ b/models/jsdata_pojo.py @@ -265,8 +265,8 @@ class JsDataPojo: self.emt = False self.bfr = False self.dbov = False - self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.25.1 - self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzU2MzEpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hYl0gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODg2NjkpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMzY3" # 4.26.0 + self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyMDcpXSgnXHg2Mlx4NzJceDZmXHg3N1x4NzNceDY1XHg3Mlx4NjZceDZjXHg2Zlx4NzdceDJkXHg2M1x4NmZceDZlXHg3NFx4NjFceDY5XHg2ZVx4NjVceDcyJyk7aWYobyl7IWZ1bmN0aW9uIHQoKXt2YXIg" # 4.26.1 + self.stcfp = "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzUwNTIpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWIgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODkyMjMpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMTE0" # 4.26.1 self.ckwa = True def to_url_encoded_json(self): diff --git a/workers/captcha_result_getter.py b/workers/captcha_result_getter.py index f0464fc..0de996c 100644 --- a/workers/captcha_result_getter.py +++ b/workers/captcha_result_getter.py @@ -105,7 +105,7 @@ class CaptchaResultGetter: return None def get_ch_raw_data_from_js_data(self, js_data: JsDataPojo, old_valid_cookie) -> str: - _tag_version = "4.26.0" + _tag_version = "4.26.1" _raw_data = "jsData={}&eventCounters=%5B%5D&jsType=ch&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv={}".format( js_data.to_url_encoded_json(), old_valid_cookie, _tag_version) print("raw ch data is " + _raw_data) @@ -172,7 +172,7 @@ class CaptchaResultGetter: # _le_js_raw_data = self.get_le_raw_data_from_js_data(js_le_type_data=js_le_type_data, # old_valid_cookie=old_valid_cookie) _cid = get_datadome_cookies(old_valid_cookie) - _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.26.0".format( + _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.26.1".format( js_le_type_data.to_url_encoded_json(), mousemove_count, click_count, scroll_count, touch_count, touch_count, touch_move, key_count, From a5e3366de678c7ed22771536a00515d8be13e683 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 6 May 2024 18:26:00 +0200 Subject: [PATCH 11/45] support 4.27.0 --- docs/4_27_0/ch_jsdata_generated.json | 126 ++++++++++++++++ docs/4_27_0/ch_jsdata_original.json | 127 ++++++++++++++++ docs/4_27_0/ch_jsdata_original_pixel8.json | 126 ++++++++++++++++ docs/4_27_0/ch_raw_data_generated | 1 + docs/4_27_0/ch_raw_data_original | 1 + docs/4_27_0/le_jsdata_original.json | 148 +++++++++++++++++++ docs/4_27_0/le_jsdata_original_pixel8.json | 141 ++++++++++++++++++ models/jsdata_le_pojo.py | 4 +- models/jsdata_pojo.py | 4 +- workers/captcha_result_getter.py | 4 +- workers/link_validator_with_provided_list.py | 10 +- 11 files changed, 682 insertions(+), 10 deletions(-) create mode 100644 docs/4_27_0/ch_jsdata_generated.json create mode 100644 docs/4_27_0/ch_jsdata_original.json create mode 100644 docs/4_27_0/ch_jsdata_original_pixel8.json create mode 100644 docs/4_27_0/ch_raw_data_generated create mode 100644 docs/4_27_0/ch_raw_data_original create mode 100644 docs/4_27_0/le_jsdata_original.json create mode 100644 docs/4_27_0/le_jsdata_original_pixel8.json diff --git a/docs/4_27_0/ch_jsdata_generated.json b/docs/4_27_0/ch_jsdata_generated.json new file mode 100644 index 0000000..e9c5457 --- /dev/null +++ b/docs/4_27_0/ch_jsdata_generated.json @@ -0,0 +1,126 @@ +{ + "ttst": 43, + "ifov": false, + "hc": 4, + "br_oh": 663, + "br_ow": 384, + "ua": "Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 5.167040384311697, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 663, + "br_w": 384, + "isf": true, + "nddc": 1, + "rs_h": 811, + "rs_w": 384, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.8125, + "ars_h": 811, + "ars_w": 384, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "c1iRnTJ,YUxg36GL", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 4, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyNjApXShuKDI5NCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1tpKDI2MyldW2koMjYwKV0oJ1x4NjJceDcyXHg2Zlx4NzdceDczXHg2NVx4NzJceDY2XHg2Y1x4NmZceDc3XHgy", + "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzNjQpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODApCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxOTI4", + "ckwa": true, + "glvd": "Qualcomm", + "glrd": "Adreno (TM) 630", + "wwl": false, + "jset": 1715011727 +} \ No newline at end of file diff --git a/docs/4_27_0/ch_jsdata_original.json b/docs/4_27_0/ch_jsdata_original.json new file mode 100644 index 0000000..6ddd623 --- /dev/null +++ b/docs/4_27_0/ch_jsdata_original.json @@ -0,0 +1,127 @@ +{ + "ttst": 59, + "ifov": false, + "hc": 7, + "br_oh": 684, + "br_ow": 393, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 4.775199583703946, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 684, + "br_w": 393, + "isf": true, + "nddc": 1, + "rs_h": 873, + "rs_w": 393, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.75, + "ars_h": 873, + "ars_w": 393, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": false, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "8DBn6dt1,WTwBAAA", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 2, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyNjApXShuKDI5NCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1tpKDI2MyldW2koMjYwKV0oJ1x4NjJceDcyXHg2Zlx4NzdceDczXHg2NVx4NzJceDY2XHg2Y1x4NmZceDc3XHgy", + "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzNjQpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODApCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxOTI4", + "ckwa": true, + "emd": "k:ai,vi,ao", + "glvd": "Google Inc. (ARM)", + "glrd": "ANGLE (ARM, Mali-G57 MC2, OpenGL ES 3.2)", + "wwl": false, + "jset": 1715011187 +} \ No newline at end of file diff --git a/docs/4_27_0/ch_jsdata_original_pixel8.json b/docs/4_27_0/ch_jsdata_original_pixel8.json new file mode 100644 index 0000000..88c9ba1 --- /dev/null +++ b/docs/4_27_0/ch_jsdata_original_pixel8.json @@ -0,0 +1,126 @@ +{ + "ttst": 21.199999999953434, + "ifov": false, + "hc": 9, + "br_oh": 784, + "br_ow": 412, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 5.393727248230813, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 784, + "br_w": 412, + "isf": true, + "nddc": 1, + "rs_h": 915, + "rs_w": 412, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.625, + "ars_h": 915, + "ars_w": 412, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": false, + "plgod": false, + "plg": 0, + "plgne": "NA", + "plgre": "NA", + "plgof": "NA", + "plggt": "NA", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "empty", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 8, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyNjApXShuKDI5NCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1tpKDI2MyldW2koMjYwKV0oJ1x4NjJceDcyXHg2Zlx4NzdceDczXHg2NVx4NzJceDY2XHg2Y1x4NmZceDc3XHgy", + "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzNjQpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODApCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxOTI4", + "ckwa": true, + "glvd": "Google Inc. (ARM)", + "glrd": "ANGLE (ARM, Mali-G715, OpenGL ES 3.2)", + "wwl": false, + "jset": 1715012381 +} \ No newline at end of file diff --git a/docs/4_27_0/ch_raw_data_generated b/docs/4_27_0/ch_raw_data_generated new file mode 100644 index 0000000..ea1c9b7 --- /dev/null +++ b/docs/4_27_0/ch_raw_data_generated @@ -0,0 +1 @@ +jsData={"ttst":43,"ifov":false,"hc":4,"br_oh":663,"br_ow":384,"ua":"Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36","wbd":false,"tagpu":5.167040384311697,"wdif":false,"wdifrm":false,"npmtm":false,"br_h":663,"br_w":384,"isf":true,"nddc":1,"rs_h":811,"rs_w":384,"rs_cd":24,"phe":false,"nm":false,"jsf":false,"lg":"fr-FR","pr":2.8125,"ars_h":811,"ars_w":384,"tz":-120,"str_ss":true,"str_ls":true,"str_idb":true,"str_odb":true,"plgod":false,"plg":2,"plgne":"err","plgre":"err","plgof":"err","plggt":"err","pltod":false,"hcovdr":false,"hcovdr2":false,"plovdr":false,"plovdr2":false,"ftsovdr":false,"ftsovdr2":false,"lb":false,"eva":33,"lo":false,"ts_mtp":5,"ts_tec":true,"ts_tsa":true,"vnd":"Google Inc.","bid":"NA","mmt":"empty","plu":"c1iRnTJ,YUxg36GL","hdn":false,"awe":false,"geb":false,"dat":false,"med":"defined","aco":"probably","acots":false,"acmp":"probably","acmpts":true,"acw":"probably","acwts":false,"acma":"maybe","acmats":false,"acaa":"probably","acaats":true,"ac3":"","ac3ts":false,"acf":"probably","acfts":false,"acmp4":"maybe","acmp4ts":false,"acmp3":"probably","acmp3ts":false,"acwm":"maybe","acwmts":false,"ocpt":false,"vco":"","vcots":false,"vch":"probably","vchts":true,"vcw":"probably","vcwts":true,"vc3":"maybe","vc3ts":false,"vcmp":"","vcmpts":false,"vcq":"","vcqts":false,"vc1":"probably","vc1ts":true,"dvm":4,"sqt":false,"so":"portrait-primary","wdw":true,"cokys":"bG9hZFRpbWVzY3NpL=","ecpc":false,"lgs":true,"lgsod":false,"psn":true,"edp":false,"addt":true,"wsdc":true,"ccsr":true,"nuad":true,"bcda":true,"idn":true,"capi":false,"svde":false,"vpbq":true,"ucdv":false,"spwn":false,"emt":false,"bfr":false,"dbov":false,"cfpfe":"ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyNjApXShuKDI5NCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1tpKDI2MyldW2koMjYwKV0oJ1x4NjJceDcyXHg2Zlx4NzdceDczXHg2NVx4NzJceDY2XHg2Y1x4NmZceDc3XHgy","stcfp":"Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzNjQpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODApCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxOTI4","ckwa":true,"glvd":"Qualcomm","glrd":"Adreno (TM) 630","wwl":false,"jset":1715011727}&eventCounters=[]&jsType=ch&cid=Z6f9p4ZRzW1JXIdlNOcBUngqO~MVGN26rwebFQS82MfPxZWKdUeKm0ze2vmyNolXb5EaDpAUe~sBFPwOq3voW2F4YHmFNZTK0IMZfwWPvy4V7zWwyINGYTVueb5ghyni&ddk=789361B674144528D0B7EE76B35826&Referer=https%3A%2F%2Frendezvousparis.hermes.com%2Fclient%2Fregister&request=%2Fclient%2Fregister&responsePage=origin&ddv=4.27.0 \ No newline at end of file diff --git a/docs/4_27_0/ch_raw_data_original b/docs/4_27_0/ch_raw_data_original new file mode 100644 index 0000000..75971dc --- /dev/null +++ b/docs/4_27_0/ch_raw_data_original @@ -0,0 +1 @@ +jsData={"ttst":59,"ifov":false,"hc":7,"br_oh":684,"br_ow":393,"ua":"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36","wbd":false,"tagpu":4.775199583703946,"wdif":false,"wdifrm":false,"npmtm":false,"br_h":684,"br_w":393,"isf":true,"nddc":1,"rs_h":873,"rs_w":393,"rs_cd":24,"phe":false,"nm":false,"jsf":false,"lg":"fr-FR","pr":2.75,"ars_h":873,"ars_w":393,"tz":-120,"str_ss":true,"str_ls":true,"str_idb":true,"str_odb":false,"plgod":false,"plg":2,"plgne":"err","plgre":"err","plgof":"err","plggt":"err","pltod":false,"hcovdr":false,"hcovdr2":false,"plovdr":false,"plovdr2":false,"ftsovdr":false,"ftsovdr2":false,"lb":false,"eva":33,"lo":false,"ts_mtp":5,"ts_tec":true,"ts_tsa":true,"vnd":"Google Inc.","bid":"NA","mmt":"empty","plu":"8DBn6dt1,WTwBAAA","hdn":false,"awe":false,"geb":false,"dat":false,"med":"defined","aco":"probably","acots":false,"acmp":"probably","acmpts":true,"acw":"probably","acwts":false,"acma":"maybe","acmats":false,"acaa":"probably","acaats":true,"ac3":"","ac3ts":false,"acf":"probably","acfts":false,"acmp4":"maybe","acmp4ts":false,"acmp3":"probably","acmp3ts":false,"acwm":"maybe","acwmts":false,"ocpt":false,"vco":"","vcots":false,"vch":"probably","vchts":true,"vcw":"probably","vcwts":true,"vc3":"maybe","vc3ts":false,"vcmp":"","vcmpts":false,"vcq":"","vcqts":false,"vc1":"probably","vc1ts":true,"dvm":2,"sqt":false,"so":"portrait-primary","wdw":true,"cokys":"bG9hZFRpbWVzY3NpL=","ecpc":false,"lgs":true,"lgsod":false,"psn":true,"edp":false,"addt":true,"wsdc":true,"ccsr":true,"nuad":true,"bcda":true,"idn":true,"capi":false,"svde":false,"vpbq":true,"ucdv":false,"spwn":false,"emt":false,"bfr":false,"dbov":false,"cfpfe":"ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyNjApXShuKDI5NCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1tpKDI2MyldW2koMjYwKV0oJ1x4NjJceDcyXHg2Zlx4NzdceDczXHg2NVx4NzJceDY2XHg2Y1x4NmZceDc3XHgy","stcfp":"Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzNjQpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODApCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxOTI4","ckwa":true,"emd":"k:ai,vi,ao","glvd":"Google Inc. (ARM)","glrd":"ANGLE (ARM, Mali-G57 MC2, OpenGL ES 3.2)","wwl":false,"jset":1715011187}&eventCounters=[]&jsType=ch&cid=Dz3C75xS0iEMyb2RyRxXvWjC5yKk6O8XLyHJLyMIMXtSJxOO~tzCcX3IvG_QNoI9HFxa~bFPA8YUebk9NxyFhj2JeZMMzMWH6MYzDWn5xWrzlt1dwHpwgDfW~xbqO66h&ddk=789361B674144528D0B7EE76B35826&Referer=https%3A%2F%2Frendezvousparis.hermes.com%2Fclient%2Fregister&request=%2Fclient%2Fregister&responsePage=origin&ddv=4.27.0 \ No newline at end of file diff --git a/docs/4_27_0/le_jsdata_original.json b/docs/4_27_0/le_jsdata_original.json new file mode 100644 index 0000000..4e42466 --- /dev/null +++ b/docs/4_27_0/le_jsdata_original.json @@ -0,0 +1,148 @@ +{ + "ttst": 59, + "ifov": false, + "hc": 7, + "br_oh": 684, + "br_ow": 393, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 4.775199583703946, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 684, + "br_w": 393, + "isf": true, + "nddc": 1, + "rs_h": 873, + "rs_w": 393, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.75, + "ars_h": 873, + "ars_w": 393, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": false, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "8DBn6dt1,WTwBAAA", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 2, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "RXJyb3I6IENhbm5vdCByZWFkIHByb3BlcnRpZXMgb2YgbnVsbA==", + "stcfp": "ckFsbCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo3NjM2NCkKICAgIGF0IEhUTUxEb2N1bWVudC5yIChodHRwczovL3JlbmRlenZvdXNwYXJpcy5oZXJtZXMuY29tL2Rpc3QvdmVuZG9yLWM2Mjk1ZTljMjg3ZGExNzYwODllLmpzOjI6ODg2Nzkp", + "ckwa": true, + "emd": "k:ai,vi,ao", + "glvd": "Google Inc. (ARM)", + "glrd": "ANGLE (ARM, Mali-G57 MC2, OpenGL ES 3.2)", + "wwl": false, + "jset": 1715011198, + "dcok": ".hermes.com", + "log2": true, + "m_fmi": false, + "mp_cx": 264, + "mp_cy": 401, + "mp_tr": true, + "mp_mx": 0, + "mp_my": 0, + "mp_sx": 264, + "mp_sy": 494, + "tbce": 0, + "es_sigmdn": 0.000030160432800383137, + "es_mumdn": 9.16338817778005, + "es_distmdn": 123.06502346320826, + "es_angsmdn": -3.1090837853077677, + "es_angemdn": 0.032508868282025365, + "m_s_c": 0, + "m_m_c": 5, + "m_c_c": 5, + "m_cm_r": 1, + "m_ms_r": -1 +} \ No newline at end of file diff --git a/docs/4_27_0/le_jsdata_original_pixel8.json b/docs/4_27_0/le_jsdata_original_pixel8.json new file mode 100644 index 0000000..14a645c --- /dev/null +++ b/docs/4_27_0/le_jsdata_original_pixel8.json @@ -0,0 +1,141 @@ +{ + "ttst": 21.199999999953434, + "ifov": false, + "hc": 9, + "br_oh": 784, + "br_ow": 412, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 5.393727248230813, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 784, + "br_w": 412, + "isf": true, + "nddc": 1, + "rs_h": 915, + "rs_w": 412, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.625, + "ars_h": 915, + "ars_w": 412, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": false, + "plgod": false, + "plg": 0, + "plgne": "NA", + "plgre": "NA", + "plgof": "NA", + "plggt": "NA", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "empty", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 8, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyNjApXShuKDI5NCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1tpKDI2MyldW2koMjYwKV0oJ1x4NjJceDcyXHg2Zlx4NzdceDczXHg2NVx4NzJceDY2XHg2Y1x4NmZceDc3XHgy", + "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzNjQpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODApCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxOTI4", + "ckwa": true, + "glvd": "Google Inc. (ARM)", + "glrd": "ANGLE (ARM, Mali-G715, OpenGL ES 3.2)", + "wwl": false, + "jset": 1715012497, + "emd": "k:ai,vi,ao", + "dcok": ".hermes.com", + "log2": true, + "m_fmi": false, + "tbce": 0, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "m_s_c": 71, + "m_m_c": 3, + "m_c_c": 4, + "m_cm_r": 1.3333333333333333, + "m_ms_r": 0.04225352112676056 +} \ No newline at end of file diff --git a/models/jsdata_le_pojo.py b/models/jsdata_le_pojo.py index b139f18..dce0f08 100644 --- a/models/jsdata_le_pojo.py +++ b/models/jsdata_le_pojo.py @@ -37,8 +37,8 @@ class JsDataLeTypePojo(JsDataPojo): self.m_c_c = m_c_c self.m_cm_r = m_cm_r self.m_ms_r = m_ms_r - self.cfpfe = "RXJyb3I6IENhbm5vdCByZWFkIHByb3BlcnRpZXMgb2YgbnVsbA==" # 4.26.1 - self.stcfp = "ckFsbCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo3NTA1MikKICAgIGF0IEhUTUxEb2N1bWVudC5yIChodHRwczovL3JlbmRlenZvdXNwYXJpcy5oZXJtZXMuY29tL2Rpc3QvdmVuZG9yLWM2Mjk1ZTljMjg3ZGExNzYwODllLmpzOjI6ODg2Nzkp" # 4.26.1 + self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyNjApXShuKDI5NCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1tpKDI2MyldW2koMjYwKV0oJ1x4NjJceDcyXHg2Zlx4NzdceDczXHg2NVx4NzJceDY2XHg2Y1x4NmZceDc3XHgy" # 4.27.0 + self.stcfp = "ckFsbCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo3NjM2NCkKICAgIGF0IEhUTUxEb2N1bWVudC5yIChodHRwczovL3JlbmRlenZvdXNwYXJpcy5oZXJtZXMuY29tL2Rpc3QvdmVuZG9yLWM2Mjk1ZTljMjg3ZGExNzYwODllLmpzOjI6ODg2Nzkp" # 4.27.0 test_data_json = """{"glvd": "Qualcomm", "glrd": "Adreno (TM) 630", "hc": 2, "br_oh": 663, "br_ow": 384, "br_h": 663, diff --git a/models/jsdata_pojo.py b/models/jsdata_pojo.py index 05b802a..7820d76 100644 --- a/models/jsdata_pojo.py +++ b/models/jsdata_pojo.py @@ -265,8 +265,8 @@ class JsDataPojo: self.emt = False self.bfr = False self.dbov = False - self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyMDcpXSgnXHg2Mlx4NzJceDZmXHg3N1x4NzNceDY1XHg3Mlx4NjZceDZjXHg2Zlx4NzdceDJkXHg2M1x4NmZceDZlXHg3NFx4NjFceDY5XHg2ZVx4NjVceDcyJyk7aWYobyl7IWZ1bmN0aW9uIHQoKXt2YXIg" # 4.26.1 - self.stcfp = "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzUwNTIpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWIgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODkyMjMpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMTE0" # 4.26.1 + self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyNjApXShuKDI5NCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1tpKDI2MyldW2koMjYwKV0oJ1x4NjJceDcyXHg2Zlx4NzdceDczXHg2NVx4NzJceDY2XHg2Y1x4NmZceDc3XHgy" # 4.27.0 + self.stcfp = "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzNjQpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODApCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxOTI4" # 4.27.0 self.ckwa = True def to_url_encoded_json(self): diff --git a/workers/captcha_result_getter.py b/workers/captcha_result_getter.py index 0de996c..c744c91 100644 --- a/workers/captcha_result_getter.py +++ b/workers/captcha_result_getter.py @@ -105,7 +105,7 @@ class CaptchaResultGetter: return None def get_ch_raw_data_from_js_data(self, js_data: JsDataPojo, old_valid_cookie) -> str: - _tag_version = "4.26.1" + _tag_version = "4.27.0" _raw_data = "jsData={}&eventCounters=%5B%5D&jsType=ch&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv={}".format( js_data.to_url_encoded_json(), old_valid_cookie, _tag_version) print("raw ch data is " + _raw_data) @@ -172,7 +172,7 @@ class CaptchaResultGetter: # _le_js_raw_data = self.get_le_raw_data_from_js_data(js_le_type_data=js_le_type_data, # old_valid_cookie=old_valid_cookie) _cid = get_datadome_cookies(old_valid_cookie) - _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.26.1".format( + _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.27.0".format( js_le_type_data.to_url_encoded_json(), mousemove_count, click_count, scroll_count, touch_count, touch_count, touch_move, key_count, diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index 63cbbb6..0073764 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -203,16 +203,18 @@ def validate_links(cookiesPublisher, queue_name: str, link_list: list): def validate_all_links(): all_link_list = MONGO_STORE_MANAGER.get_links_to_validate() + # get the first 50 links + _first_50_links = all_link_list[0:50] _queue_name = TEST_QUEUE - _segment_number = 4 + _segment_number = 10 last_thread = None for i in range(0, _segment_number): - logger.info("{}:{} links to validate".format(threading.currentThread().name, len(all_link_list))) + logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_50_links))) logger.info("segment is {}".format(i)) _cookiesPublisher = CookiesPublisher(queue_name=TEST_QUEUE) _cookiesPublisher.set_up_connection() - _step = int(len(all_link_list) / _segment_number) - _sublist = all_link_list[i * _step:_step * (i + 1)] + _step = int(len(_first_50_links) / _segment_number) + _sublist = _first_50_links[i * _step:_step * (i + 1)] _thread1 = threading.Thread(target=validate_links, args=(_cookiesPublisher, SHARED_OBJECT, _sublist)) last_thread = _thread1 _thread1.start() From bb08f2d4fa7eec65a265f4a0a33e1ee6c143e141 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 10 May 2024 00:33:55 +0200 Subject: [PATCH 12/45] add more logs --- queue_message/appointmentrequestsender.py | 1 + workers/link_validator_with_provided_list.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/queue_message/appointmentrequestsender.py b/queue_message/appointmentrequestsender.py index 20557dc..fcf4880 100644 --- a/queue_message/appointmentrequestsender.py +++ b/queue_message/appointmentrequestsender.py @@ -231,6 +231,7 @@ class AppointmentRequestSender(threading.Thread): for mail in _mail_list: for _contact in self.list_to_retrieve_mails: if _contact.mail == mail.mail: + self.logger.info("will get mail from " + mail.mail) _mail_list_filtered.append(mail) self.logger.info("will call find_links_to_validate_from_mail_list, size = " + str(len(_mail_list_filtered))) find_links_to_validate_from_mail_list(_mail_list_filtered) diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index 0073764..836344a 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -206,7 +206,7 @@ def validate_all_links(): # get the first 50 links _first_50_links = all_link_list[0:50] _queue_name = TEST_QUEUE - _segment_number = 10 + _segment_number = 20 last_thread = None for i in range(0, _segment_number): logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_50_links))) From 0e94e754b21856f8053a0002b9e6972cf3b9140c Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 10 May 2024 14:32:20 +0200 Subject: [PATCH 13/45] update mails configurations --- mail/mail_constants.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mail/mail_constants.py b/mail/mail_constants.py index 7e88937..4806d16 100755 --- a/mail/mail_constants.py +++ b/mail/mail_constants.py @@ -14,12 +14,14 @@ DOMAN_BTVM_NE_JP = "btvm.ne.jp" DOMAN_AURORA_DTI_NE_JP = "aurora.dti.ne.jp" DOMAN_GMAIL = "gmail.com" DOMAIN_GMX = "gmx.com" +DOMAIN_GMX_NET = "gmx.net" DOMAIN_GMX_FR = "gmx.fr" DOMAIN_GMX_US = "gmx.us" DOMAIN_GMX_CH = "gmx.ch" DOMAIN_ONET = "onet.pl" DOMAIN_NAVER = "naver.com" DOMAIN_INBOX_LV = "inbox.lv" +DOMAIN_GMX_DE = "gmx.de" DOMAIN_PISS_MAIL = "pissmail.com" DOMAIN_INCEL_EMAIL = "incel.email" @@ -34,6 +36,7 @@ IMAP_SERVER_163 = "imap.163.com" IMAP_SERVER_SINA = "imap.sina.com" YAHOO_IMAP_SERVER = "imap.mail.yahoo.com" HOTMAIL_IMAP_SERVER = "outlook.office365.com" + RAMBLER_IMAP_SERVER = "imap.rambler.ru" ALICE_IMAP_SERVER = "in.alice.it" TIME_IT_SERVER = "imap.tim.it" @@ -43,6 +46,7 @@ BTVM_NE_JP = "imap.btvm.ne.jp" SEREVER_GMAIL = "imap.gmail.com" SERVER_IMAGE_ONET = "imap.poczta.onet.pl" SERVER_GMX = "imap.gmx.com" +SERVER_GMX_NET = "imap.gmx.net" SERVER_PISS_MAIL = "mail.pissmail.com" INBOX_LV = "mail.inbox.lv" @@ -97,6 +101,8 @@ def create_imap(login: str): imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143) elif DOMAIN_NAVER in login: imap = imaplib.IMAP4_SSL(NAVER_SERVER, port=993) + elif DOMAIN_GMX_DE in login or DOMAIN_GMX_NET in login: + imap = imaplib.IMAP4_SSL(SERVER_GMX_NET, port=993) elif DOMAIN_INBOX_LV in login: imap = imaplib.IMAP4_SSL(INBOX_LV, port=993) elif DOMAIN_PISS_MAIL in login or DOMAIN_CHILD_PIZZA in login or DOMAIN_DMC_CHAT in login or DOMAIN_GENOCIDE_FUN in login or DOMAIN_HATESJE_WS in login or DOMAIN_INCEL_EMAIL in login or DOMAIN_SHITPOSTING_EXPERT in login: From 42ff1426d6aec05746f1605048b45b42a369bc85 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 10 May 2024 14:33:05 +0200 Subject: [PATCH 14/45] update mails configurations --- mail/mail_reader_all_contacts.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/mail/mail_reader_all_contacts.py b/mail/mail_reader_all_contacts.py index 3e2232f..8a747e8 100755 --- a/mail/mail_reader_all_contacts.py +++ b/mail/mail_reader_all_contacts.py @@ -30,7 +30,7 @@ def check_email_address(email): print("Valid Email") return True else: - print("Invalid Email") + print("Invalid Email:" + email) return False @@ -203,16 +203,12 @@ def need_to_valid_url(url: str, successful_items) -> bool: parts = url.split('/') id = parts[5] if len(id) == 6: - if id == "CS93VB": - print("found") for item in successful_items: if item.id == id: if item.url_validated is not None: return not item.url_validated else: # if url_validated is None - if item.url_validated is not None: - return not item.url_validated return True return True else: @@ -236,7 +232,7 @@ def need_to_check_email(mail: str, successful_items) -> bool: return True -def find_links_to_validate_from_mail_list(mail_list: list): +def find_links_to_validate_from_mail_list(mail_list: list, logger): # check time before start checking emails contact_to_book_list = MONGO_STORE_MANAGER.get_all_contact_to_book_list() successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day() @@ -252,14 +248,14 @@ def find_links_to_validate_from_mail_list(mail_list: list): # if need_to_check_email(mail.mail, successful_items): # mail_reader = MailReader(mail.mail, mail.password) # mail_reader.read_emails(mails_messages) - + _refreshed_successful_items = MONGO_STORE_MANAGER.get_all_successful_items_for_day() for mail in mails_messages: match = re.search(VALIDATION_URL_REGEX, mail.body) if match: url = match.group(0) - if need_to_valid_url(url, successful_items): - print("need to validate url: " + url) + if need_to_valid_url(url, _refreshed_successful_items): + logger.info("need to validate url: " + url) MONGO_STORE_MANAGER.save_links_to_validate(url, mail.to_address, _all_contact_list=contact_to_book_list) else: - print("do not need to click url --> {}".format(mail.mail_address)) + logger.info("do not need to click url --> {}".format(mail.mail_address)) From c3dd17cf1841f4211b46d326b88262516a515608 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 10 May 2024 14:33:19 +0200 Subject: [PATCH 15/45] update mails configurations --- queue_message/appointmentrequestsender.py | 2 +- workers/link_validator_with_provided_list.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/queue_message/appointmentrequestsender.py b/queue_message/appointmentrequestsender.py index fcf4880..2b4aadc 100644 --- a/queue_message/appointmentrequestsender.py +++ b/queue_message/appointmentrequestsender.py @@ -234,7 +234,7 @@ class AppointmentRequestSender(threading.Thread): self.logger.info("will get mail from " + mail.mail) _mail_list_filtered.append(mail) self.logger.info("will call find_links_to_validate_from_mail_list, size = " + str(len(_mail_list_filtered))) - find_links_to_validate_from_mail_list(_mail_list_filtered) + find_links_to_validate_from_mail_list(_mail_list_filtered, self.logger) self.already_read_emails = True else: self.logger.info("already read emails, is there any contacts to use") diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index 836344a..4b51c73 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -12,7 +12,8 @@ from db.mongo_manager import MONGO_STORE_MANAGER from models.LinkPojo import LinkPojo from models.result_pojo import RequestResult from proxy_manager.proxy_manager import ProxyManager -from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST, TEST_QUEUE, SHARED_OBJECT +from queue_message.CookiesPublisher import CookiesPublisher, REQUEST_DATA_QUEUE_TEST, TEST_QUEUE, SHARED_OBJECT, \ + MORNING_DATA_CACHE_BAK from queue_message.appointmentrequestsender import QUEUE_HOST, REQUEST_DATA_QUEUE, credentials from utils.AppLogging import init_logger from workers.proxies_constants import PROXY_LIST_FR @@ -204,18 +205,18 @@ def validate_links(cookiesPublisher, queue_name: str, link_list: list): def validate_all_links(): all_link_list = MONGO_STORE_MANAGER.get_links_to_validate() # get the first 50 links - _first_50_links = all_link_list[0:50] - _queue_name = TEST_QUEUE + _first_50_links = all_link_list[0:(int(len(all_link_list) / 4))] + _queue_name = MORNING_DATA_CACHE_BAK _segment_number = 20 last_thread = None for i in range(0, _segment_number): logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_50_links))) logger.info("segment is {}".format(i)) - _cookiesPublisher = CookiesPublisher(queue_name=TEST_QUEUE) + _cookiesPublisher = CookiesPublisher(queue_name=_queue_name) _cookiesPublisher.set_up_connection() _step = int(len(_first_50_links) / _segment_number) _sublist = _first_50_links[i * _step:_step * (i + 1)] - _thread1 = threading.Thread(target=validate_links, args=(_cookiesPublisher, SHARED_OBJECT, _sublist)) + _thread1 = threading.Thread(target=validate_links, args=(_cookiesPublisher, MORNING_DATA_CACHE_BAK, _sublist)) last_thread = _thread1 _thread1.start() last_thread.join() @@ -223,4 +224,7 @@ def validate_all_links(): if __name__ == '__main__': # generate test data - validate_all_links() + while True: + validate_all_links() + print("wait for 30 seconds") + time.sleep(30) From 20e6c0b5e01780ea710b7bb862630b1a4c7214ab Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 10 May 2024 15:58:01 +0200 Subject: [PATCH 16/45] use 28 threads --- scheduler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scheduler_test.py b/scheduler_test.py index 2ae5022..6e0ae38 100644 --- a/scheduler_test.py +++ b/scheduler_test.py @@ -5,7 +5,7 @@ from request_sender_test import start_send_requests def start_book_appointment(): - start_send_requests(thread_number=25, file_path='~/Desktop/contact_list_2024-05-04.xlsx') + start_send_requests(thread_number=28, file_path='~/Desktop/contact_list_2024-05-09.xlsx') def start_check_results_job(sched): From 8f432e08592bd853a6f46adf2fd2cb2183fa0caf Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 10 May 2024 17:40:33 +0200 Subject: [PATCH 17/45] support data impulse --- models/ReserveResultPojo.py | 2 ++ proxy_manager/proxy_manager.py | 35 +++++++++++++++++++++-- queue_message/appointmentrequestsender.py | 18 +++++------- request_sender_test.py | 6 ++-- workers/sender.py | 4 +++ 5 files changed, 49 insertions(+), 16 deletions(-) diff --git a/models/ReserveResultPojo.py b/models/ReserveResultPojo.py index 0d41ae1..2cf6892 100755 --- a/models/ReserveResultPojo.py +++ b/models/ReserveResultPojo.py @@ -34,6 +34,7 @@ class ReserveResultPojo: url_validated = None created_at = None validated_at = None + proxy: str = None @staticmethod def from_firestore_dict(source): @@ -127,6 +128,7 @@ class ReserveResultPojo: u'store_type': self.store_type, u'accepted': self.accepted, u'url_validated': self.url_validated, + u'proxy': self.proxy, } return dest diff --git a/proxy_manager/proxy_manager.py b/proxy_manager/proxy_manager.py index 0ab6d56..589c2c0 100644 --- a/proxy_manager/proxy_manager.py +++ b/proxy_manager/proxy_manager.py @@ -1,3 +1,5 @@ +import random + FR_ASOCKS_MOBILE_PROXY = { 'http': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046', 'https': 'http://11797317-mobile-country-FR:nv958134x@190.2.151.110:14046', @@ -17,13 +19,42 @@ FR_PROXY_ASOCK_RES_2 = { 'http': 'http://10488120-res-country-FR:k94fsbn9a@217.23.6.161:11287', 'https': 'http://10488120-res-country-FR:k94fsbn9a@217.23.6.161:11287' } +FR_PROXY_MOB_OXY_STICKY = { + 'http': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}', + 'https': 'http://customer-rendezvousmob-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}' +} + +FR_PROXY_DATA_IMPULSE_STICKY = { + 'http': 'http://1d568220dbefeff21ad4__cr.fr:df99489a25fa72d9@gw.dataimpulse.com:{}', + 'https': 'http://1d568220dbefeff21ad4__cr.fr:df99489a25fa72d9@gw.dataimpulse.com:{}' +} + +MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_DATA_IMPULSE_STICKY] class ProxyManager: + + def __init__(self, logger): + self.logger = logger + def get_link_validate_proxy(self, links_to_validate: list) -> list: if len(links_to_validate) > 15: - return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY] + return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2] # return [FR_PROXY_RES_OXY] else: - return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_ASOCKS_MOBILE_PROXY] + return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2] # return [FR_PROXY_RES_OXY] + + def get_proxy_for_appointment_request(self) -> dict: + _chosen_proxy = random.choice(MOBILE_PROXY_LIST) + if "oxylabs" in _chosen_proxy["http"]: + self.logger.info("use oxylabs proxy") + _port = random.randint(40001, 49999) + else: + self.logger.info("use dataimpulse proxy") + _port = random.randint(10000, 20000) + self.logger.info("generated port is {}".format(_port)) + _proxy_to_use = {} + _proxy_to_use["http"] = _chosen_proxy["http"].format(_port) + _proxy_to_use["https"] = _chosen_proxy["https"].format(_port) + return _proxy_to_use diff --git a/queue_message/appointmentrequestsender.py b/queue_message/appointmentrequestsender.py index 2b4aadc..5ddb2d9 100644 --- a/queue_message/appointmentrequestsender.py +++ b/queue_message/appointmentrequestsender.py @@ -13,6 +13,7 @@ from models.contact_pojo import ContactPojo from models.jsdata_le_pojo import JsDataLeTypePojo from models.jsdata_pojo import JsDataPojo from models.result_pojo import RequestResult +from proxy_manager.proxy_manager import ProxyManager from queue_message.CookiesPublisher import CookiesPublisher from utiles import is_time_between from workers.captcha_result_getter import CaptchaResultGetter, HERMES_REGISTER @@ -62,7 +63,7 @@ def is_open(): class AppointmentRequestSender(threading.Thread): - def __init__(self, sub_contact_list: list, proxy_to_use_list, logger, cookiesPublisher: CookiesPublisher, + def __init__(self, sub_contact_list: list, logger, cookiesPublisher: CookiesPublisher, just_send=False, queue_name=REQUEST_DATA_QUEUE): super().__init__() @@ -77,7 +78,7 @@ class AppointmentRequestSender(threading.Thread): self.initial_contact_list = sub_contact_list self.contact_list = sub_contact_list self.queue_name = queue_name - self.proxy_to_use_list = proxy_to_use_list + self.proxy_manager = ProxyManager(logger) self.already_read_emails = False def set_up_connection(self): @@ -96,13 +97,6 @@ class AppointmentRequestSender(threading.Thread): self.logger.info("message count in queue is {}".format(_message_count)) _received_object = body.decode("UTF-8") self.logger.info(f" [x] Received {_received_object}") - _port = random.randint(40001, 49999) - self.logger.info("generated port is {}".format(_port)) - _chosen_proxy = random.choice(self.proxy_to_use_list) - _proxy_to_use = {} - _proxy_to_use["http"] = _chosen_proxy["http"].format(_port) - _proxy_to_use["https"] = _chosen_proxy["https"].format(_port) - print(_proxy_to_use) if "glrd" in _received_object: _received_dict = json.loads(_received_object) js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'], @@ -114,8 +108,6 @@ class AppointmentRequestSender(threading.Thread): rs_h=_received_dict['rs_h'], rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd']) _received_cookies = _received_dict["cookiesStr"] - sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict, - proxy_to_use=_proxy_to_use, logger=self.logger) # remove already sent contacts if self.just_send: self.contact_list = filter_contacts(self.contact_list, self.already_tried_contact_list) @@ -128,6 +120,10 @@ class AppointmentRequestSender(threading.Thread): self.logger.info("contact number is {}".format(len(self.contact_list))) # self.contact_list = filter_contacts(self.contact_list) for con in self.contact_list: + _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() + print(_proxy_to_use) + sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict, + proxy_to_use=_proxy_to_use, logger=self.logger) self.logger.info(con.mail) if self.valid_csrf is None: csrf_result = captchaResultGetter.get_csrf( diff --git a/request_sender_test.py b/request_sender_test.py index 05d19a1..95ee5df 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -60,7 +60,7 @@ def send_appointment_request(message_queue_name, _contact_list): logger.info(_contact) _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name) _cookiesPublisher.set_up_connection() - receiver = AppointmentRequestSender(sub_contact_list=_contact_list, proxy_to_use_list=MOBILE_PROXY_LIST_FR, + receiver = AppointmentRequestSender(sub_contact_list=_contact_list, queue_name=message_queue_name, just_send=True, cookiesPublisher=_cookiesPublisher, logger=logger) print("count is " + str(count)) @@ -79,11 +79,11 @@ def start_send_requests(thread_number, file_path): logger.info("segment is {}".format(i)) _step = int(len(_contact_list_to_book) / _segment_number) _sublist = _contact_list_to_book[i * _step:_step * (i + 1)] - _thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE, _sublist)) + _thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE_2, _sublist)) last_thread = _thread1 _thread1.start() last_thread.join() if __name__ == '__main__': - start_send_requests(thread_number=25, file_path='~/Desktop/contact_list_2024-04-17.xlsx') + start_send_requests(thread_number=5, file_path='~/Desktop/contact_list_2024-04-24.xlsx') diff --git a/workers/sender.py b/workers/sender.py index 6ea2316..f35b034 100644 --- a/workers/sender.py +++ b/workers/sender.py @@ -36,6 +36,10 @@ class Sender: result = ReserveResultPojo(type=status, phone=contact.phone, message=status.value, url=url, firstName=contact.first_name, lastName=contact.last_name, email=contact.mail, passport=contact.passport, ccid=contact.ccid) + if "oxylabs" in self.proxy_to_use["http"]: + result.proxy = "oxylabs" + else: + result.proxy = "data_impulse" result.id = id result.store_type = store_type result.created_at = time.strftime("%H:%M:%S", time.localtime()) From 3989066d7f9a0b54c73fd3e6b2ec85f552188ead Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Sat, 11 May 2024 09:09:09 +0200 Subject: [PATCH 18/45] support data impulse --- proxy_manager/proxy_manager.py | 3 ++- workers/link_validator_with_provided_list.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/proxy_manager/proxy_manager.py b/proxy_manager/proxy_manager.py index 589c2c0..2c23068 100644 --- a/proxy_manager/proxy_manager.py +++ b/proxy_manager/proxy_manager.py @@ -29,7 +29,8 @@ FR_PROXY_DATA_IMPULSE_STICKY = { 'https': 'http://1d568220dbefeff21ad4__cr.fr:df99489a25fa72d9@gw.dataimpulse.com:{}' } -MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_DATA_IMPULSE_STICKY] +MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, + FR_PROXY_DATA_IMPULSE_STICKY] class ProxyManager: diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index 4b51c73..ad8f231 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -38,7 +38,7 @@ class LinkValidatorWithProvidedList(threading.Thread): self.queue_to_listen = queue_to_listen self.ip_country = ip_country self.filter_with_ip_country() - self.proxy_manager = ProxyManager() + self.proxy_manager = ProxyManager(logger) self.limit = limit def set_up_connection(self): From cb0a4df5a1337baf55b21e6cbeac83fb58ae7f8d Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Sat, 11 May 2024 11:24:42 +0200 Subject: [PATCH 19/45] same cookie use same ip address --- proxy_manager/proxy_manager.py | 14 +++++++----- queue_message/appointmentrequestsender.py | 7 ++++-- request_sender.py | 14 ++++++------ request_sender_test.py | 2 +- scheduler_test.py | 2 +- workers/link_validator_with_provided_list.py | 23 +++++++++++++------- 6 files changed, 38 insertions(+), 24 deletions(-) diff --git a/proxy_manager/proxy_manager.py b/proxy_manager/proxy_manager.py index 2c23068..e46cbec 100644 --- a/proxy_manager/proxy_manager.py +++ b/proxy_manager/proxy_manager.py @@ -29,7 +29,13 @@ FR_PROXY_DATA_IMPULSE_STICKY = { 'https': 'http://1d568220dbefeff21ad4__cr.fr:df99489a25fa72d9@gw.dataimpulse.com:{}' } -MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, +FR_DATA_IMPULSE_RES = { + 'http': 'http://ac2b8183c3b02714cbd1__cr.fr:336e7f8f4b17e5bd@gw.dataimpulse.com:823', + 'https': 'http://ac2b8183c3b02714cbd1__cr.fr:336e7f8f4b17e5bd@gw.dataimpulse.com:823' +} +# 八分之一用data_impulse +MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, + FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_DATA_IMPULSE_STICKY] @@ -40,11 +46,9 @@ class ProxyManager: def get_link_validate_proxy(self, links_to_validate: list) -> list: if len(links_to_validate) > 15: - return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2] - # return [FR_PROXY_RES_OXY] + return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES] else: - return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2] - # return [FR_PROXY_RES_OXY] + return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES] def get_proxy_for_appointment_request(self) -> dict: _chosen_proxy = random.choice(MOBILE_PROXY_LIST) diff --git a/queue_message/appointmentrequestsender.py b/queue_message/appointmentrequestsender.py index 5ddb2d9..1d962d6 100644 --- a/queue_message/appointmentrequestsender.py +++ b/queue_message/appointmentrequestsender.py @@ -97,6 +97,9 @@ class AppointmentRequestSender(threading.Thread): self.logger.info("message count in queue is {}".format(_message_count)) _received_object = body.decode("UTF-8") self.logger.info(f" [x] Received {_received_object}") + # 同一个COOKIES保持用同一个IP + _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() + print("proxy to use is {}".format(_proxy_to_use)) if "glrd" in _received_object: _received_dict = json.loads(_received_object) js_data = JsDataPojo(glrd=_received_dict['glrd'], glvd=_received_dict['glvd'], hc=_received_dict['hc'], @@ -120,8 +123,8 @@ class AppointmentRequestSender(threading.Thread): self.logger.info("contact number is {}".format(len(self.contact_list))) # self.contact_list = filter_contacts(self.contact_list) for con in self.contact_list: - _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() - print(_proxy_to_use) + # _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() + # print(_proxy_to_use) sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict, proxy_to_use=_proxy_to_use, logger=self.logger) self.logger.info(con.mail) diff --git a/request_sender.py b/request_sender.py index 14ea7ab..25077ae 100644 --- a/request_sender.py +++ b/request_sender.py @@ -6,7 +6,8 @@ from threading import Thread from db.mongo_manager import MONGO_STORE_MANAGER from excel_reader import read_contacts from models.contact_pojo import ContactPojo -from queue_message.CookiesPublisher import CookiesPublisher, SHARED_OBJECT, TEST_QUEUE +from queue_message.CookiesPublisher import CookiesPublisher, SHARED_OBJECT, TEST_QUEUE, MORNING_DATA_CACHE_2, \ + MORNING_DATA_CACHE from queue_message.appointmentrequestsender import AppointmentRequestSender from utiles import is_time_between from utils.AppLogging import init_logger @@ -62,8 +63,7 @@ def send_appointment_request(message_queue_name, _contact_list): logger.info(_contact) _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name) _cookiesPublisher.set_up_connection() - receiver = AppointmentRequestSender(sub_contact_list=_contact_list, proxy_to_use_list=MOBILE_PROXY_LIST_FR, - queue_name=message_queue_name, just_send=True, + receiver = AppointmentRequestSender(sub_contact_list=_contact_list, queue_name=message_queue_name, just_send=True, cookiesPublisher=_cookiesPublisher, logger=logger) print("count is " + str(count)) receiver.run() @@ -71,17 +71,17 @@ def send_appointment_request(message_queue_name, _contact_list): def start_send_requests(): print("start send requests") - contacts_file_path = '~/Desktop/contact_list_2024-04-19.xlsx' - _contact_list = read_contacts(contacts_file_path) + contacts_file_path = '~/Desktop/11_05_to_test.xlsx' + _contact_list = read_contacts(contacts_file_path)[:-1] _contact_list_to_book = filter_contacts(_contact_list) - _segment_number = 1 + _segment_number = 5 logger.info("{} contacts to book".format(len(_contact_list_to_book))) last_thread = None for i in range(0, _segment_number): logger.info("segment is {}".format(i)) _step = int(len(_contact_list_to_book) / _segment_number) _sublist = _contact_list_to_book[i * _step:_step * (i + 1)] - _thread1 = Thread(target=send_appointment_request, args=(SHARED_OBJECT, _sublist)) + _thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE, _sublist)) last_thread = _thread1 _thread1.start() last_thread.join() diff --git a/request_sender_test.py b/request_sender_test.py index 95ee5df..5ca058e 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -86,4 +86,4 @@ def start_send_requests(thread_number, file_path): if __name__ == '__main__': - start_send_requests(thread_number=5, file_path='~/Desktop/contact_list_2024-04-24.xlsx') + start_send_requests(thread_number=28, file_path='~/Desktop/contact_list_2024-05-10.xlsx') diff --git a/scheduler_test.py b/scheduler_test.py index 6e0ae38..f347f07 100644 --- a/scheduler_test.py +++ b/scheduler_test.py @@ -5,7 +5,7 @@ from request_sender_test import start_send_requests def start_book_appointment(): - start_send_requests(thread_number=28, file_path='~/Desktop/contact_list_2024-05-09.xlsx') + start_send_requests(thread_number=28, file_path='~/Desktop/contact_list_2024-05-10.xlsx') def start_check_results_job(sched): diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index ad8f231..f338eca 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -68,8 +68,6 @@ class LinkValidatorWithProvidedList(threading.Thread): 'Sec-Fetch-Site': 'same-origin', 'Sec-Fetch-Dest': 'document', 'Accept-Language': 'fr-FR,fr;q=0.6'} - _proxy_to_use = random.choice(self.proxy_manager.get_link_validate_proxy(self.link_to_validate_list)) - print(_proxy_to_use) print("received cookie is " + str(self.cookie_str)) print("send request for link: " + linkPojo.url) try: @@ -161,7 +159,9 @@ class LinkValidatorWithProvidedList(threading.Thread): can_continue = None for link_to_validate in self.link_to_validate_list: print(link_to_validate) - self.proxy_to_use = random.choice(PROXY_LIST_FR) + self.proxy_to_use = random.choice( + self.proxy_manager.get_link_validate_proxy(self.link_to_validate_list)) + print("proxy to use is {}".format(self.proxy_to_use)) can_continue = self.send_request(link_to_validate, _received_dict) # remove the tested link from link list self.link_to_validate_list.remove(link_to_validate) @@ -205,17 +205,24 @@ def validate_links(cookiesPublisher, queue_name: str, link_list: list): def validate_all_links(): all_link_list = MONGO_STORE_MANAGER.get_links_to_validate() # get the first 50 links - _first_50_links = all_link_list[0:(int(len(all_link_list) / 4))] + if len(all_link_list) == 0: + return + divided = 4 + _first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))] _queue_name = MORNING_DATA_CACHE_BAK - _segment_number = 20 + if len(_first_25_percent_links) > 40: + _segment_number = 20 + else: + _first_25_percent_links = all_link_list + _segment_number = int(len(_first_25_percent_links) / divided) last_thread = None for i in range(0, _segment_number): - logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_50_links))) + logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_25_percent_links))) logger.info("segment is {}".format(i)) _cookiesPublisher = CookiesPublisher(queue_name=_queue_name) _cookiesPublisher.set_up_connection() - _step = int(len(_first_50_links) / _segment_number) - _sublist = _first_50_links[i * _step:_step * (i + 1)] + _step = int(len(_first_25_percent_links) / _segment_number) + _sublist = _first_25_percent_links[i * _step:_step * (i + 1)] _thread1 = threading.Thread(target=validate_links, args=(_cookiesPublisher, MORNING_DATA_CACHE_BAK, _sublist)) last_thread = _thread1 _thread1.start() From ff20c1a003193b60fd898b2b1389b0d796fba25d Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Sat, 11 May 2024 11:47:11 +0200 Subject: [PATCH 20/45] same cookie use same ip address --- workers/link_validator_with_provided_list.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index f338eca..cd01e01 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -210,11 +210,14 @@ def validate_all_links(): divided = 4 _first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))] _queue_name = MORNING_DATA_CACHE_BAK - if len(_first_25_percent_links) > 40: + if len(all_link_list) > 40: _segment_number = 20 else: _first_25_percent_links = all_link_list - _segment_number = int(len(_first_25_percent_links) / divided) + if len(_first_25_percent_links) > divided: + _segment_number = int(len(_first_25_percent_links) / divided) + else: + _segment_number = 1 last_thread = None for i in range(0, _segment_number): logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_25_percent_links))) @@ -226,7 +229,8 @@ def validate_all_links(): _thread1 = threading.Thread(target=validate_links, args=(_cookiesPublisher, MORNING_DATA_CACHE_BAK, _sublist)) last_thread = _thread1 _thread1.start() - last_thread.join() + if last_thread is not None: + last_thread.join() if __name__ == '__main__': From 1dd0fd673f39bc9ef23261961ca8c2a12fa9b1d5 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Mon, 13 May 2024 11:28:47 +0200 Subject: [PATCH 21/45] use variables --- workers/link_validator_with_provided_list.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index cd01e01..393cc31 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -208,10 +208,11 @@ def validate_all_links(): if len(all_link_list) == 0: return divided = 4 + default_segment_number = 20 _first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))] _queue_name = MORNING_DATA_CACHE_BAK - if len(all_link_list) > 40: - _segment_number = 20 + if len(all_link_list) > divided * default_segment_number: + _segment_number = default_segment_number else: _first_25_percent_links = all_link_list if len(_first_25_percent_links) > divided: From e7bcd22cfea9f175ecc94753ef33b600dc6683a4 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 13 May 2024 12:38:35 +0200 Subject: [PATCH 22/45] add more logs --- workers/link_validator_with_provided_list.py | 1 + 1 file changed, 1 insertion(+) diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index 393cc31..ba3e835 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -237,6 +237,7 @@ def validate_all_links(): if __name__ == '__main__': # generate test data while True: + print("call validate_all_links()") validate_all_links() print("wait for 30 seconds") time.sleep(30) From 36ba7ad47f35970bd2d74e09f191e9a2d1b2317d Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 13 May 2024 12:44:54 +0200 Subject: [PATCH 23/45] added start_validate_links.sh --- start_validate_links.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 start_validate_links.sh diff --git a/start_validate_links.sh b/start_validate_links.sh new file mode 100644 index 0000000..f0d7a49 --- /dev/null +++ b/start_validate_links.sh @@ -0,0 +1,2 @@ +source ./venv/bin/activate +python workers/link_validator_with_provided_list.py \ No newline at end of file From 34135073dfb3e1e1ab1bfcdfc4c5162368e502d9 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Wed, 15 May 2024 08:42:06 +0200 Subject: [PATCH 24/45] added start_validate_links.sh --- link_validator_executor.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/link_validator_executor.py b/link_validator_executor.py index 318f598..3dd0b71 100644 --- a/link_validator_executor.py +++ b/link_validator_executor.py @@ -1,5 +1,15 @@ -from workers.link_validator import validate_with_FR_ip +import time + +from workers.link_validator_with_provided_list import validate_all_links + +# if __name__ == '__main__': +# # link_list = MONGO_STORE_MANAGER.get_links_to_validate() +# validate_with_FR_ip(segment_position=2) if __name__ == '__main__': - # link_list = MONGO_STORE_MANAGER.get_links_to_validate() - validate_with_FR_ip(segment_position=2) + # generate test data + while True: + print("call validate_all_links()") + validate_all_links() + print("wait for 30 seconds") + time.sleep(30) From 35ec23fbaa6b18bedb1ef5547fb2ad0bbe69ef28 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Wed, 15 May 2024 08:42:35 +0200 Subject: [PATCH 25/45] added start_validate_links.sh --- workers/link_validator_with_provided_list.py | 1 + 1 file changed, 1 insertion(+) diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index ba3e835..9fb8c0c 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -203,6 +203,7 @@ def validate_links(cookiesPublisher, queue_name: str, link_list: list): def validate_all_links(): + print("will get all links") all_link_list = MONGO_STORE_MANAGER.get_links_to_validate() # get the first 50 links if len(all_link_list) == 0: From 67d6633951fdd558d206654c6ff2aec742be4ef1 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Wed, 15 May 2024 10:29:02 +0200 Subject: [PATCH 26/45] default queue MORNING_DATA_CACHE --- request_sender_test.py | 7 ++++--- scheduler_test.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/request_sender_test.py b/request_sender_test.py index 5ca058e..dc61d44 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -67,7 +67,7 @@ def send_appointment_request(message_queue_name, _contact_list): receiver.run() -def start_send_requests(thread_number, file_path): +def start_send_requests(thread_number, file_path, data_queue_name=MORNING_DATA_CACHE): print("start send requests") contacts_file_path = file_path _contact_list = read_contacts(contacts_file_path) @@ -79,11 +79,12 @@ def start_send_requests(thread_number, file_path): logger.info("segment is {}".format(i)) _step = int(len(_contact_list_to_book) / _segment_number) _sublist = _contact_list_to_book[i * _step:_step * (i + 1)] - _thread1 = Thread(target=send_appointment_request, args=(MORNING_DATA_CACHE_2, _sublist)) + _thread1 = Thread(target=send_appointment_request, args=(data_queue_name, _sublist)) last_thread = _thread1 _thread1.start() last_thread.join() if __name__ == '__main__': - start_send_requests(thread_number=28, file_path='~/Desktop/contact_list_2024-05-10.xlsx') + start_send_requests(thread_number=30, file_path='~/Desktop/contact_list_2024-05-14.xlsx', + data_queue_name=MORNING_DATA_CACHE_2) diff --git a/scheduler_test.py b/scheduler_test.py index f347f07..137da42 100644 --- a/scheduler_test.py +++ b/scheduler_test.py @@ -5,7 +5,7 @@ from request_sender_test import start_send_requests def start_book_appointment(): - start_send_requests(thread_number=28, file_path='~/Desktop/contact_list_2024-05-10.xlsx') + start_send_requests(thread_number=31, file_path='~/Desktop/contact_list_2024-05-14.xlsx') def start_check_results_job(sched): From 65014b5c71668368b527039e9f9a04bca6a4c2b5 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Thu, 16 May 2024 23:28:24 +0200 Subject: [PATCH 27/45] support file list --- proxy_manager/proxy_manager.py | 5 ++-- queue_message/appointmentrequestsender.py | 14 +++++------ request_sender_test.py | 29 +++++++++++++++-------- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/proxy_manager/proxy_manager.py b/proxy_manager/proxy_manager.py index e46cbec..6c2473c 100644 --- a/proxy_manager/proxy_manager.py +++ b/proxy_manager/proxy_manager.py @@ -35,8 +35,9 @@ FR_DATA_IMPULSE_RES = { } # 八分之一用data_impulse MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, - FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, - FR_PROXY_DATA_IMPULSE_STICKY] + FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY + # FR_PROXY_DATA_IMPULSE_STICKY + ] class ProxyManager: diff --git a/queue_message/appointmentrequestsender.py b/queue_message/appointmentrequestsender.py index 1d962d6..baf1972 100644 --- a/queue_message/appointmentrequestsender.py +++ b/queue_message/appointmentrequestsender.py @@ -64,18 +64,16 @@ def is_open(): class AppointmentRequestSender(threading.Thread): def __init__(self, sub_contact_list: list, logger, cookiesPublisher: CookiesPublisher, - just_send=False, queue_name=REQUEST_DATA_QUEUE): super().__init__() self.connection = None - self.just_send = just_send self.logger = logger self.already_tried_contact_list = [] self.cookiesPublisher = cookiesPublisher self.channel = None self.valid_csrf = None self.list_to_retrieve_mails = sub_contact_list - self.initial_contact_list = sub_contact_list + self.initial_contact_list = sub_contact_list.copy() self.contact_list = sub_contact_list self.queue_name = queue_name self.proxy_manager = ProxyManager(logger) @@ -112,10 +110,7 @@ class AppointmentRequestSender(threading.Thread): rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd']) _received_cookies = _received_dict["cookiesStr"] # remove already sent contacts - if self.just_send: - self.contact_list = filter_contacts(self.contact_list, self.already_tried_contact_list) - else: - self.contact_list = filter_contacts(self.contact_list) + self.contact_list = filter_contacts(self.contact_list, self.already_tried_contact_list) # remove already booked contacts random.shuffle(self.contact_list) if len(self.contact_list) > 0 and is_open(): @@ -222,6 +217,7 @@ class AppointmentRequestSender(threading.Thread): self.channel.start_consuming() def retrieve_invalidate_urls(self): + # 如果没有已读邮件，而且需要读邮件的联系人表不为空，就读取未读邮件 if not self.already_read_emails and len(self.list_to_retrieve_mails) > 0: self.logger.info("will retrieve validate urls") time.sleep(30) @@ -238,9 +234,13 @@ class AppointmentRequestSender(threading.Thread): else: self.logger.info("already read emails, is there any contacts to use") self.logger.info("reset already_tried_contact_list") + # 重置已尝试的联系人 self.already_tried_contact_list = [] self.contact_list = filter_contacts(self.initial_contact_list, self.already_tried_contact_list) self.logger.info("contact_list size is " + str(len(self.contact_list))) if len(self.contact_list) > 0: self.logger.info("set already_read_emails to False") self.already_read_emails = False + else: + self.logger.info("already read emails, no contact to use -> stop") + self.channel.stop_consuming() diff --git a/request_sender_test.py b/request_sender_test.py index dc61d44..dae1e23 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -61,30 +61,39 @@ def send_appointment_request(message_queue_name, _contact_list): _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name) _cookiesPublisher.set_up_connection() receiver = AppointmentRequestSender(sub_contact_list=_contact_list, - queue_name=message_queue_name, just_send=True, + queue_name=message_queue_name, cookiesPublisher=_cookiesPublisher, logger=logger) print("count is " + str(count)) receiver.run() -def start_send_requests(thread_number, file_path, data_queue_name=MORNING_DATA_CACHE): +def start_send_requests(thread_number, contact_list, data_queue_name=MORNING_DATA_CACHE): print("start send requests") - contacts_file_path = file_path - _contact_list = read_contacts(contacts_file_path) - _contact_list_to_book = filter_contacts(_contact_list) + _contact_list_to_book = filter_contacts(contact_list) _segment_number = thread_number logger.info("{} contacts to book".format(len(_contact_list_to_book))) - last_thread = None + # last_thread = None + thread_list = [] for i in range(0, _segment_number): logger.info("segment is {}".format(i)) _step = int(len(_contact_list_to_book) / _segment_number) _sublist = _contact_list_to_book[i * _step:_step * (i + 1)] _thread1 = Thread(target=send_appointment_request, args=(data_queue_name, _sublist)) - last_thread = _thread1 + thread_list.append(_thread1) _thread1.start() - last_thread.join() + for _thread in thread_list: + _thread.join() + + +def send_request_for_file_list(_file_list: list[str], thread_number: int = 20, data_queue_name=MORNING_DATA_CACHE): + for _file_path in file_list: + logger.info("send request for file: " + _file_path) + _contact_list = read_contacts(_file_path) + start_send_requests(thread_number=thread_number, contact_list=_contact_list, + data_queue_name=data_queue_name) if __name__ == '__main__': - start_send_requests(thread_number=30, file_path='~/Desktop/contact_list_2024-05-14.xlsx', - data_queue_name=MORNING_DATA_CACHE_2) + file_list = ['~/Desktop/16_05_to_test.xlsx'] + send_request_for_file_list(_file_list=file_list, thread_number=20, + data_queue_name=MORNING_DATA_CACHE_2) From 3c23b96ee2db8f1852e3ff0b2019dfed0a690310 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 17 May 2024 00:45:55 +0200 Subject: [PATCH 28/45] support 4.28.0 --- docs/4_28_0/js_data_ch_4.28.0.json | 127 +++++++++++++++++ docs/4_28_0/js_data_ch_4_27.0_generated.json | 126 +++++++++++++++++ docs/4_28_0/js_data_ch_oneplus.json | 127 +++++++++++++++++ docs/4_28_0/js_data_le_4.28.0.json | 141 +++++++++++++++++++ models/jsdata_le_pojo.py | 4 +- models/jsdata_pojo.py | 4 +- workers/captcha_result_getter.py | 4 +- 7 files changed, 527 insertions(+), 6 deletions(-) create mode 100644 docs/4_28_0/js_data_ch_4.28.0.json create mode 100644 docs/4_28_0/js_data_ch_4_27.0_generated.json create mode 100644 docs/4_28_0/js_data_ch_oneplus.json create mode 100644 docs/4_28_0/js_data_le_4.28.0.json diff --git a/docs/4_28_0/js_data_ch_4.28.0.json b/docs/4_28_0/js_data_ch_4.28.0.json new file mode 100644 index 0000000..5bb7646 --- /dev/null +++ b/docs/4_28_0/js_data_ch_4.28.0.json @@ -0,0 +1,127 @@ +{ + "ttst": 43, + "ifov": false, + "hc": 5, + "br_oh": 646, + "br_ow": 360, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 0.29755279781407795, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 646, + "br_w": 360, + "isf": true, + "nddc": 1, + "rs_h": 800, + "rs_w": 360, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 800, + "ars_w": 360, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "euAfPPm,zZzCJMtW", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 0.5, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzc0MjUpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk5NTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUyOTU3", + "ckwa": true, + "emd": "k:ai,ao,vi,vi d:defau,defau,d104b,f06c5 g:14588,defau,33a71,7143d", + "glvd": "Google Inc. (Qualcomm)", + "glrd": "ANGLE (Qualcomm, Adreno (TM) 650, OpenGL ES 3.2)", + "wwl": false, + "jset": 1715895418 +} \ No newline at end of file diff --git a/docs/4_28_0/js_data_ch_4_27.0_generated.json b/docs/4_28_0/js_data_ch_4_27.0_generated.json new file mode 100644 index 0000000..e9c5457 --- /dev/null +++ b/docs/4_28_0/js_data_ch_4_27.0_generated.json @@ -0,0 +1,126 @@ +{ + "ttst": 43, + "ifov": false, + "hc": 4, + "br_oh": 663, + "br_ow": 384, + "ua": "Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 5.167040384311697, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 663, + "br_w": 384, + "isf": true, + "nddc": 1, + "rs_h": 811, + "rs_w": 384, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.8125, + "ars_h": 811, + "ars_w": 384, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "c1iRnTJ,YUxg36GL", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 4, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyNjApXShuKDI5NCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1tpKDI2MyldW2koMjYwKV0oJ1x4NjJceDcyXHg2Zlx4NzdceDczXHg2NVx4NzJceDY2XHg2Y1x4NmZceDc3XHgy", + "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzNjQpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODApCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxOTI4", + "ckwa": true, + "glvd": "Qualcomm", + "glrd": "Adreno (TM) 630", + "wwl": false, + "jset": 1715011727 +} \ No newline at end of file diff --git a/docs/4_28_0/js_data_ch_oneplus.json b/docs/4_28_0/js_data_ch_oneplus.json new file mode 100644 index 0000000..574b9b6 --- /dev/null +++ b/docs/4_28_0/js_data_ch_oneplus.json @@ -0,0 +1,127 @@ +{ + "ttst": 101, + "ifov": false, + "hc": 8, + "br_oh": 663, + "br_ow": 384, + "ua": "Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 3.375630067013038, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 663, + "br_w": 384, + "isf": true, + "nddc": 1, + "rs_h": 811, + "rs_w": 384, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.8125, + "ars_h": 811, + "ars_w": 384, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "q899Hiw,ZMGiwYMO", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 4, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzc0MjUpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk5NTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUyOTU3", + "ckwa": true, + "emd": "k:ai,vi,ao,vi d:defau,9c510,defau,049c9 g:f7cab,6255a,defau,b376f", + "glvd": "Google Inc. (Qualcomm)", + "glrd": "ANGLE (Qualcomm, Adreno (TM) 630, OpenGL ES 3.2)", + "wwl": false, + "jset": 1715896159 +} \ No newline at end of file diff --git a/docs/4_28_0/js_data_le_4.28.0.json b/docs/4_28_0/js_data_le_4.28.0.json new file mode 100644 index 0000000..a766012 --- /dev/null +++ b/docs/4_28_0/js_data_le_4.28.0.json @@ -0,0 +1,141 @@ +{ + "ttst": 43, + "ifov": false, + "hc": 5, + "br_oh": 646, + "br_ow": 360, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 0.29755279781407795, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 646, + "br_w": 360, + "isf": true, + "nddc": 1, + "rs_h": 800, + "rs_w": 360, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 800, + "ars_w": 360, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "euAfPPm,zZzCJMtW", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 0.5, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzc0MjUpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk5NTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUyOTU3", + "ckwa": true, + "emd": "k:ai,ao,vi,vi d:defau,defau,d104b,f06c5 g:14588,defau,33a71,7143d", + "glvd": "Google Inc. (Qualcomm)", + "glrd": "ANGLE (Qualcomm, Adreno (TM) 650, OpenGL ES 3.2)", + "wwl": false, + "jset": 1715895446, + "dcok": ".hermes.com", + "log2": true, + "m_fmi": false, + "tbce": 0, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "m_s_c": 114, + "m_m_c": 1, + "m_c_c": 1, + "m_cm_r": 1, + "m_ms_r": 0.008771929824561403 +} \ No newline at end of file diff --git a/models/jsdata_le_pojo.py b/models/jsdata_le_pojo.py index dce0f08..f663c80 100644 --- a/models/jsdata_le_pojo.py +++ b/models/jsdata_le_pojo.py @@ -37,8 +37,8 @@ class JsDataLeTypePojo(JsDataPojo): self.m_c_c = m_c_c self.m_cm_r = m_cm_r self.m_ms_r = m_ms_r - self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyNjApXShuKDI5NCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1tpKDI2MyldW2koMjYwKV0oJ1x4NjJceDcyXHg2Zlx4NzdceDczXHg2NVx4NzJceDY2XHg2Y1x4NmZceDc3XHgy" # 4.27.0 - self.stcfp = "ckFsbCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo3NjM2NCkKICAgIGF0IEhUTUxEb2N1bWVudC5yIChodHRwczovL3JlbmRlenZvdXNwYXJpcy5oZXJtZXMuY29tL2Rpc3QvdmVuZG9yLWM2Mjk1ZTljMjg3ZGExNzYwODllLmpzOjI6ODg2Nzkp" # 4.27.0 + self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.28.0 + self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzc0MjUpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk5NTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUyOTU3" # 4.28.0 test_data_json = """{"glvd": "Qualcomm", "glrd": "Adreno (TM) 630", "hc": 2, "br_oh": 663, "br_ow": 384, "br_h": 663, diff --git a/models/jsdata_pojo.py b/models/jsdata_pojo.py index 7820d76..d815f2e 100644 --- a/models/jsdata_pojo.py +++ b/models/jsdata_pojo.py @@ -265,8 +265,8 @@ class JsDataPojo: self.emt = False self.bfr = False self.dbov = False - self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyNjApXShuKDI5NCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1tpKDI2MyldW2koMjYwKV0oJ1x4NjJceDcyXHg2Zlx4NzdceDczXHg2NVx4NzJceDY2XHg2Y1x4NmZceDc3XHgy" # 4.27.0 - self.stcfp = "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzNjQpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODApCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxOTI4" # 4.27.0 + self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.28.0 + self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzc0MjUpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk5NTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUyOTU3" # 4.28.0 self.ckwa = True def to_url_encoded_json(self): diff --git a/workers/captcha_result_getter.py b/workers/captcha_result_getter.py index c744c91..a563227 100644 --- a/workers/captcha_result_getter.py +++ b/workers/captcha_result_getter.py @@ -105,7 +105,7 @@ class CaptchaResultGetter: return None def get_ch_raw_data_from_js_data(self, js_data: JsDataPojo, old_valid_cookie) -> str: - _tag_version = "4.27.0" + _tag_version = "4.28.0" _raw_data = "jsData={}&eventCounters=%5B%5D&jsType=ch&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv={}".format( js_data.to_url_encoded_json(), old_valid_cookie, _tag_version) print("raw ch data is " + _raw_data) @@ -172,7 +172,7 @@ class CaptchaResultGetter: # _le_js_raw_data = self.get_le_raw_data_from_js_data(js_le_type_data=js_le_type_data, # old_valid_cookie=old_valid_cookie) _cid = get_datadome_cookies(old_valid_cookie) - _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.27.0".format( + _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.28.0".format( js_le_type_data.to_url_encoded_json(), mousemove_count, click_count, scroll_count, touch_count, touch_count, touch_move, key_count, From baa3537fbc4ecb355206300463f7568ea2364660 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 17 May 2024 08:37:15 +0200 Subject: [PATCH 29/45] support 4.28.0 --- ....json => js_data_ch_4_28.0_generated.json} | 41 ++--- docs/4_28_0/js_data_le_4_28.0_generated.json | 141 ++++++++++++++++++ models/jsdata_le_pojo.py | 13 +- models/jsdata_pojo.py | 13 +- queue_message/appointmentrequestsender.py | 7 +- queue_message/parallel_requestsender.py | 4 +- utils/generate_js_data.py | 2 +- 7 files changed, 178 insertions(+), 43 deletions(-) rename docs/4_28_0/{js_data_ch_4_27.0_generated.json => js_data_ch_4_28.0_generated.json} (66%) create mode 100644 docs/4_28_0/js_data_le_4_28.0_generated.json diff --git a/docs/4_28_0/js_data_ch_4_27.0_generated.json b/docs/4_28_0/js_data_ch_4_28.0_generated.json similarity index 66% rename from docs/4_28_0/js_data_ch_4_27.0_generated.json rename to docs/4_28_0/js_data_ch_4_28.0_generated.json index e9c5457..288f466 100644 --- a/docs/4_28_0/js_data_ch_4_27.0_generated.json +++ b/docs/4_28_0/js_data_ch_4_28.0_generated.json @@ -1,29 +1,29 @@ { - "ttst": 43, + "ttst": 138, "ifov": false, - "hc": 4, - "br_oh": 663, - "br_ow": 384, - "ua": "Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36", + "hc": 5, + "br_oh": 620, + "br_ow": 360, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36", "wbd": false, - "tagpu": 5.167040384311697, + "tagpu": 9.423374907896758, "wdif": false, "wdifrm": false, "npmtm": false, - "br_h": 663, - "br_w": 384, + "br_h": 620, + "br_w": 360, "isf": true, "nddc": 1, - "rs_h": 811, - "rs_w": 384, + "rs_h": 804, + "rs_w": 360, "rs_cd": 24, "phe": false, "nm": false, "jsf": false, "lg": "fr-FR", - "pr": 2.8125, - "ars_h": 811, - "ars_w": 384, + "pr": 3, + "ars_h": 804, + "ars_w": 360, "tz": -120, "str_ss": true, "str_ls": true, @@ -51,7 +51,7 @@ "vnd": "Google Inc.", "bid": "NA", "mmt": "empty", - "plu": "c1iRnTJ,YUxg36GL", + "plu": "eTJjZUp,iZMOHLs1", "hdn": false, "awe": false, "geb": false, @@ -116,11 +116,12 @@ "emt": false, "bfr": false, "dbov": false, - "cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbbigyNjApXShuKDI5NCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1tpKDI2MyldW2koMjYwKV0oJ1x4NjJceDcyXHg2Zlx4NzdceDczXHg2NVx4NzJceDY2XHg2Y1x4NmZceDc3XHgy", - "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzYzNjQpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODApCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxOTI4", + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzc0MjUpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk5NTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUyOTU3", "ckwa": true, - "glvd": "Qualcomm", - "glrd": "Adreno (TM) 630", + "emd": "k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801", + "glvd": "ARM", + "glrd": "Mali-G57 MC2", "wwl": false, - "jset": 1715011727 -} \ No newline at end of file + "jset": 1715927413 +} diff --git a/docs/4_28_0/js_data_le_4_28.0_generated.json b/docs/4_28_0/js_data_le_4_28.0_generated.json new file mode 100644 index 0000000..d2ab5e3 --- /dev/null +++ b/docs/4_28_0/js_data_le_4_28.0_generated.json @@ -0,0 +1,141 @@ +{ + "ttst": 50, + "ifov": false, + "hc": 5, + "br_oh": 620, + "br_ow": 360, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 4.919088817133858, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 620, + "br_w": 360, + "isf": true, + "nddc": 1, + "rs_h": 804, + "rs_w": 360, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 804, + "ars_w": 360, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "eTJjZUp,iZMOHLs1", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 4, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzc0MjUpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk5NTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUyOTU3", + "ckwa": true, + "emd": "k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801", + "glvd": "ARM", + "glrd": "Mali-G57 MC2", + "wwl": false, + "jset": 1715927566, + "dcok": ".hermes.com", + "log2": true, + "m_fmi": false, + "tbce": 0, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "m_s_c": 0, + "m_m_c": 3, + "m_c_c": 3, + "m_cm_r": 1, + "m_ms_r": -1 +} diff --git a/models/jsdata_le_pojo.py b/models/jsdata_le_pojo.py index f663c80..cf78278 100644 --- a/models/jsdata_le_pojo.py +++ b/models/jsdata_le_pojo.py @@ -20,9 +20,9 @@ from models.jsdata_pojo import JsDataPojo class JsDataLeTypePojo(JsDataPojo): def __init__(self, glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu, - vnd, pr, ts_mtp, dvm, m_s_c, m_m_c, m_c_c, m_cm_r, m_ms_r): + vnd, pr, ts_mtp, dvm, m_s_c, m_m_c, m_c_c, m_cm_r, m_ms_r, emd: str): super().__init__(glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu, - vnd, pr, ts_mtp, dvm) + vnd, pr, ts_mtp, dvm, emd=emd) self.dcok = ".hermes.com" self.log2 = True self.m_fmi = False @@ -41,12 +41,7 @@ class JsDataLeTypePojo(JsDataPojo): self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzc0MjUpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk5NTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUyOTU3" # 4.28.0 -test_data_json = """{"glvd": "Qualcomm", "glrd": "Adreno (TM) 630", "hc": 2, "br_oh": 663, "br_ow": 384, "br_h": 663, - "br_w": 384, "rs_h": 811, "rs_w": 384, "rs_cd": 24, "ars_h": 811, "ars_w": 384, "plg": 2, "eva": 33, - "vnd": "Google Inc.", "plu": ["5gQIEKN", "Iv26GLka"], - "ua": "Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36", - "dvm": 4, "acw": "probably", "pr": 2.8125, "ts_mtp": 5, - "cookiesStr": "app.sig=Arp0SzXCTBF2ordHf3pPTd6cS14;datadome=o_7W8TjHcQjqvnFjFBGAZZ0OmNOd0fnJXfzgqOajAHBBNv7zNMFi2~Jn7wQbTEIZdr2qvg0IFFQQr~Zk5ifzM154V1XxZVhgDZeVkNyRKrQewLBIzVeSagWs587fBqdX;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoieVVRWmJBWnlRTnQ5bFlNbDdNN3R6eHhqIn0=;policy=accepted;lang=fr;"}""" +test_data_json = """{"glvd":"ARM","glrd":"Mali-G57 MC2","hc":5,"br_oh":620,"br_ow":360,"br_h":620,"br_w":360,"rs_h":804,"rs_w":360,"rs_cd":24,"ars_h":804,"ars_w":360,"plg":2,"eva":33,"vnd":"Google Inc.","plu":["eTJjZUp","iZMOHLs1"],"ua":"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36","dvm":4,"acw":"probably","pr":3,"ts_mtp":5,"tz":-120,"emd":"k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801","cookiesStr":"app.sig=_FqS3_SPN2D40Z4Qlq-_ybxzRvQ;datadome=VSGCNMYkY6L81typRJUpurlyd39dHxdfxIYc6DzRWDT7_rtZ8UodGcbow1okQR2zg_t4d_NcP0rC1ZHNz3XSDAGFZCJhFI49lai~x2Nb2unkhOlY8o4E04Wjmde~auDZ;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiLW9jbktLVXAzMjVTOEFEdFpLUU9kSXlMIn0=;policy=accepted;lang=fr;"}""" if __name__ == '__main__': test_data_dict = json.loads(test_data_json) @@ -58,5 +53,5 @@ if __name__ == '__main__': ts_mtp=test_data_dict['ts_mtp'], eva=test_data_dict['eva'], rs_h=test_data_dict['rs_h'], rs_w=test_data_dict['rs_w'], rs_cd=test_data_dict['rs_cd'], m_s_c=0, m_m_c=3, m_c_c=3, - m_cm_r=1, m_ms_r=-1) + m_cm_r=1, m_ms_r=-1, emd=test_data_dict['emd']) print(le_js_data.to_url_encoded_json()) diff --git a/models/jsdata_pojo.py b/models/jsdata_pojo.py index d815f2e..83ed6f3 100644 --- a/models/jsdata_pojo.py +++ b/models/jsdata_pojo.py @@ -130,13 +130,14 @@ class JsDataPojo: cfpfe: str stcfp: str ckwa: bool + emd: str glvd: str glrd: str wwl: bool jset: int def __init__(self, glrd, glvd, hc, br_oh, ua, br_ow, br_h, br_w, rs_h, rs_w, rs_cd, ars_h, ars_w, plg, eva, plu, - vnd, pr, ts_mtp, dvm): + vnd, pr, ts_mtp, dvm, emd): # self.opts = "endpoint,ajaxListenerPath" self.ttst = random.randint(38, 148) self.ifov = False @@ -236,6 +237,7 @@ class JsDataPojo: self.vc1ts = True self.dvm = dvm self.set_default_values() + self.emd = emd self.glvd = glvd self.glrd = glrd self.wwl = False @@ -275,12 +277,7 @@ class JsDataPojo: return _safe_string -test_data_json = """{"glvd": "Qualcomm", "glrd": "Adreno (TM) 630", "hc": 2, "br_oh": 663, "br_ow": 384, "br_h": 663, - "br_w": 384, "rs_h": 811, "rs_w": 384, "rs_cd": 24, "ars_h": 811, "ars_w": 384, "plg": 2, "eva": 33, - "vnd": "Google Inc.", "plu": ["5gQIEKN", "Iv26GLka"], - "ua": "Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36", - "dvm": 4, "acw": "probably", "pr": 2.8125, "ts_mtp": 5, - "cookiesStr": "app.sig=Arp0SzXCTBF2ordHf3pPTd6cS14;datadome=o_7W8TjHcQjqvnFjFBGAZZ0OmNOd0fnJXfzgqOajAHBBNv7zNMFi2~Jn7wQbTEIZdr2qvg0IFFQQr~Zk5ifzM154V1XxZVhgDZeVkNyRKrQewLBIzVeSagWs587fBqdX;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoieVVRWmJBWnlRTnQ5bFlNbDdNN3R6eHhqIn0=;policy=accepted;lang=fr;"}""" +test_data_json = """{"glvd":"ARM","glrd":"Mali-G57 MC2","hc":5,"br_oh":620,"br_ow":360,"br_h":620,"br_w":360,"rs_h":804,"rs_w":360,"rs_cd":24,"ars_h":804,"ars_w":360,"plg":2,"eva":33,"vnd":"Google Inc.","plu":["eTJjZUp","iZMOHLs1"],"ua":"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36","dvm":4,"acw":"probably","pr":3,"ts_mtp":5,"tz":-120,"emd":"k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801","cookiesStr":"app.sig=_FqS3_SPN2D40Z4Qlq-_ybxzRvQ;datadome=VSGCNMYkY6L81typRJUpurlyd39dHxdfxIYc6DzRWDT7_rtZ8UodGcbow1okQR2zg_t4d_NcP0rC1ZHNz3XSDAGFZCJhFI49lai~x2Nb2unkhOlY8o4E04Wjmde~auDZ;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiLW9jbktLVXAzMjVTOEFEdFpLUU9kSXlMIn0=;policy=accepted;lang=fr;"}""" if __name__ == '__main__': test_data_dict = json.loads(test_data_json) @@ -290,7 +287,7 @@ if __name__ == '__main__': plg=test_data_dict['plg'], br_h=test_data_dict['br_h'], br_w=test_data_dict['br_w'], plu=test_data_dict['plu'], vnd=test_data_dict['vnd'], dvm=test_data_dict['dvm'], ts_mtp=test_data_dict['ts_mtp'], eva=test_data_dict['eva'], rs_h=test_data_dict['rs_h'], - rs_w=test_data_dict['rs_w'], rs_cd=test_data_dict['rs_cd']) + rs_w=test_data_dict['rs_w'], rs_cd=test_data_dict['rs_cd'], emd=test_data_dict['emd']) print(js.to_url_encoded_json()) # text_file = open("jsdata.txt", "w") # # write string to file diff --git a/queue_message/appointmentrequestsender.py b/queue_message/appointmentrequestsender.py index 1d962d6..9cd7559 100644 --- a/queue_message/appointmentrequestsender.py +++ b/queue_message/appointmentrequestsender.py @@ -109,7 +109,7 @@ class AppointmentRequestSender(threading.Thread): plu=_received_dict['plu'], vnd=_received_dict['vnd'], dvm=_received_dict['dvm'], ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'], rs_h=_received_dict['rs_h'], - rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd']) + rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'], emd=_received_dict['emd']) _received_cookies = _received_dict["cookiesStr"] # remove already sent contacts if self.just_send: @@ -125,7 +125,8 @@ class AppointmentRequestSender(threading.Thread): for con in self.contact_list: # _proxy_to_use = self.proxy_manager.get_proxy_for_appointment_request() # print(_proxy_to_use) - sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, received_dict=_received_dict, + sender = Sender(_received_cookies, cookiesPublisher=self.cookiesPublisher, + received_dict=_received_dict, proxy_to_use=_proxy_to_use, logger=self.logger) self.logger.info(con.mail) if self.valid_csrf is None: @@ -168,7 +169,7 @@ class AppointmentRequestSender(threading.Thread): rs_h=_received_dict['rs_h'], rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'], m_s_c=m_s_c, m_m_c=m_m_c, m_c_c=m_c_c, - m_cm_r=m_cm_r, m_ms_r=m_ms_r) + m_cm_r=m_cm_r, m_ms_r=m_ms_r, emd=_received_dict['emd']) time.sleep(random.randint(1, 4)) _new_le_cookies = captchaResultGetter.get_le_valid_cookie(proxy_to_use=_proxy_to_use, js_le_type_data=js_le_data, diff --git a/queue_message/parallel_requestsender.py b/queue_message/parallel_requestsender.py index 1c5ad23..02a3fc8 100644 --- a/queue_message/parallel_requestsender.py +++ b/queue_message/parallel_requestsender.py @@ -103,7 +103,7 @@ class ParallelRequestSender(threading.Thread): rs_h=_received_dict['rs_h'], rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'], m_s_c=m_s_c, m_m_c=m_m_c, m_c_c=m_c_c, - m_cm_r=m_cm_r, m_ms_r=m_ms_r) + m_cm_r=m_cm_r, m_ms_r=m_ms_r, emd=_received_dict['emd']) time.sleep(random.randint(1, 4)) _new_le_cookies = captchaResultGetter.get_le_valid_cookie(proxy_to_use=_proxy_to_use, js_le_type_data=js_le_data, @@ -149,7 +149,7 @@ class ParallelRequestSender(threading.Thread): plu=_received_dict['plu'], vnd=_received_dict['vnd'], dvm=_received_dict['dvm'], ts_mtp=_received_dict['ts_mtp'], eva=_received_dict['eva'], rs_h=_received_dict['rs_h'], - rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd']) + rs_w=_received_dict['rs_w'], rs_cd=_received_dict['rs_cd'], emd=_received_dict['emd']) def generate_proxy(self): _port = random.randint(40001, 49999) diff --git a/utils/generate_js_data.py b/utils/generate_js_data.py index 4ed23ed..f897f70 100644 --- a/utils/generate_js_data.py +++ b/utils/generate_js_data.py @@ -25,6 +25,6 @@ if __name__ == '__main__': ua = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36" jsData = JsDataPojo(glrd=glrd, hc=hc, ua=ua, br_oh=br_oh, br_ow=br_ow, br_h=br_h, br_w=br_w, rs_h=rs_h, rs_w=rs_w, eva=eva, - rs_cd=rs_cd, plg=plg, plu=plu, ars_h=ars_h, ars_w=ars_w, vnd=vnd, glvd=glvd) + rs_cd=rs_cd, plg=plg, plu=plu, ars_h=ars_h, ars_w=ars_w, vnd=vnd, glvd=glvd,emd="NA") print(jsData) \ No newline at end of file From 66a029da221879a79056975f49f6fa2640136c4c Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 17 May 2024 10:40:50 +0200 Subject: [PATCH 30/45] exclude data_impulase proxy --- proxy_manager/proxy_manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/proxy_manager/proxy_manager.py b/proxy_manager/proxy_manager.py index e46cbec..6c2473c 100644 --- a/proxy_manager/proxy_manager.py +++ b/proxy_manager/proxy_manager.py @@ -35,8 +35,9 @@ FR_DATA_IMPULSE_RES = { } # 八分之一用data_impulse MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, - FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, - FR_PROXY_DATA_IMPULSE_STICKY] + FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY + # FR_PROXY_DATA_IMPULSE_STICKY + ] class ProxyManager: From 75ed3f50639dc50816123f6660bf62031abed966 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 17 May 2024 11:15:08 +0200 Subject: [PATCH 31/45] exclude data_impulase proxy --- request_sender_test.py | 2 +- scheduler_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/request_sender_test.py b/request_sender_test.py index dc61d44..9d6b829 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -86,5 +86,5 @@ def start_send_requests(thread_number, file_path, data_queue_name=MORNING_DATA_C if __name__ == '__main__': - start_send_requests(thread_number=30, file_path='~/Desktop/contact_list_2024-05-14.xlsx', + start_send_requests(thread_number=30, file_path='~/Desktop/contact_list_2024-05-15.xlsx', data_queue_name=MORNING_DATA_CACHE_2) diff --git a/scheduler_test.py b/scheduler_test.py index 137da42..6f2bfae 100644 --- a/scheduler_test.py +++ b/scheduler_test.py @@ -5,7 +5,7 @@ from request_sender_test import start_send_requests def start_book_appointment(): - start_send_requests(thread_number=31, file_path='~/Desktop/contact_list_2024-05-14.xlsx') + start_send_requests(thread_number=31, file_path='~/Desktop/contact_list_2024-05-16.xlsx') def start_check_results_job(sched): From 3590a6fa401f5ff0d7fe086d1778452832b714b7 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Fri, 17 May 2024 18:27:52 +0200 Subject: [PATCH 32/45] test oxylabs res proxy --- docs/4_28_0/js_data_ch_4.28.0.json | 14 +++---- docs/4_28_0/js_data_ch_4_28.0_generated.json | 40 ++++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/docs/4_28_0/js_data_ch_4.28.0.json b/docs/4_28_0/js_data_ch_4.28.0.json index 5bb7646..22e1d29 100644 --- a/docs/4_28_0/js_data_ch_4.28.0.json +++ b/docs/4_28_0/js_data_ch_4.28.0.json @@ -1,12 +1,12 @@ { - "ttst": 43, + "ttst": 44, "ifov": false, - "hc": 5, + "hc": 7, "br_oh": 646, "br_ow": 360, "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", "wbd": false, - "tagpu": 0.29755279781407795, + "tagpu": 7.998561200498068, "wdif": false, "wdifrm": false, "npmtm": false, @@ -51,7 +51,7 @@ "vnd": "Google Inc.", "bid": "NA", "mmt": "empty", - "plu": "euAfPPm,zZzCJMtW", + "plu": "pc1a0Do7,zZrdWLN", "hdn": false, "awe": false, "geb": false, @@ -92,7 +92,7 @@ "vcqts": false, "vc1": "probably", "vc1ts": true, - "dvm": 0.5, + "dvm": 8, "sqt": false, "so": "portrait-primary", "wdw": true, @@ -119,9 +119,9 @@ "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", "stcfp": "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzc0MjUpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk5NTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUyOTU3", "ckwa": true, - "emd": "k:ai,ao,vi,vi d:defau,defau,d104b,f06c5 g:14588,defau,33a71,7143d", + "emd": "k:ai,ao,vi,vi d:defau,defau,d104b,f06c5 g:87354,defau,063a3,1bd2c", "glvd": "Google Inc. (Qualcomm)", "glrd": "ANGLE (Qualcomm, Adreno (TM) 650, OpenGL ES 3.2)", "wwl": false, - "jset": 1715895418 + "jset": 1715934750 } \ No newline at end of file diff --git a/docs/4_28_0/js_data_ch_4_28.0_generated.json b/docs/4_28_0/js_data_ch_4_28.0_generated.json index 288f466..7bdf37e 100644 --- a/docs/4_28_0/js_data_ch_4_28.0_generated.json +++ b/docs/4_28_0/js_data_ch_4_28.0_generated.json @@ -1,29 +1,29 @@ { - "ttst": 138, + "ttst": 88, "ifov": false, - "hc": 5, - "br_oh": 620, - "br_ow": 360, - "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36", + "hc": 3, + "br_oh": 786, + "br_ow": 432, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", "wbd": false, - "tagpu": 9.423374907896758, + "tagpu": 2.659617201677909, "wdif": false, "wdifrm": false, "npmtm": false, - "br_h": 620, - "br_w": 360, + "br_h": 786, + "br_w": 432, "isf": true, "nddc": 1, - "rs_h": 804, - "rs_w": 360, + "rs_h": 984, + "rs_w": 432, "rs_cd": 24, "phe": false, "nm": false, "jsf": false, "lg": "fr-FR", - "pr": 3, - "ars_h": 804, - "ars_w": 360, + "pr": 2.5, + "ars_h": 984, + "ars_w": 432, "tz": -120, "str_ss": true, "str_ls": true, @@ -51,7 +51,7 @@ "vnd": "Google Inc.", "bid": "NA", "mmt": "empty", - "plu": "eTJjZUp,iZMOHLs1", + "plu": "0Do78e2b,SRIECgQ", "hdn": false, "awe": false, "geb": false, @@ -92,7 +92,7 @@ "vcqts": false, "vc1": "probably", "vc1ts": true, - "dvm": 4, + "dvm": 2, "sqt": false, "so": "portrait-primary", "wdw": true, @@ -119,9 +119,9 @@ "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", "stcfp": "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzc0MjUpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk5NTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUyOTU3", "ckwa": true, - "emd": "k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801", - "glvd": "ARM", - "glrd": "Mali-G57 MC2", + "emd": "k:ai,ao,vi,vi d:defau,defau,99cea,25fa3 g:69b6f,defau,ed059,57073", + "glvd": "Qualcomm", + "glrd": "Adreno (TM) 619", "wwl": false, - "jset": 1715927413 -} + "jset": 1715953162 +} \ No newline at end of file From 40a02e8bcf05f694e61e0efe597484ffa68c6732 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Sat, 18 May 2024 09:24:08 +0200 Subject: [PATCH 33/45] log oxylabs res proxy --- workers/sender.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/workers/sender.py b/workers/sender.py index f35b034..894d07b 100644 --- a/workers/sender.py +++ b/workers/sender.py @@ -37,7 +37,10 @@ class Sender: firstName=contact.first_name, lastName=contact.last_name, email=contact.mail, passport=contact.passport, ccid=contact.ccid) if "oxylabs" in self.proxy_to_use["http"]: - result.proxy = "oxylabs" + if "mob" in self.proxy_to_use["http"]: + result.proxy = "oxylabs_mob" + else: + result.proxy = "oxylabs_res" else: result.proxy = "data_impulse" result.id = id From 21122ea5404460fca5511a4931fd2929fe5074c6 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Sat, 18 May 2024 11:51:27 +0200 Subject: [PATCH 34/45] log oxylabs res proxy --- docs/note.md | 15 +++++++++++++++ proxy_manager/proxy_manager.py | 6 +++++- workers/link_validator_with_provided_list.py | 13 +++++++------ 3 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 docs/note.md diff --git a/docs/note.md b/docs/note.md new file mode 100644 index 0000000..b2db5e1 --- /dev/null +++ b/docs/note.md @@ -0,0 +1,15 @@ +##stcfp +获得stcfp的方法，但是取决于tags.js的版本 +let stcfp = "" +try { + null[0]; +} catch (n) { + //if ("string" != typeof n[["stack"]]) return t[["apply"]](this, arguments); + stcfp = window.btoa(n[["stack"]].slice(-150)); + var w = n[["stack"]].split("\n"); +// if (i) try { + // w.length > 1 && c[["test"]](w[2]) && (e[["cfpp"]] = true, d("asyncChallengeFinished")), w.length > 2 && l[["test"]](w[w.length - 3]) && (e[["cfcpw"]] = true, d("asyncChallengeFinished")), w.length > 8 && //h[["test"]](w[w.length - 4]) && (e[["cfse"]] = true, d("asyncChallengeFinished")); + //} catch (e) {} else if (a) try { + // w.length > 2 && l[["test"]](w[w.length - 3]) && (e[["cffpw"]] = true, d("asyncChallengeFinished")); + //} catch (e) {} +} diff --git a/proxy_manager/proxy_manager.py b/proxy_manager/proxy_manager.py index 6c2473c..85181ff 100644 --- a/proxy_manager/proxy_manager.py +++ b/proxy_manager/proxy_manager.py @@ -29,13 +29,17 @@ FR_PROXY_DATA_IMPULSE_STICKY = { 'https': 'http://1d568220dbefeff21ad4__cr.fr:df99489a25fa72d9@gw.dataimpulse.com:{}' } +FR_PROXY_RES_OXY_STICKY = { + 'http': 'http://customer-rendezvous-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}', + 'https': 'http://customer-rendezvous-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}' +} FR_DATA_IMPULSE_RES = { 'http': 'http://ac2b8183c3b02714cbd1__cr.fr:336e7f8f4b17e5bd@gw.dataimpulse.com:823', 'https': 'http://ac2b8183c3b02714cbd1__cr.fr:336e7f8f4b17e5bd@gw.dataimpulse.com:823' } # 八分之一用data_impulse MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, - FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY + FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_RES_OXY_STICKY # FR_PROXY_DATA_IMPULSE_STICKY ] diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index 9fb8c0c..fc1b73b 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -220,7 +220,7 @@ def validate_all_links(): _segment_number = int(len(_first_25_percent_links) / divided) else: _segment_number = 1 - last_thread = None + _thread_list = [] for i in range(0, _segment_number): logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_25_percent_links))) logger.info("segment is {}".format(i)) @@ -229,10 +229,10 @@ def validate_all_links(): _step = int(len(_first_25_percent_links) / _segment_number) _sublist = _first_25_percent_links[i * _step:_step * (i + 1)] _thread1 = threading.Thread(target=validate_links, args=(_cookiesPublisher, MORNING_DATA_CACHE_BAK, _sublist)) - last_thread = _thread1 + _thread_list.append(_thread1) _thread1.start() - if last_thread is not None: - last_thread.join() + for _thread in _thread_list: + _thread.join() if __name__ == '__main__': @@ -240,5 +240,6 @@ if __name__ == '__main__': while True: print("call validate_all_links()") validate_all_links() - print("wait for 30 seconds") - time.sleep(30) + delay = random.randint(10, 30) + print("wait for {} seconds".format(delay)) + time.sleep(delay) From 7225bf056e2d8e519e85f1aaba19c7b9f0cfbf6a Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 20 May 2024 11:10:27 +0200 Subject: [PATCH 35/45] correction for param name --- request_sender_test.py | 11 +++++++---- scheduler_test.py | 9 +++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/request_sender_test.py b/request_sender_test.py index ce0555d..ad07aae 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -85,7 +85,7 @@ def start_send_requests(thread_number, contact_list, data_queue_name=MORNING_DAT _thread.join() -def send_request_for_file_list(_file_list: list[str], thread_number: int = 20, data_queue_name=MORNING_DATA_CACHE): +def send_request_for_file_list(file_list: list[str], thread_number: int = 20, data_queue_name=MORNING_DATA_CACHE): for _file_path in file_list: logger.info("send request for file: " + _file_path) _contact_list = read_contacts(_file_path) @@ -94,6 +94,9 @@ def send_request_for_file_list(_file_list: list[str], thread_number: int = 20, d if __name__ == '__main__': - file_list = ['~/Desktop/09_05_to_test.xlsx','~/Desktop/contact_list_2024-05-13.xlsx'] - send_request_for_file_list(_file_list=file_list, thread_number=5, - data_queue_name=MORNING_DATA_CACHE_2) + file_list = ['~/Desktop/contact_list_2024-05-19.xlsx', '~/Desktop/contact_list_2024-05-17.xlsx', + '~/Desktop/15_05_to_test.xlsx'] + # file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx'] + # file_list = ['~/Desktop/contact_all_old.xlsx'] + send_request_for_file_list(file_list=file_list, thread_number=32, + data_queue_name=MORNING_DATA_CACHE) diff --git a/scheduler_test.py b/scheduler_test.py index 6f2bfae..2ffb563 100644 --- a/scheduler_test.py +++ b/scheduler_test.py @@ -1,11 +1,16 @@ from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor from apscheduler.schedulers.blocking import BlockingScheduler -from request_sender_test import start_send_requests +from queue_message.CookiesPublisher import MORNING_DATA_CACHE +from request_sender_test import send_request_for_file_list def start_book_appointment(): - start_send_requests(thread_number=31, file_path='~/Desktop/contact_list_2024-05-16.xlsx') + file_list = ['~/Desktop/contact_list_2024-05-19.xlsx', + '~/Desktop/contact_list_2024-05-17.xlsx', + '~/Desktop/15_05_to_test.xlsx'] + send_request_for_file_list(_file_list=file_list, thread_number=32, + data_queue_name=MORNING_DATA_CACHE) def start_check_results_job(sched): From e0bc031456f1adb1126fd52f13e058368e72ffda Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Tue, 21 May 2024 01:01:12 +0200 Subject: [PATCH 36/45] correction for param name --- scheduler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scheduler_test.py b/scheduler_test.py index 2ffb563..dea26e9 100644 --- a/scheduler_test.py +++ b/scheduler_test.py @@ -9,7 +9,7 @@ def start_book_appointment(): file_list = ['~/Desktop/contact_list_2024-05-19.xlsx', '~/Desktop/contact_list_2024-05-17.xlsx', '~/Desktop/15_05_to_test.xlsx'] - send_request_for_file_list(_file_list=file_list, thread_number=32, + send_request_for_file_list(file_list=file_list, thread_number=32, data_queue_name=MORNING_DATA_CACHE) From d56af46314c75d7759614d52da1a12e9b6cfff9e Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Tue, 21 May 2024 15:31:59 +0200 Subject: [PATCH 37/45] try to wait for more after 20h and before 9 h --- workers/link_validator_with_provided_list.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index fc1b73b..318b72d 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -3,6 +3,7 @@ import logging import random import threading import time +from datetime import datetime from http.cookies import SimpleCookie import pika @@ -202,13 +203,15 @@ def validate_links(cookiesPublisher, queue_name: str, link_list: list): receiver.listen_to_queue(receiver.on_message) +# default_segment_number 并发数，决定速度 +# divided = 4，越小，一次处理得越多 def validate_all_links(): print("will get all links") all_link_list = MONGO_STORE_MANAGER.get_links_to_validate() # get the first 50 links if len(all_link_list) == 0: return - divided = 4 + divided = 3 default_segment_number = 20 _first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))] _queue_name = MORNING_DATA_CACHE_BAK @@ -241,5 +244,11 @@ if __name__ == '__main__': print("call validate_all_links()") validate_all_links() delay = random.randint(10, 30) + current_time = datetime.now() + current_hour = current_time.hour + print("Current hour ", current_time.hour) + print("Current minute ", current_time.minute) + if current_hour > 20 or current_hour < 9: + delay = 3600 print("wait for {} seconds".format(delay)) time.sleep(delay) From b96b6522d89b3d4ad6ab9c7e97b76c969fe3d1ac Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Tue, 21 May 2024 15:34:14 +0200 Subject: [PATCH 38/45] timezone utc --- workers/link_validator_with_provided_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index 318b72d..558ba71 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -248,7 +248,7 @@ if __name__ == '__main__': current_hour = current_time.hour print("Current hour ", current_time.hour) print("Current minute ", current_time.minute) - if current_hour > 20 or current_hour < 9: + if current_hour > 18 or current_hour < 7: delay = 3600 print("wait for {} seconds".format(delay)) time.sleep(delay) From b537ba670395c18244e11967c33940ae888e1bbd Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Wed, 22 May 2024 23:52:06 +0200 Subject: [PATCH 39/45] multi files --- request_sender_test.py | 6 ++++-- scheduler_test.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/request_sender_test.py b/request_sender_test.py index ad07aae..1e84e3d 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -94,9 +94,11 @@ def send_request_for_file_list(file_list: list[str], thread_number: int = 20, da if __name__ == '__main__': - file_list = ['~/Desktop/contact_list_2024-05-19.xlsx', '~/Desktop/contact_list_2024-05-17.xlsx', + file_list = ['~/Desktop/contact_list_2024-05-21.xlsx', + '~/Desktop/contact_list_2024-05-20.xlsx', + '~/Desktop/contact_list_2024-05-19.xlsx', '~/Desktop/15_05_to_test.xlsx'] # file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx'] # file_list = ['~/Desktop/contact_all_old.xlsx'] - send_request_for_file_list(file_list=file_list, thread_number=32, + send_request_for_file_list(file_list=file_list, thread_number=30, data_queue_name=MORNING_DATA_CACHE) diff --git a/scheduler_test.py b/scheduler_test.py index dea26e9..ccc7973 100644 --- a/scheduler_test.py +++ b/scheduler_test.py @@ -6,8 +6,9 @@ from request_sender_test import send_request_for_file_list def start_book_appointment(): - file_list = ['~/Desktop/contact_list_2024-05-19.xlsx', - '~/Desktop/contact_list_2024-05-17.xlsx', + file_list = ['~/Desktop/contact_list_2024-05-21.xlsx', + '~/Desktop/contact_list_2024-05-20.xlsx', + '~/Desktop/contact_list_2024-05-19.xlsx', '~/Desktop/15_05_to_test.xlsx'] send_request_for_file_list(file_list=file_list, thread_number=32, data_queue_name=MORNING_DATA_CACHE) From fe137d5a666009b405123214ec45de0d2b6a506a Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Thu, 23 May 2024 00:17:35 +0200 Subject: [PATCH 40/45] support 4.29.0 --- docs/{ => 4.26.0}/js_data_ch_4.26.0.json | 0 .../js_data_ch_4.26.0_generated.json | 0 docs/{ => 4.26.0}/js_data_le_4.26.0.json | 0 .../js_data_le_4.26.0_generated.json | 0 docs/4.29.0/jsdata_ch_4_29_0.json | 127 ++++++++++++++++ docs/4.29.0/jsdata_le_4_29_0.json | 139 ++++++++++++++++++ docs/4.29.0/jsdata_le_4_29_0_2.json | 139 ++++++++++++++++++ docs/note.md | 8 +- models/jsdata_le_pojo.py | 8 +- models/jsdata_pojo.py | 4 +- request_sender.py | 2 +- workers/captcha_result_getter.py | 4 +- 12 files changed, 421 insertions(+), 10 deletions(-) rename docs/{ => 4.26.0}/js_data_ch_4.26.0.json (100%) rename docs/{ => 4.26.0}/js_data_ch_4.26.0_generated.json (100%) rename docs/{ => 4.26.0}/js_data_le_4.26.0.json (100%) rename docs/{ => 4.26.0}/js_data_le_4.26.0_generated.json (100%) create mode 100644 docs/4.29.0/jsdata_ch_4_29_0.json create mode 100644 docs/4.29.0/jsdata_le_4_29_0.json create mode 100644 docs/4.29.0/jsdata_le_4_29_0_2.json diff --git a/docs/js_data_ch_4.26.0.json b/docs/4.26.0/js_data_ch_4.26.0.json similarity index 100% rename from docs/js_data_ch_4.26.0.json rename to docs/4.26.0/js_data_ch_4.26.0.json diff --git a/docs/js_data_ch_4.26.0_generated.json b/docs/4.26.0/js_data_ch_4.26.0_generated.json similarity index 100% rename from docs/js_data_ch_4.26.0_generated.json rename to docs/4.26.0/js_data_ch_4.26.0_generated.json diff --git a/docs/js_data_le_4.26.0.json b/docs/4.26.0/js_data_le_4.26.0.json similarity index 100% rename from docs/js_data_le_4.26.0.json rename to docs/4.26.0/js_data_le_4.26.0.json diff --git a/docs/js_data_le_4.26.0_generated.json b/docs/4.26.0/js_data_le_4.26.0_generated.json similarity index 100% rename from docs/js_data_le_4.26.0_generated.json rename to docs/4.26.0/js_data_le_4.26.0_generated.json diff --git a/docs/4.29.0/jsdata_ch_4_29_0.json b/docs/4.29.0/jsdata_ch_4_29_0.json new file mode 100644 index 0000000..7ca2721 --- /dev/null +++ b/docs/4.29.0/jsdata_ch_4_29_0.json @@ -0,0 +1,127 @@ +{ + "ttst": 42, + "ifov": false, + "hc": 7, + "br_oh": 646, + "br_ow": 360, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 3.5620980093605126, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 646, + "br_w": 360, + "isf": true, + "nddc": 1, + "rs_h": 800, + "rs_w": 360, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 800, + "ars_w": 360, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "pc1a0Do7,zZrdWLN", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 8, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "ZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzY1MDcpCiAgICBhdCB0Ljxjb21wdXRlZD4uZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxNjk2", + "ckwa": true, + "emd": "k:ai,ao,vi,vi d:defau,defau,d104b,f06c5 g:396ea,defau,81918,33ce0", + "glvd": "Google Inc. (Qualcomm)", + "glrd": "ANGLE (Qualcomm, Adreno (TM) 650, OpenGL ES 3.2)", + "wwl": false, + "jset": 1716415160 +} \ No newline at end of file diff --git a/docs/4.29.0/jsdata_le_4_29_0.json b/docs/4.29.0/jsdata_le_4_29_0.json new file mode 100644 index 0000000..980ca4c --- /dev/null +++ b/docs/4.29.0/jsdata_le_4_29_0.json @@ -0,0 +1,139 @@ +{ + "ttst": 42, + "ifov": false, + "hc": 7, + "br_oh": 646, + "br_ow": 360, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 3.5620980093605126, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 646, + "br_w": 360, + "isf": true, + "nddc": 1, + "rs_h": 800, + "rs_w": 360, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 800, + "ars_w": 360, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "pc1a0Do7,zZrdWLN", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 8, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "ZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzY1MDcpCiAgICBhdCB0Ljxjb21wdXRlZD4uZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxNjk2", + "ckwa": true, + "emd": "k:ai,ao,vi,vi d:defau,defau,d104b,f06c5 g:396ea,defau,81918,33ce0", + "glvd": "Google Inc. (Qualcomm)", + "glrd": "ANGLE (Qualcomm, Adreno (TM) 650, OpenGL ES 3.2)", + "wwl": false, + "jset": 1716415179, + "dcok": ".hermes.com", + "log2": true, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "m_s_c": 110, + "m_m_c": 0, + "m_c_c": 0, + "m_cm_r": -1, + "m_ms_r": 0 +} \ No newline at end of file diff --git a/docs/4.29.0/jsdata_le_4_29_0_2.json b/docs/4.29.0/jsdata_le_4_29_0_2.json new file mode 100644 index 0000000..545a315 --- /dev/null +++ b/docs/4.29.0/jsdata_le_4_29_0_2.json @@ -0,0 +1,139 @@ +{ + "ttst": 25, + "ifov": false, + "hc": 7, + "br_oh": 646, + "br_ow": 360, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 6.426279443512415, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 646, + "br_w": 360, + "isf": true, + "nddc": 1, + "rs_h": 800, + "rs_w": 360, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 800, + "ars_w": 360, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "pc1a0Do7,zZrdWLN", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 8, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "ZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzY1MDcpCiAgICBhdCB0Ljxjb21wdXRlZD4uZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxNjk2", + "ckwa": true, + "emd": "k:ai,ao,vi,vi d:defau,defau,d104b,f06c5 g:c9148,defau,1c35e,61dc5", + "glvd": "Google Inc. (Qualcomm)", + "glrd": "ANGLE (Qualcomm, Adreno (TM) 650, OpenGL ES 3.2)", + "wwl": false, + "jset": 1716415693, + "dcok": ".hermes.com", + "log2": true, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "m_s_c": 120, + "m_m_c": 0, + "m_c_c": 0, + "m_cm_r": -1, + "m_ms_r": 0 +} \ No newline at end of file diff --git a/docs/note.md b/docs/note.md index b2db5e1..56dd48e 100644 --- a/docs/note.md +++ b/docs/note.md @@ -1,4 +1,5 @@ -##stcfp + +## stcfp 获得stcfp的方法，但是取决于tags.js的版本 let stcfp = "" try { @@ -13,3 +14,8 @@ try { // w.length > 2 && l[["test"]](w[w.length - 3]) && (e[["cffpw"]] = true, d("asyncChallengeFinished")); //} catch (e) {} } + + + +## 4.29.0 +在4.29.0的 jsdata le中找不到 *m_fmi* 和 *tbce*， 这2个属性在4.28.0中存在 \ No newline at end of file diff --git a/models/jsdata_le_pojo.py b/models/jsdata_le_pojo.py index cf78278..1b2432c 100644 --- a/models/jsdata_le_pojo.py +++ b/models/jsdata_le_pojo.py @@ -25,8 +25,8 @@ class JsDataLeTypePojo(JsDataPojo): vnd, pr, ts_mtp, dvm, emd=emd) self.dcok = ".hermes.com" self.log2 = True - self.m_fmi = False - self.tbce = 0 + # self.m_fmi = False 这2个字段在4.29.0中没有再出现了 + # self.tbce = 0 self.es_sigmdn = None self.es_mumdn = None self.es_distmdn = None @@ -37,8 +37,8 @@ class JsDataLeTypePojo(JsDataPojo): self.m_c_c = m_c_c self.m_cm_r = m_cm_r self.m_ms_r = m_ms_r - self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.28.0 - self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzc0MjUpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk5NTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUyOTU3" # 4.28.0 + self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.29.0 + self.stcfp = "ZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzY1MDcpCiAgICBhdCB0Ljxjb21wdXRlZD4uZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxNjk2" # 4.29.0 test_data_json = """{"glvd":"ARM","glrd":"Mali-G57 MC2","hc":5,"br_oh":620,"br_ow":360,"br_h":620,"br_w":360,"rs_h":804,"rs_w":360,"rs_cd":24,"ars_h":804,"ars_w":360,"plg":2,"eva":33,"vnd":"Google Inc.","plu":["eTJjZUp","iZMOHLs1"],"ua":"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36","dvm":4,"acw":"probably","pr":3,"ts_mtp":5,"tz":-120,"emd":"k:ai,vi,ao,vi d:defau,8f9de,defau,24476 g:4e652,f9991,defau,d5801","cookiesStr":"app.sig=_FqS3_SPN2D40Z4Qlq-_ybxzRvQ;datadome=VSGCNMYkY6L81typRJUpurlyd39dHxdfxIYc6DzRWDT7_rtZ8UodGcbow1okQR2zg_t4d_NcP0rC1ZHNz3XSDAGFZCJhFI49lai~x2Nb2unkhOlY8o4E04Wjmde~auDZ;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiLW9jbktLVXAzMjVTOEFEdFpLUU9kSXlMIn0=;policy=accepted;lang=fr;"}""" diff --git a/models/jsdata_pojo.py b/models/jsdata_pojo.py index 83ed6f3..fc5b7fc 100644 --- a/models/jsdata_pojo.py +++ b/models/jsdata_pojo.py @@ -267,8 +267,8 @@ class JsDataPojo: self.emt = False self.bfr = False self.dbov = False - self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.28.0 - self.stcfp = "aWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6Nzc0MjUpCiAgICBhdCA8Y29tcHV0ZWQ+IFthcyBkZF9hY10gKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk5NTcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUyOTU3" # 4.28.0 + self.cfpfe = "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2" # 4.29.0 + self.stcfp = "ZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzY1MDcpCiAgICBhdCB0Ljxjb21wdXRlZD4uZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxNjk2" # 4.29.0 self.ckwa = True def to_url_encoded_json(self): diff --git a/request_sender.py b/request_sender.py index 25077ae..2826c30 100644 --- a/request_sender.py +++ b/request_sender.py @@ -63,7 +63,7 @@ def send_appointment_request(message_queue_name, _contact_list): logger.info(_contact) _cookiesPublisher = CookiesPublisher(queue_name=message_queue_name) _cookiesPublisher.set_up_connection() - receiver = AppointmentRequestSender(sub_contact_list=_contact_list, queue_name=message_queue_name, just_send=True, + receiver = AppointmentRequestSender(sub_contact_list=_contact_list, queue_name=message_queue_name, cookiesPublisher=_cookiesPublisher, logger=logger) print("count is " + str(count)) receiver.run() diff --git a/workers/captcha_result_getter.py b/workers/captcha_result_getter.py index a563227..ce6a9c1 100644 --- a/workers/captcha_result_getter.py +++ b/workers/captcha_result_getter.py @@ -105,7 +105,7 @@ class CaptchaResultGetter: return None def get_ch_raw_data_from_js_data(self, js_data: JsDataPojo, old_valid_cookie) -> str: - _tag_version = "4.28.0" + _tag_version = "4.29.0" _raw_data = "jsData={}&eventCounters=%5B%5D&jsType=ch&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv={}".format( js_data.to_url_encoded_json(), old_valid_cookie, _tag_version) print("raw ch data is " + _raw_data) @@ -172,7 +172,7 @@ class CaptchaResultGetter: # _le_js_raw_data = self.get_le_raw_data_from_js_data(js_le_type_data=js_le_type_data, # old_valid_cookie=old_valid_cookie) _cid = get_datadome_cookies(old_valid_cookie) - _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.28.0".format( + _raw_data = "jsData={}&eventCounters=%7B%22mousemove%22%3A{}%2C%22click%22%3A{}%2C%22scroll%22%3A{}%2C%22touchstart%22%3A{}%2C%22touchend%22%3A{}%2C%22touchmove%22%3A{}%2C%22keydown%22%3A{}%2C%22keyup%22%3A{}%7D&jsType=le&cid={}&ddk=789361B674144528D0B7EE76B35826&Referer=https%253A%252F%252Frendezvousparis.hermes.com%252Fclient%252Fregister&request=%252Fclient%252Fregister&responsePage=origin&ddv=4.29.0".format( js_le_type_data.to_url_encoded_json(), mousemove_count, click_count, scroll_count, touch_count, touch_count, touch_move, key_count, From e55251c043780af229d128172dcbba1d258ed160 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Thu, 23 May 2024 00:22:13 +0200 Subject: [PATCH 41/45] support 4.29.0 --- docs/4.29.0/jsdata_le_4_29_0_oneplus6.json | 141 +++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 docs/4.29.0/jsdata_le_4_29_0_oneplus6.json diff --git a/docs/4.29.0/jsdata_le_4_29_0_oneplus6.json b/docs/4.29.0/jsdata_le_4_29_0_oneplus6.json new file mode 100644 index 0000000..9d68e4c --- /dev/null +++ b/docs/4.29.0/jsdata_le_4_29_0_oneplus6.json @@ -0,0 +1,141 @@ +{ + "opts": "{\"endpoint\":\"https://d.digital.hermes/js/\",\"ajaxListenerPath\":true}", + "xhr_opts": "", + "ttst": 71, + "ifov": false, + "hc": 8, + "br_oh": 663, + "br_ow": 384, + "ua": "Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 8.178250208073038, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 663, + "br_w": 384, + "isf": true, + "nddc": 1, + "rs_h": 811, + "rs_w": 384, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.8125, + "ars_h": 811, + "ars_w": 384, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "q899Hiw,ZMGiwYMO", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 4, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "ZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzY1MDcpCiAgICBhdCB0Ljxjb21wdXRlZD4uZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxNjk2", + "ckwa": true, + "emd": "k:ai,vi,ao,vi d:defau,9c510,defau,049c9 g:2a88d,21833,defau,fc0d0", + "glvd": "Google Inc. (Qualcomm)", + "glrd": "ANGLE (Qualcomm, Adreno (TM) 630, OpenGL ES 3.2)", + "wwl": false, + "jset": 1716416458, + "dcok": ".hermes.com", + "log2": true, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "m_s_c": 123, + "m_m_c": 0, + "m_c_c": 0, + "m_cm_r": -1, + "m_ms_r": 0 +} \ No newline at end of file From f9b40e99d19022da3e729626e3b695514ed3c774 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Tue, 28 May 2024 14:15:27 +0200 Subject: [PATCH 42/45] multi files --- request_sender_test.py | 10 +++++----- scheduler_test.py | 11 ++++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/request_sender_test.py b/request_sender_test.py index 1e84e3d..35782e6 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -94,11 +94,11 @@ def send_request_for_file_list(file_list: list[str], thread_number: int = 20, da if __name__ == '__main__': - file_list = ['~/Desktop/contact_list_2024-05-21.xlsx', - '~/Desktop/contact_list_2024-05-20.xlsx', - '~/Desktop/contact_list_2024-05-19.xlsx', - '~/Desktop/15_05_to_test.xlsx'] + # file_list = ['~/Desktop/contact_list_2024-05-23.xlsx', + # '~/Desktop/contact_list_2024-05-22.xlsx', + # '~/Desktop/contact_list_2024-05-21.xlsx', + # '~/Desktop/15_05_to_test.xlsx'] # file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx'] - # file_list = ['~/Desktop/contact_all_old.xlsx'] + file_list = ['~/Desktop/contact_list_all.xlsx'] send_request_for_file_list(file_list=file_list, thread_number=30, data_queue_name=MORNING_DATA_CACHE) diff --git a/scheduler_test.py b/scheduler_test.py index ccc7973..936bdaa 100644 --- a/scheduler_test.py +++ b/scheduler_test.py @@ -6,11 +6,12 @@ from request_sender_test import send_request_for_file_list def start_book_appointment(): - file_list = ['~/Desktop/contact_list_2024-05-21.xlsx', - '~/Desktop/contact_list_2024-05-20.xlsx', - '~/Desktop/contact_list_2024-05-19.xlsx', - '~/Desktop/15_05_to_test.xlsx'] - send_request_for_file_list(file_list=file_list, thread_number=32, + file_list = ['~/Desktop/contact_list_2024-05-27.xlsx', + '~/Desktop/contact_list_2024-05-25.xlsx' + '~/Desktop/contact_list_2024-05-24.xlsx', + '~/Desktop/contact_list_2024-05-23.xlsx', + '~/Desktop/contact_list_2024-05-22.xlsx'] + send_request_for_file_list(file_list=file_list, thread_number=40, data_queue_name=MORNING_DATA_CACHE) From c610f8d72751692255f9e7b5b4952f360662f8c6 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Tue, 28 May 2024 14:17:29 +0200 Subject: [PATCH 43/45] support 4.29.0 --- docs/4.29.0/jsdata_le_4_29_0_generated.json | 139 ++++++++++++++++++++ queue_message/CookiesPublisher.py | 1 + 2 files changed, 140 insertions(+) create mode 100644 docs/4.29.0/jsdata_le_4_29_0_generated.json diff --git a/docs/4.29.0/jsdata_le_4_29_0_generated.json b/docs/4.29.0/jsdata_le_4_29_0_generated.json new file mode 100644 index 0000000..f6da232 --- /dev/null +++ b/docs/4.29.0/jsdata_le_4_29_0_generated.json @@ -0,0 +1,139 @@ +{ + "ttst": 83, + "ifov": false, + "hc": 2, + "br_oh": 680, + "br_ow": 384, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 9.796197350378677, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 680, + "br_w": 384, + "isf": true, + "nddc": 1, + "rs_h": 854, + "rs_w": 384, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 2.8125, + "ars_h": 854, + "ars_w": 384, + "tz": -120, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "zZzCBn6l,oct9eu2", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 2, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgdD1kb2N1bWVudFsnXHg3MVx4NzVceDY1XHg3Mlx4NzlceDUzXHg2NVx4NmNceDY1XHg2M1x4NzRceDZmXHg3MiddKCdceDYyXHg3Mlx4NmZceDc3XHg3M1x4NjVceDcyXHg2Nlx4NmNceDZmXHg3N1x4MmRceDYzXHg2Zlx4NmVceDc0XHg2", + "stcfp": "ZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzY1MDcpCiAgICBhdCB0Ljxjb21wdXRlZD4uZGRfYWMgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODk3ODcpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxNjk2", + "ckwa": true, + "emd": "k:ai,vi,vi,ao,vi,vi d:defau,fefef,920f5,defau,6a718,633e3 g:f8b61,daaa0,4e9e0,defau,31d26,40f29", + "glvd": "Qualcomm", + "glrd": "Adreno (TM) 619", + "wwl": false, + "jset": 1716473093, + "dcok": ".hermes.com", + "log2": true, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "m_s_c": 0, + "m_m_c": 3, + "m_c_c": 3, + "m_cm_r": 1.0, + "m_ms_r": -1 +} \ No newline at end of file diff --git a/queue_message/CookiesPublisher.py b/queue_message/CookiesPublisher.py index 99295bb..46ffbf8 100644 --- a/queue_message/CookiesPublisher.py +++ b/queue_message/CookiesPublisher.py @@ -12,6 +12,7 @@ MORNING_DATA_CACHE_BAK = 'MORNING_DATA_CACHE_BAK' REQUEST_DATA_QUEUE_DE = 'REQUEST_DATA_DE' REQUEST_DATA_OBJECT = 'REQUEST_DATA_OBJECT' REGISTER_QUEUE = 'REGISTER_QUEUE' +RES_PROXY_DATA = 'RES_PROXY_DATA' credentials = pika.PlainCredentials('appointment', 'ZyuhJZ2xEYWhElhpJjy7YEpZGZwNYJz2fHIu') From 82ed4763f59b71c2507cfb4b244e79fca3064e63 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Tue, 28 May 2024 14:19:10 +0200 Subject: [PATCH 44/45] add missed , --- scheduler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scheduler_test.py b/scheduler_test.py index 936bdaa..656a5b3 100644 --- a/scheduler_test.py +++ b/scheduler_test.py @@ -7,7 +7,7 @@ from request_sender_test import send_request_for_file_list def start_book_appointment(): file_list = ['~/Desktop/contact_list_2024-05-27.xlsx', - '~/Desktop/contact_list_2024-05-25.xlsx' + '~/Desktop/contact_list_2024-05-25.xlsx', '~/Desktop/contact_list_2024-05-24.xlsx', '~/Desktop/contact_list_2024-05-23.xlsx', '~/Desktop/contact_list_2024-05-22.xlsx'] From 8fb2204591ff9b0c0324d88d0dd0f8509ef049b3 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Sat, 1 Jun 2024 09:49:43 +0200 Subject: [PATCH 45/45] reuse dataimpulse proxy --- proxy_manager/proxy_manager.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/proxy_manager/proxy_manager.py b/proxy_manager/proxy_manager.py index 85181ff..a337da5 100644 --- a/proxy_manager/proxy_manager.py +++ b/proxy_manager/proxy_manager.py @@ -39,9 +39,8 @@ FR_DATA_IMPULSE_RES = { } # 八分之一用data_impulse MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, - FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_RES_OXY_STICKY - # FR_PROXY_DATA_IMPULSE_STICKY - ] + FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_RES_OXY_STICKY, + FR_PROXY_DATA_IMPULSE_STICKY] class ProxyManager: @@ -51,9 +50,9 @@ class ProxyManager: def get_link_validate_proxy(self, links_to_validate: list) -> list: if len(links_to_validate) > 15: - return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES] + return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY] else: - return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES] + return [FR_PROXY_RES_OXY, FR_PROXY_ASOCK_RES_2, FR_DATA_IMPULSE_RES, FR_ASOCKS_MOBILE_PROXY] def get_proxy_for_appointment_request(self) -> dict: _chosen_proxy = random.choice(MOBILE_PROXY_LIST)