diff --git a/docs/js_data_23_01.json b/docs/js_data_23_01.json new file mode 100644 index 0000000..03f0064 --- /dev/null +++ b/docs/js_data_23_01.json @@ -0,0 +1,147 @@ +{ + "log1": false, + "opts": "endpoint,ajaxListenerPath", + "ttst": 41, + "ifov": false, + "hc": 3, + "br_oh": 646, + "br_ow": 360, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 3.9694857617239587, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 646, + "br_w": 360, + "nddc": 1, + "rs_h": 800, + "rs_w": 360, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 800, + "ars_w": 360, + "tz": -60, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "WyZrdOm,EtePPm6d", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 8, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "RXJyb3I6IENhbm5vdCByZWFkIHByb3BlcnRpZXMgb2YgbnVsbCAocmVhZGluZyAndG9TdHJpbmcnKQ==", + "stcfp": "ckFsbCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo3NTIzNykKICAgIGF0IEhUTUxEb2N1bWVudC5yIChodHRwczovL3JlbmRlenZvdXNwYXJpcy5oZXJtZXMuY29tL2Rpc3QvdmVuZG9yLWM2Mjk1ZTljMjg3ZGExNzYwODllLmpzOjI6ODg2Nzkp", + "ckwa": true, + "glvd": "Google Inc. (Qualcomm)", + "glrd": "ANGLE (Qualcomm, Adreno (TM) 650, OpenGL ES 3.2)", + "wwl": false, + "jset": 1706032747, + "dcok": ".hermes.com", + "log2": true, + "mp_cx": 36, + "mp_cy": 405, + "mp_tr": true, + "mp_mx": 0, + "mp_my": 0, + "mp_sx": 36, + "mp_sy": 495, + "tbce": 0, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "m_s_c": 8, + "m_m_c": 4, + "m_c_c": 4, + "m_cm_r": 1, + "m_ms_r": 0.5 +} \ No newline at end of file diff --git a/docs/jsdata_4_19_2.json b/docs/jsdata_4_19_2.json new file mode 100644 index 0000000..9639bc7 --- /dev/null +++ b/docs/jsdata_4_19_2.json @@ -0,0 +1,140 @@ +{ + "log1": false, + "opts": "endpoint,ajaxListenerPath", + "ttst": 38, + "ifov": false, + "hc": 2, + "br_oh": 646, + "br_ow": 360, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 3.457033231955487, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 646, + "br_w": 360, + "nddc": 1, + "rs_h": 800, + "rs_w": 360, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 800, + "ars_w": 360, + "tz": -60, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "QIMtWyh,MOHDBny4", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 4, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "RXJyb3I6IENhbm5vdCByZWFkIHByb3BlcnRpZXMgb2YgbnVsbCAocmVhZGluZyAndG9TdHJpbmcnKQ==", + "stcfp": "ckFsbCAoaHR0cHM6Ly9kLmRpZ2l0YWwuaGVybWVzL3RhZ3MuanM6Mjo3NTEyMikKICAgIGF0IEhUTUxEb2N1bWVudC5yIChodHRwczovL3JlbmRlenZvdXNwYXJpcy5oZXJtZXMuY29tL2Rpc3QvdmVuZG9yLWM2Mjk1ZTljMjg3ZGExNzYwODllLmpzOjI6ODg2Nzkp", + "ckwa": true, + "glvd": "Google Inc. (Qualcomm)", + "glrd": "ANGLE (Qualcomm, Adreno (TM) 650, OpenGL ES 3.2)", + "wwl": false, + "jset": 1705750267, + "dcok": ".hermes.com", + "log2": true, + "tbce": 0, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "m_s_c": 3, + "m_m_c": 5, + "m_c_c": 5, + "m_cm_r": 1, + "m_ms_r": 1.6666666666666667 +} diff --git a/docs/jsdata_4_19_3.json b/docs/jsdata_4_19_3.json new file mode 100644 index 0000000..99647c0 --- /dev/null +++ b/docs/jsdata_4_19_3.json @@ -0,0 +1,140 @@ +{ + "log1": false, + "opts": "endpoint,ajaxListenerPath", + "ttst": 41, + "ifov": false, + "hc": 7, + "br_oh": 646, + "br_ow": 360, + "ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36", + "wbd": false, + "tagpu": 4.241117852528069, + "wdif": false, + "wdifrm": false, + "npmtm": false, + "br_h": 646, + "br_w": 360, + "nddc": 1, + "rs_h": 800, + "rs_w": 360, + "rs_cd": 24, + "phe": false, + "nm": false, + "jsf": false, + "lg": "fr-FR", + "pr": 3, + "ars_h": 800, + "ars_w": 360, + "tz": -60, + "str_ss": true, + "str_ls": true, + "str_idb": true, + "str_odb": true, + "plgod": false, + "plg": 2, + "plgne": "err", + "plgre": "err", + "plgof": "err", + "plggt": "err", + "pltod": false, + "hcovdr": false, + "hcovdr2": false, + "plovdr": false, + "plovdr2": false, + "ftsovdr": false, + "ftsovdr2": false, + "lb": false, + "eva": 33, + "lo": false, + "ts_mtp": 5, + "ts_tec": true, + "ts_tsa": true, + "vnd": "Google Inc.", + "bid": "NA", + "mmt": "empty", + "plu": "6GLkaVK,jRvAfXq0", + "hdn": false, + "awe": false, + "geb": false, + "dat": false, + "med": "defined", + "aco": "probably", + "acots": false, + "acmp": "probably", + "acmpts": true, + "acw": "probably", + "acwts": false, + "acma": "maybe", + "acmats": false, + "acaa": "probably", + "acaats": true, + "ac3": "", + "ac3ts": false, + "acf": "probably", + "acfts": false, + "acmp4": "maybe", + "acmp4ts": false, + "acmp3": "probably", + "acmp3ts": false, + "acwm": "maybe", + "acwmts": false, + "ocpt": false, + "vco": "", + "vcots": false, + "vch": "probably", + "vchts": true, + "vcw": "probably", + "vcwts": true, + "vc3": "maybe", + "vc3ts": false, + "vcmp": "", + "vcmpts": false, + "vcq": "", + "vcqts": false, + "vc1": "probably", + "vc1ts": true, + "dvm": 2, + "sqt": false, + "so": "portrait-primary", + "wdw": true, + "cokys": "bG9hZFRpbWVzY3NpL=", + "ecpc": false, + "lgs": true, + "lgsod": false, + "psn": true, + "edp": false, + "addt": true, + "wsdc": true, + "ccsr": true, + "nuad": true, + "bcda": true, + "idn": true, + "capi": false, + "svde": false, + "vpbq": true, + "ucdv": false, + "spwn": false, + "emt": false, + "bfr": false, + "dbov": false, + "cfpfe": "ZnVuY3Rpb24oKXt2YXIgbj10LG89ZG9jdW1lbnRbJ1x4NzFceDc1XHg2NVx4NzJceDc5XHg1M1x4NjVceDZjXHg2NVx4NjNceDc0XHg2Zlx4NzInXShuKDQ3MCkpO2lmKG8peyFmdW5jdGlvbiB0KCl7dmFyIGk9bjt0cnl7dmFyIGE9b1snXHg3M1x4NjhceDYxXHg2", + "stcfp": "Oi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6NzUyMzcpCiAgICBhdCB0LmV4cG9ydHMuZGRfYWEgKGh0dHBzOi8vZC5kaWdpdGFsLmhlcm1lcy90YWdzLmpzOjI6ODg0NDkpCiAgICBhdCBodHRwczovL2QuZGlnaXRhbC5oZXJtZXMvdGFncy5qczoyOjUxMTY1", + "ckwa": true, + "glvd": "Google Inc. (Qualcomm)", + "glrd": "ANGLE (Qualcomm, Adreno (TM) 650, OpenGL ES 3.2)", + "wwl": false, + "jset": 1705750207, + "dcok": ".hermes.com", + "log2": true, + "tbce": 0, + "es_sigmdn": null, + "es_mumdn": null, + "es_distmdn": null, + "es_angsmdn": null, + "es_angemdn": null, + "m_s_c": 1, + "m_m_c": 4, + "m_c_c": 4, + "m_cm_r": 1, + "m_ms_r": 4 +} \ No newline at end of file diff --git a/queue_message/appointmentrequestsendor.py b/queue_message/appointmentrequestsendor.py index 3683062..3cddad5 100644 --- a/queue_message/appointmentrequestsendor.py +++ b/queue_message/appointmentrequestsendor.py @@ -74,7 +74,8 @@ def is_open(): class AppointmentRequestSendor(threading.Thread): - def __init__(self, sub_contact_list: list, cookiesPublisher: CookiesPublisher, queue_name=REQUEST_DATA_QUEUE): + def __init__(self, sub_contact_list: list, proxy_to_use_list, cookiesPublisher: CookiesPublisher, + queue_name=REQUEST_DATA_QUEUE): super().__init__() self.connection = None self.cookiesPublisher = cookiesPublisher @@ -82,6 +83,7 @@ class AppointmentRequestSendor(threading.Thread): self.valid_csrf = None self.contact_list = sub_contact_list self.queue_name = queue_name + self.proxy_to_use_list = proxy_to_use_list def set_up_connection(self): self.connection = pika.BlockingConnection( @@ -111,7 +113,8 @@ class AppointmentRequestSendor(threading.Thread): print(con.mail) # time.sleep(random.randint(1, 5)) if self.valid_csrf is None: - self.valid_csrf = captchaResultGetter.get_csrf(body.decode("UTF-8")) + self.valid_csrf = captchaResultGetter.get_csrf(proxy_to_use=random.choice(self.proxy_to_use_list), + cookie=body.decode("UTF-8")) can_continue = sender.send_request(HERMES_REGISTER, con, csrf=self.valid_csrf) if not can_continue: print("cannot continue, valid_csrf is " + str(self.valid_csrf)) diff --git a/workers/link_validator.py b/workers/link_validator.py index c675282..6fdc315 100644 --- a/workers/link_validator.py +++ b/workers/link_validator.py @@ -16,7 +16,8 @@ from workers.proxies_constants import PROXY_LIST_FR class LinkValidator(threading.Thread): def __init__(self, link_to_validate_list: list, - cookiesPublisher: CookiesPublisher, proxy_to_use, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR"): + cookiesPublisher: CookiesPublisher, proxy_to_use, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR", + limit=40): super().__init__() self.cookie = SimpleCookie() self.cookiesPublisher = cookiesPublisher @@ -25,6 +26,7 @@ class LinkValidator(threading.Thread): self.ip_country = ip_country self.filter_with_ip_country() self.proxy_to_use = proxy_to_use + self.limit = limit # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;' def set_up_connection(self): @@ -85,11 +87,13 @@ class LinkValidator(threading.Thread): def on_message(self, ch, method, properties, body): print(f" [x] Received {body}") + _message_in_queue_count = self.cookiesPublisher.message_count() + print("message count in queue is {}".format(_message_in_queue_count)) self.link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate() self.filter_with_ip_country() self.cookie_str = body.decode("UTF-8") random.shuffle(self.link_to_validate_list) - if len(self.link_to_validate_list) > 0: + if len(self.link_to_validate_list) > 0 and _message_in_queue_count > self.limit: print("links number is {}".format(len(self.link_to_validate_list))) for con in self.link_to_validate_list: # if not is_already_sent(con): @@ -117,7 +121,7 @@ class LinkValidator(threading.Thread): def validate_with_FR_ip(): - _queue_name = REQUEST_DATA_QUEUE_TEST + _queue_name = REQUEST_DATA_QUEUE link_list = MONGO_STORE_MANAGER.get_links_to_validate() cookiesPublisher = CookiesPublisher(queue_name=_queue_name) cookiesPublisher.set_up_connection() @@ -131,7 +135,7 @@ def validate_with_FR_ip(): _fr_proxy_to_use = random.choice(PROXY_LIST_FR) receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher, proxy_to_use=_fr_proxy_to_use, - queue_to_listen=_queue_name, ip_country="FR") + queue_to_listen=_queue_name, ip_country="FR", limit=10) receiver.set_up_connection() receiver.listen_to_queue(receiver.on_message) pass diff --git a/workers/validate_links_DE.py b/workers/validate_links_DE.py index ea5a70f..2f2578d 100644 --- a/workers/validate_links_DE.py +++ b/workers/validate_links_DE.py @@ -23,7 +23,7 @@ def validate_with_DE_ip(): _de_proxy_to_use = random.choice(PROXY_LIST_DE) receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher, proxy_to_use=_de_proxy_to_use, - queue_to_listen=_queue_name, ip_country="DE") + queue_to_listen=_queue_name, ip_country="DE", limit=0) receiver.set_up_connection() receiver.listen_to_queue(receiver.on_message) pass