From 21122ea5404460fca5511a4931fd2929fe5074c6 Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Sat, 18 May 2024 11:51:27 +0200 Subject: [PATCH] log oxylabs res proxy --- docs/note.md | 15 +++++++++++++++ proxy_manager/proxy_manager.py | 6 +++++- workers/link_validator_with_provided_list.py | 13 +++++++------ 3 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 docs/note.md diff --git a/docs/note.md b/docs/note.md new file mode 100644 index 0000000..b2db5e1 --- /dev/null +++ b/docs/note.md @@ -0,0 +1,15 @@ +##stcfp +获得stcfp的方法，但是取决于tags.js的版本 +let stcfp = "" +try { + null[0]; +} catch (n) { + //if ("string" != typeof n[["stack"]]) return t[["apply"]](this, arguments); + stcfp = window.btoa(n[["stack"]].slice(-150)); + var w = n[["stack"]].split("\n"); +// if (i) try { + // w.length > 1 && c[["test"]](w[2]) && (e[["cfpp"]] = true, d("asyncChallengeFinished")), w.length > 2 && l[["test"]](w[w.length - 3]) && (e[["cfcpw"]] = true, d("asyncChallengeFinished")), w.length > 8 && //h[["test"]](w[w.length - 4]) && (e[["cfse"]] = true, d("asyncChallengeFinished")); + //} catch (e) {} else if (a) try { + // w.length > 2 && l[["test"]](w[w.length - 3]) && (e[["cffpw"]] = true, d("asyncChallengeFinished")); + //} catch (e) {} +} diff --git a/proxy_manager/proxy_manager.py b/proxy_manager/proxy_manager.py index 6c2473c..85181ff 100644 --- a/proxy_manager/proxy_manager.py +++ b/proxy_manager/proxy_manager.py @@ -29,13 +29,17 @@ FR_PROXY_DATA_IMPULSE_STICKY = { 'https': 'http://1d568220dbefeff21ad4__cr.fr:df99489a25fa72d9@gw.dataimpulse.com:{}' } +FR_PROXY_RES_OXY_STICKY = { + 'http': 'http://customer-rendezvous-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}', + 'https': 'http://customer-rendezvous-cc-FR:Rdv202220212023@fr-pr.oxylabs.io:{}' +} FR_DATA_IMPULSE_RES = { 'http': 'http://ac2b8183c3b02714cbd1__cr.fr:336e7f8f4b17e5bd@gw.dataimpulse.com:823', 'https': 'http://ac2b8183c3b02714cbd1__cr.fr:336e7f8f4b17e5bd@gw.dataimpulse.com:823' } # 八分之一用data_impulse MOBILE_PROXY_LIST = [FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, - FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY + FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_MOB_OXY_STICKY, FR_PROXY_RES_OXY_STICKY # FR_PROXY_DATA_IMPULSE_STICKY ] diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index 9fb8c0c..fc1b73b 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -220,7 +220,7 @@ def validate_all_links(): _segment_number = int(len(_first_25_percent_links) / divided) else: _segment_number = 1 - last_thread = None + _thread_list = [] for i in range(0, _segment_number): logger.info("{}:{} links to validate".format(threading.currentThread().name, len(_first_25_percent_links))) logger.info("segment is {}".format(i)) @@ -229,10 +229,10 @@ def validate_all_links(): _step = int(len(_first_25_percent_links) / _segment_number) _sublist = _first_25_percent_links[i * _step:_step * (i + 1)] _thread1 = threading.Thread(target=validate_links, args=(_cookiesPublisher, MORNING_DATA_CACHE_BAK, _sublist)) - last_thread = _thread1 + _thread_list.append(_thread1) _thread1.start() - if last_thread is not None: - last_thread.join() + for _thread in _thread_list: + _thread.join() if __name__ == '__main__': @@ -240,5 +240,6 @@ if __name__ == '__main__': while True: print("call validate_all_links()") validate_all_links() - print("wait for 30 seconds") - time.sleep(30) + delay = random.randint(10, 30) + print("wait for {} seconds".format(delay)) + time.sleep(delay)