try playwright stelth

requirements.txt

@@ -1,11 +1,13 @@
 dataclasses_json==0.5.7
 firebase_admin==5.2.0
 pandas~=1.3.5
-playwright==1.23.0
+playwright==1.25.0
 dataclasses~=0.6
 pymongo==4.1.1
+anticaptchaofficial==1.0.43
 oci~=2.54.1
 XlsxWriter~=3.0.3
+playwright-stealth==1.0.5
 boto3~=1.21.13
 openpyxl==3.0.9
 google-cloud-firestore==2.4.0

src/workers/commandor_page.py

@@ -8,6 +8,10 @@ import time
 import traceback
 from typing import Union
 
+from anticaptchaofficial.antigatetask import antigateTask
+from playwright._impl._api_structures import SetCookieParam
+from playwright_stealth import stealth_sync
+
 from src import params, definitions
 from src.db.mongo_manager import MONGO_STORE_MANAGER
 from src.pojo.ModeEnum import ModeEnum
@@ -53,6 +57,7 @@ class CommandorPage:
                  mode: ModeEnum = ModeEnum.MANUAL, headless=False):
         self.otp_value = None
         self.is_finished = False
+        self.current_context = None
         self.contact = contact
         self.contact.phone = self.contact.phone.replace(".0", "")  # remove the .0 if the Excel format is not correct
         self.logger = logging.getLogger("约会页面:" + str(self.contact.phone))
@@ -119,26 +124,15 @@ class CommandorPage:
 
     def start_browser(self, proxy, pwright, device) -> Union[str, None]:
         try:
-            self.browser = pwright.firefox.launch(headless=self.headless, timeout=PAGE_TIMEOUT, proxy=proxy)
+            self.browser = pwright.webkit.launch(headless=self.headless, timeout=PAGE_TIMEOUT, proxy=proxy)
             # userAgent = random.choice(params.firefox_user_agent_list)
             simulated_mobile = pwright.devices[device]
             userAgent = simulated_mobile['user_agent']
             print("user_agent is " + userAgent)
             # context = self.browser.new_context(**simulated_mobile, locale='fr-FR')
-            context = self.browser.new_context(user_agent=userAgent, locale='fr-FR')
+            context = self.browser.new_context(**simulated_mobile, locale='fr-FR')
-            self.page = context.new_page()
+            self.current_context = context
-            # hide webdriver information
+            self.create_and_config_page(context)
-            self.page.add_init_script("""() => {
-                Object.defineProperty(navigator,'webdriver',{get: () => undefined});
-                Object.defineProperty(navigator, 'platform', {
-                    get: () => {
-                        return "iPhone";
-                    }});
-            }
-                        """)
-            self.page.on("load", self._on_page_loaded)
-            self.page.on("response", self.handle_response)
-            self.page.goto(RDV_URL, timeout=PAGE_TIMEOUT)
             return self.page.content()
             # self.browser = pwright.webkit.launch(headless=self.headless, timeout=PAGE_TIMEOUT, proxy=proxy)
             # self.logger.info("模拟设备: " + device)
@@ -172,6 +166,22 @@ class CommandorPage:
             self.browser.close()
             return None
 
+    def create_and_config_page(self, context):
+        self.page = context.new_page()
+        stealth_sync(self.page)
+        # hide webdriver information
+        self.page.add_init_script("""() => {
+                Object.defineProperty(navigator,'webdriver',{get: () => undefined});
+                Object.defineProperty(navigator, 'platform', {
+                    get: () => {
+                        return "iPhone";
+                    }});
+            }
+                        """)
+        self.page.on("load", self._on_page_loaded)
+        self.page.on("response", self.handle_response)
+        self.page.goto(RDV_URL, timeout=PAGE_TIMEOUT)
+
     def handle_response(self, response):
         pattern = re.compile(REGEX_RDV_URL)
         if pattern.match(response.url):
@@ -182,6 +192,34 @@ class CommandorPage:
         e = threading.Event()
         self._run(proxy)
 
+    def solve_datadome_captcha(self):
+        print("solve_datadome_captcha")
+        solver = antigateTask()
+        solver.set_verbose(1)
+        solver.set_key("ede6a69396fc961af351e7c8ffda9059")
+        solver.set_website_url(RDV_URL)
+        solver.set_template_name("Anti-bot screen bypass")
+        solver.set_variables({
+            "css_selector": ".captcha__human__container"
+        })
+        result = solver.solve_and_return_solution()
+        if result != 0:
+            cookies, localStorage, fingerprint, url, domain = result["cookies"], result["localStorage"], result[
+                "fingerprint"], result["url"], result["domain"]
+            print("cookies: ", cookies)
+            print("localStorage: ", localStorage)
+            print("fingerprint: ", fingerprint)
+            print("url: " + url)
+            print("domain: " + domain)
+            #     add cookies to playwright
+            cookie_list = []
+            cookie_list.append(SetCookieParam(name='datadome', value=cookies['datadome'], url=url))
+            self.page.context.add_cookies(cookie_list)
+            self.config_page_with_fingerprint(fingerprint)
+            self.page.reload()
+        else:
+            print("task finished with error " + solver.error_code)
+
     def _on_page_loaded(self):
         # time.sleep(40000)
         self.logger.info("页面加载完毕")
@@ -198,6 +236,8 @@ class CommandorPage:
                 self.get_errors()
             except Exception as error:
                 self.logger.error(error)
+        # else:
+        #     self.solve_datadome_captcha()
 
     def on_document_loaded(self):
         self.logger.info("on_document_loaded called")
@@ -377,6 +417,17 @@ class CommandorPage:
         MONGO_STORE_MANAGER.insert_captcha_error_contact(self.contact)
         params.oracle_log_sender.send_captcha_error(self.contact)
 
+    def config_page_with_fingerprint(self, fingerprint):
+        self.page.add_init_script("""() => {
+                Object.defineProperty(navigator,'webdriver',{get: () => undefined});
+                Object.defineProperty(navigator, 'platform', {
+                    get: () => {
+                        return "iPhone";
+                    }});
+                
+            }
+                        """)
+
 
 def on_success(result: ReserveResultPojo):
     pass