can slide when blocked by captcha

package.json

@@ -21,6 +21,8 @@
     "node-xlsx": "^0.21.0",
     "playwright": "1.23.0",
     "read-ini-file": "^3.0.1",
+    "uuid": "^9.0.0",
+    "image-size": "^1.0.2",
     "winston": "^3.8.2",
     "yarn": "^1.22.19"
   },

src/excel/ExcelUtil.js

@@ -10,7 +10,10 @@ class ExcelUtil {
             (info, index, list) => {
                 if (index > 0 && info.length > 0) {
                     // console.log(info)
-                    let name = info[0].split(" ")
+                    if (info[0].includes("Aijing")){
+                        console.log("found");
+                    }
+                    let name = info[0].trim().split(" ")
                     let firstName = name[1];
                     let lastName = name[0];
                     let phoneNumber = info[1];

src/models/Position.js

@@ -0,0 +1,7 @@
+class Position {
+    constructor(x, y) {
+        this.x = x;
+        this.y = y;
+    }
+}
+module.exports = Position

src/workers/CommandorPage.js

@@ -7,6 +7,7 @@ const {
     shell
 } = require('electron')
 const GeoCaptchaSolver = require("./GeoCaptchaSolver");
+const SlidingCaptchaSolver = require("./SlidingCaptchaSolver");
 // const RDV_URL = "http://192.168.0.41:8000/test_appointment.html"
 const RDV_URL = "https://rendezvousparis.hermes.com/client/register";
 const BLANK_URL = "about:blank"
@@ -36,7 +37,7 @@ const CAPTCHA_ERROR_MESSAGE = "Error verifying captcha, please try again"
 const CAPTCHA_ERROR_MESSAGE_FR = "La vérification du captcha a échoué"
 const REGEX_RDV_URL = "https:\/\/rendezvousparis\.hermes\.com\/client\/register\/[A-Z0-9]+"
 const DEFAULT_STORE = 'faubourg';
-const searchTexts = ['hermes+rdv', 'hermes+rendezvous', 'hermes+appointment', 'hermes+appointment+online', 'appointment+hermes', 'hermes+rendez+vous','hermes+rendez+vous+paris', 'online+appointment+hermes', 'hermes+online+appointment', 'paris+hermes+online+appointment']
+const searchTexts = ['hermes+rdv', 'hermes+rendezvous', 'hermes+appointment', 'hermes+appointment+online', 'appointment+hermes', 'hermes+rendez+vous', 'hermes+rendez+vous+paris', 'online+appointment+hermes', 'hermes+online+appointment', 'paris+hermes+online+appointment']
 
 function delay(delayInMs) {
     return new Promise(resolve => {
@@ -96,7 +97,6 @@ class CommandorPage {
         try {
             const item = searchTexts[Math.floor(Math.random() * searchTexts.length)];
             await this.page.goto("https://www.google.com/search?q=" + item + "&lr=lang_en", {timeout: 90 * 1000});
-            // await this.page.goto(RDV_URL, {timeout: 90 * 1000});
         } catch (e) {
             log(e)
             this.isTerminated = true
@@ -117,7 +117,9 @@ class CommandorPage {
             this.page.locator(':nth-match(:text("Online Appointment"), 1)').click()
         } catch (e) {
             log(e)
-            this.isTerminated = true
+            if (!this.page.url().includes(RDV_URL)) {
+                this.isTerminated = true;
+            }
         }
 
         let cancel
@@ -386,20 +388,26 @@ class CommandorPage {
     async checkAudioBtn() {
         let isBlocked = await this.isBlocked()
         if (!isBlocked) {
-            let audioBtn = await this.page.frameLocator("iframe").locator("#captcha__audio__button");
+            //try to sliding capthca at first
-            log("audioBtn found")
+            let slidingCaptchaSolver = new SlidingCaptchaSolver(this.device);
-            audioBtn.click()
+            await slidingCaptchaSolver.solve(this.page, async (isSuccessful) => {
-            let captchaSolver = new GeoCaptchaSolver(this.page, this.device, this.isTerminated)
+                let isAlwaysBlocked = await this.isBlocked();
-            await captchaSolver.solve((isSuccessful) => {
+                if (isAlwaysBlocked) {
-                if (!isSuccessful) {
+                    let audioBtn = await this.page.frameLocator("iframe").locator("#captcha__audio__button");
-                    this.isTerminated = true
+                    log("audioBtn found")
+                    audioBtn.click()
+                    let captchaSolver = new GeoCaptchaSolver(this.page, this.device, this.isTerminated)
+                    await captchaSolver.solve((isSuccessful) => {
+                        if (!isSuccessful) {
+                            this.isTerminated = true
+                        }
+                    })
                 }
             })
         } else {
             log("audioBtn not found")
             console.log("audioBtn not found")
             console.log("we are blocked")
-            //
             await this.resetBrowser()
         }
     }

src/workers/GeoCaptchaSolver.js

@@ -60,7 +60,7 @@ async function sendRequest(fileName, callback) {
     }
 }
 
-class GeoCaptchaSolver {
+class AudioCaptchaSolver {
     constructor(page, device, isTerminated) {
         this.page = page;
         this.isTerminated = isTerminated
@@ -170,4 +170,4 @@ class GeoCaptchaSolver {
 
 }
 
-module.exports = GeoCaptchaSolver
+module.exports = AudioCaptchaSolver

src/workers/SlidingCaptchaSolver.js

@@ -0,0 +1,120 @@
+const fs = require("fs");
+const {exec} = require("child_process");
+
+const {promisify} = require('util')
+const Position = require("../models/Position");
+
+const sizeOf = promisify(require('image-size'));
+
+const axios = require("axios");
+const {v4: uuidv4} = require('uuid');
+
+const DEEPLEARNING_CAPTCHA_HOST = "http://appointment.lpaconsulting.fr:9000"
+
+function delay(delayInMs) {
+    return new Promise(resolve => {
+        setTimeout(() => {
+            resolve(2);
+        }, delayInMs);
+    });
+}
+
+function randomIntFromInterval(min, max) { // min and max included
+    return Math.floor(Math.random() * (max - min + 1) + min)
+}
+
+const positionInfo = new Map();
+
+positionInfo.set("2109119DG", new Position(211, 1710))
+positionInfo.set("ONEPLUS A6000", new Position(211, 1706))
+positionInfo.set("SM-C5000", new Position(180, 1794))
+
+class SlidingCaptchaSolver {
+    constructor(device) {
+        this.device = device;
+    }
+
+    async solve(page, onResult) {
+        await this.sliding_captcha(this.device, onResult)
+    }
+
+    async take_screen_shot(name, device) {
+        // let stdout1 = await exec("adb -s " + device.serial() + " shell screencap -p /sdcard/" + name)
+        let stdout1 = await exec("adb -s " + device.serial() + " exec-out screencap -p > " + name)
+        await delay(1000);
+        console.log(`stdout: ${stdout1}`);
+    }
+
+
+    async sliding_captcha(device, onResult) {
+//    take screenshot
+        let blockedFileName = uuidv4();
+        let blockedImageFileName = "blocked_" + blockedFileName
+        await this.take_screen_shot(blockedImageFileName, device)
+        //get resolution of screen
+        let dimensions = await sizeOf(blockedImageFileName)
+        console.log(dimensions.width, dimensions.height);
+        await this.sendRequest(blockedImageFileName, async (detectedPosition) => {
+            console.log("detectedPosition  is " + detectedPosition);
+            let startPosition = positionInfo.get(device.model())
+            if (startPosition === undefined) {
+                startPosition = new Position(210, dimensions.height * 0.7125)
+            }
+            if (startPosition !== undefined) {
+                let y0 = startPosition.y;
+                let x0 = startPosition.x;
+                let x1 = detectedPosition.x1;
+                let x2 = detectedPosition.x2;
+                let width = x2 - detectedPosition.x1;
+                let randomTime = randomIntFromInterval(100, 500)
+                let cmd = `adb -s ${device.serial()} shell input swipe ${x0} ${y0} ${x1 + width * 0.8} ${y0} ${1000 + randomTime}`
+                await delay(2000);
+                console.log("cmd is " + cmd);
+                console.log("will slide captcha");
+                await exec(cmd);
+                await delay(3000);
+                onResult(true);
+            } else {
+                console.log("startPosition not found for " + device.model())
+                onResult(false);
+            }
+        })
+
+    }
+
+    async sendRequest(fileName, callback) {
+        const fileStream = fs.createReadStream(fileName);
+        let response = await axios.post(DEEPLEARNING_CAPTCHA_HOST, fileStream, {
+            headers: {
+                'Content-Type': 'wav',
+                'Authorization': 'Bearer 97e36f7e-340e-4c02-b329-9415faee38c3'
+            },
+            timeout: 90 * 1000
+        }).catch(err => {
+            console.log(err.code);
+            console.log(err.stack);
+            console.log(err.message);
+            callback([]);
+        })
+        if (response === undefined) {
+            await callback({});
+            return
+        }
+        if (response.status === 200) {
+            let result = response.data;
+            console.log(result);
+            await callback(result);
+            // console.log("will delete png file: " + fileName);
+            // await fs.unlink(fileName, (err) => {
+            //     if (err) throw err;
+            //     console.log(fileName + ' was deleted');
+            // })
+        } else {
+            console.log("error");
+            await callback({});
+            console.log("will delete wav file: " + fileName);
+        }
+    }
+}
+
+module.exports = SlidingCaptchaSolver