need to use openCv to identifiy text

src/appointment.js

@@ -14,7 +14,7 @@ let collectionName = formatDate(new Date())
 // device_to_excludes = ["47e7e36b", "e30eb015"]
 // device_to_excludes = ["47e7e36b"]
 // device_to_excludes = ["J4AXB761H2322WJ"]
-device_to_excludes = []
+device_to_excludes = ["e30eb015"]
 attributedPorts = []
 const device_port_info = new Map();
 startPort = 9000
@@ -171,12 +171,16 @@ async function startBookWithNumbers(startNumber, endNumber, selectedStore, pathT
         // get attributed port
         let attributedPort = device_port_info[device.serial()]
         if (attributedPort) {
-            const output = execSync('adb -s ' + device.serial() + " forward tcp:" + attributedPort + " localabstract:chrome_devtools_remote", {encoding: 'utf-8'});  // the default is 'buffer'
+            let cmd = 'adb -s ' + device.serial() + " forward tcp:" + attributedPort + " localabstract:chrome_devtools_remote";
+            console.log("cmd is " + cmd);
+            const output = execSync(cmd, {encoding: 'utf-8'});  // the default is 'buffer'
             console.log('Output was:\n', output);
         } else {
             attributedPort = startPort;
             startPort++;
-            const output = execSync('adb -s ' + device.serial() + " forward tcp:" + attributedPort + " localabstract:chrome_devtools_remote", {encoding: 'utf-8'});  // the default is 'buffer'
+            let cmd = 'adb -s ' + device.serial() + " forward tcp:" + attributedPort + " localabstract:chrome_devtools_remote";
+            console.log("cmd is " + cmd);
+            const output = execSync(cmd, {encoding: 'utf-8'});  // the default is 'buffer'
             console.log('Output was:\n', output);
         }
         return attributedPort

src/findTextWithOpenCV.js

@@ -0,0 +1,107 @@
+const path = require('path');
+const fs = require('fs');
+const { cv, drawBlueRect } = require('opencv4nodejs/ut');
+// const { extractResults } = require('./dnn/ssdUtils');
+
+if (!cv.xmodules.dnn) {
+    throw new Error('exiting: opencv4nodejs compiled without dnn module');
+}
+
+const modelPath = path.resolve(__dirname,
+    '../data/text-models/frozen_east_text_detection.pb');
+const imgPath = path.resolve(__dirname, '../data/text-data/detection.png');
+
+if (!fs.existsSync(modelPath)) {
+    console.log('could not find EAST model');
+    console.log('download the model from: https://github.com/oyyd/frozen_east_text_detection.pb/blob/71415464412c55bb1d135fcdeda498e29a67effa/frozen_east_text_detection.pb?raw=true'
+        + ' or create a .pb model from https://github.com/argman/EAST');
+    throw new Error('exiting: could not find EAST model');
+}
+
+const MIN_CONFIDENCE = 0.5;
+const NMS_THRESHOLD = 0.4;
+const SIZE = 320;
+
+function decode(scores, geometry, confThreshold) {
+    const [numRows, numCols] = scores.sizes.slice(2);
+    const boxes = [];
+    const confidences = [];
+
+    for (let y = 0; y < numRows; y += 1) {
+        for (let x = 0; x < numCols; x += 1) {
+            const score = scores.at([0, 0, y, x]);
+
+            if (score < MIN_CONFIDENCE) {
+                continue;
+            }
+
+            const offsetX = x * 4;
+            const offsetY = y * 4;
+            const angle = geometry.at([0, 4, y, x]);
+            const cos = Math.cos(angle);
+            const sin = Math.sin(angle);
+
+            const h = geometry.at([0, 0, y, x]) + geometry.at([0, 2, y, x]);
+            const w = geometry.at([0, 1, y, x]) + geometry.at([0, 3, y, x]);
+
+            const endX = offsetX + (cos * geometry.at([0, 1, y, x])) + (sin * geometry.at([0, 2, y, x]));
+            const endY = offsetY - (sin * geometry.at([0, 1, y, x])) + (cos * geometry.at([0, 2, y, x]));
+            const startX = endX - w;
+            const startY = endY - h;
+
+            boxes.push(new cv.Rect(
+                startX,
+                startY,
+                endX - startX,
+                endY - startY,
+            ));
+            confidences.push(score);
+        }
+    }
+
+    return [boxes, confidences];
+}
+
+function detection(modelAbsPath, imgAbsPath) {
+    const net = cv.readNetFromTensorflow(modelPath);
+    const img = cv.imread(imgAbsPath);
+    const [imgHeight, imgWidth] = img.sizes;
+    const widthRatio = imgWidth / SIZE;
+    const heightRatio = imgHeight / SIZE;
+
+    const inputBlob = cv.blobFromImage(img, 1,
+        new cv.Size(SIZE, SIZE), new cv.Vec3(123.68, 116.78, 103.94), true, false);
+
+    net.setInput(inputBlob);
+
+    const outBlobNames = [
+        'feature_fusion/Conv_7/Sigmoid',
+        'feature_fusion/concat_3',
+    ];
+
+    const [scores, geometry] = net.forward(outBlobNames);
+    const [boxes, confidences] = decode(scores, geometry, MIN_CONFIDENCE);
+
+    const indices = cv.NMSBoxes(
+        boxes,
+        confidences, MIN_CONFIDENCE, NMS_THRESHOLD
+    );
+
+    indices.forEach((i) => {
+        const rect = boxes[i];
+        const imgRect = new cv.Rect(
+            rect.x * widthRatio,
+            rect.y * heightRatio,
+            rect.width * widthRatio,
+            rect.height * heightRatio,
+        )
+        drawBlueRect(img, imgRect);
+    });
+
+    cv.imshowWait('EAST text detection', img);
+}
+
+detection(
+    modelPath,
+    imgPath
+);

src/openCvTest.js

@@ -12,8 +12,8 @@ const cv = require('opencv4nodejs');
 
 const findWaldo = async () => {
     // Load images
-    const originalMat = await cv.imreadAsync(`/Users/lpan/Downloads/Screenshot_2023-05-19-14-09-11-82_40deb401b9ffe8e1df2f1cc5ba480b12.jpg`);
+    const originalMat = await cv.imreadAsync(`/Users/lpan/Downloads/Screenshot_2023-05-20-15-20-22-860_com.android.chrome.jpg`);
-    const waldoMat = await cv.imreadAsync(`${__dirname}/../assets/templates/valid_btn_template.png`);
+    const waldoMat = await cv.imreadAsync(`${__dirname}/../assets/templates/welcome_later_btn.png`);
 
     // Match template (the brightest locations indicate the highest match)
     const matched = originalMat.matchTemplate(waldoMat, 3);

src/workers/CommandorPage.js

@@ -11,7 +11,6 @@ const {
 } = require('electron')
 const GeoCaptchaSolver = require("./GeoCaptchaSolver");
 const SlidingCaptchaSolver = require("./SlidingCaptchaSolver");
-const {de} = require("yarn/lib/cli");
 const OCRChecker = require("./OCRChecker");
 // const RDV_URL = "http://192.168.0.44:8000/test_appointment.html"
 const RDV_URL = "https://rendezvousparis.hermes.com/client/register";
@@ -34,12 +33,6 @@ const TIME_OUT = 60 * 1000 * 4//4 mins
 const EMPTY_RESPONSE_ERROR = "ERR_EMPTY_RESPONSE"
 const MESSAGE_URL_VALIDATION_FR = "Nous avons envoyé un lien par e-mail."
 const MESSAGE_URL_VALIDATION_EN = "Please click on the link we sent by email"
-const DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today."
-const DOUBLE_REQUEST_ERROR_MESSAGE_FR = "Une demande avec les données saisies a déjà été validée aujourd’hui."
-const TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests"
-const TOO_MANY_REQUEST_ERROR_MESSAGE_FR = "Suite à un trop grand nombre de demandes"
-const CAPTCHA_ERROR_MESSAGE = "Error verifying captcha, please try again"
-const CAPTCHA_ERROR_MESSAGE_FR = "La vérification du captcha a échoué"
 const REGEX_RDV_URL = "https:\/\/rendezvousparis\.hermes\.com\/client\/register\/[A-Z0-9]+"
 const DEFAULT_STORE = 'faubourg';
 
@@ -90,12 +83,16 @@ class CommandorPage {
 
 
     async connect_to_browser(ocrResult) {
+        console.log("connect_to_browser() called");
+        if (!this.browser.isConnected()) {
             this.browser = await puppeteer.connect({
                 browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
                 headless: false, defaultViewport: null
             })
+        }
         let pages = await this.browser.pages();
-        this.page = pages[pages.length - 1]
+        // 0 is the last page
+        this.page = pages[0];
         switch (ocrResult) {
             case OCRResult.SUCCESS:
                 //         get url and push to server
@@ -105,10 +102,9 @@ class CommandorPage {
             case OCRResult.TO_REFRESH:
                 logWithDevice("will reload page", this.device)
                 await this.page.reload();
-                logWithDevice("will disconnect browser", this.device)
+                await delay(1000);
-                await this.browser.disconnect()
+                await this.checkResultWithOcr();
                 break;
-
         }
         let content = await this.page.content();
         console.log(content);
@@ -116,7 +112,7 @@ class CommandorPage {
 
     async loadPage() {
         // Connect to the device.
-        log("loadPage() called, with port:" + this.port);
+        logWithDevice("loadPage() called, with port:" + this.port, this.device);
         try {
             this.browser = await puppeteer.connect({
                 browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
@@ -145,14 +141,14 @@ class CommandorPage {
             log(e)
             this.isTerminated = true
         }
-        try {
+        // try {
-            if (this.page.url().includes("google")) {
+        //     if (this.page.url().includes("google")) {
-                this.page.focus('button >> nth=3')
+        //         this.page.focus('button >> nth=3')
-                this.page.click()
+        //         this.page.click()
-            }
+        //     }
-        } catch (e) {
+        // } catch (e) {
-            log(e)
+        //     log(e)
-        }
+        // }
         try {
             const [button] = await this.page.$x("//a[contains(., 'rendezvousparis')]");
             console.log("button is " + button)
@@ -165,9 +161,9 @@ class CommandorPage {
             // });
         } catch (e) {
             log(e)
-            if (!this.page.url().includes(RDV_URL)) {
+            // if (!this.page.url().includes(RDV_URL)) {
-                this.isTerminated = true;
+            //     this.isTerminated = true;
-            }
+            // }
         }
 
         let cancel
@@ -182,10 +178,10 @@ class CommandorPage {
                 cancel()
                 return this.browser
             } else {
-                if (this.page.url() === RDV_URL) {
+                // if (this.page.url() === RDV_URL) {
-                    if (!this.isFillingFields)
+                //     if (!this.isFillingFields)
-                        this.fillFields(this.page);
+                //         this.fillFields(this.page);
-                }
+                // }
             }
         }, 10 * 1000)//interval of 10 seconds
 
@@ -331,7 +327,8 @@ class CommandorPage {
 
 
     async fillFields(page) {
-        log("fillFields called")
+        log("fillFields called for contact:" + this.contact.mail)
+        await this.enableDisableAirPlanMode()
         if (!this.isFillingFields) {
             this.isFillingFields = true;
             await this.chooseStore(page);
@@ -364,24 +361,12 @@ class CommandorPage {
                         // remove debug flag
                         // const validElement = await page.$('.btn');
                         console.log("will click on valid button");
-                        console.log("will click on valid button");
-                        console.log("will click on valid button");
                         await this.page.evaluate(() => {
                             document.getElementsByClassName("btn")[0].click();
                         })
                         // this.browser.disconnect();
-                        // await delay(2000);
+                        // await this.checkResultWithOcr();
-                        // let ocrChecker = new OCRChecker(this.device, this.contact);
+
-                        // let checkResult = await ocrChecker.get_result();
-                        // switch (checkResult) {
-                        //     case OCRResult.SUCCESS:
-                        //         //         reconnect to page and get url
-                        //         await this.connect_to_browser(OCRResult.SUCCESS);
-                        //         break;
-                        //     case OCRResult.BLOCKED:
-                        //         await this.resetBrowser();
-                        //         break;
-                        // }
                     } catch (e) {
                         log(e);
                     }
@@ -440,24 +425,17 @@ class CommandorPage {
     }
 
     async onPageLoad(currentPage) {
+        logWithDevice("onPageLoad called with url " + this.page.url(), this.device)
         try {
-            let content = await currentPage.content();
+            let content = await this.page.content();
             let captcha_url = "geo.captcha-delivery.com/captcha";
             if (content.toString().includes(captcha_url)) {
-                if (this.audioAnalyse) {
+                await this.checkResultWithOcr()
-                    await this.checkAudioBtn();
-                }
-                if (this.alertBeep) {
-                    for (let i = 0; i < 15; i++) {
-                        await delay(1000)
-                        shell.beep()
-                    }
-                }
                 logWithDevice("发现datadome", this.device);
             } else if (content.includes("502 Bad Gateway")) {
                 logWithDevice("502 Bad Gateway found", this.device)
                 await this.page.reload()
-            } else if (currentPage.url().includes("sorry")) {
+            } else if (this.page.url().includes("sorry")) {
                 await this.resetBrowser()
             } else if (content.includes("PROXY_CONNECTION_FAILED")) {
                 logWithDevice("PROXY_CONNECTION_FAILED, will reload page", this.device);
@@ -467,12 +445,14 @@ class CommandorPage {
                 logWithDevice("ERR_NETWORK_CHANGED, will reload page", this.device);
                 await delay(2000)
                 await this.page.reload()
-            } else if (content.includes("CACHE_MISS")) {
+            }
-                logWithDevice("CACHE_MISS, will reload page", this.device);
+                // else if (content.includes("CACHE_MISS")) {
-                await delay(2000)
+                //     logWithDevice("CACHE_MISS, will reload page", this.device);
-                // await this.setUpCookies()
+                //     await delay(2000)
-                await this.page.reload()
+                //     // await this.setUpCookies()
-            } else if (content.includes("ERR_TIMED_OUT")) {
+                //     await this.page.reload()
+            // }
+            else if (content.includes("ERR_TIMED_OUT")) {
                 logWithDevice("ERR_TIMED_OUT, will reload page", this.device);
                 await delay(2000)
                 await this.page.reload()
@@ -481,7 +461,7 @@ class CommandorPage {
                 await delay(2000)
                 await this.page.reload()
             } else {
-                if (currentPage.url() === RDV_URL) {
+                if (this.page.url() === RDV_URL) {
                     await this.fillFields(this.page);
                     await this.saveCookies()
                     // if (this.isFillingFields)
@@ -539,6 +519,14 @@ class CommandorPage {
         // }
     }
 
+    async slidingCaptcha(onResult) {
+        let slidingCaptchaSolver = new SlidingCaptchaSolver(this.device);
+        await slidingCaptchaSolver.solve(this.page, async (isSuccessful) => {
+            console.log("check isAlwaysBlocked")
+            onResult(isSuccessful)
+        })
+    }
+
     async checkAudioBtn() {
         let isBlocked = await this.isBlocked()
         if (!isBlocked) {
@@ -589,6 +577,7 @@ class CommandorPage {
 
     async push_message_to_queue(publishType) {
         let url = this.page.url();
+        logWithDevice("successful url is " + url, this.device)
         await this.push_message_to_db(publishType, url)
     }
 
@@ -640,36 +629,11 @@ class CommandorPage {
             let elementHandle = await this.page.$('[title="reCAPTCHA"]')
             const iframe = await elementHandle.contentFrame()
             await iframe.click("#recaptcha-anchor > div.recaptcha-checkbox-border")
-            //     // .getByRole('checkbox', {name: 'I\'m not a robot'})
-            // if (enCheckbox) {
-            //     enCheckbox.click()
-            // } else {
-            //     let frCheckbox = await this.page.$('[title="reCAPTCHA"]').getByRole('checkbox', {name: 'Je ne suis pas un robot'})
-            //     if (frCheckbox) {
-            //         frCheckbox.click()
-            //     } else {
-            //         console.log("recaptcha checkbox not found")
-            //     }
-            // }
         } catch (e) {
             log(e);
         }
     }
 
-    async handleError(errorContent) {
-        log("handle error:" + errorContent);
-        if (errorContent.includes(DOUBLE_REQUEST_ERROR_MESSAGE) || errorContent.includes(DOUBLE_REQUEST_ERROR_MESSAGE_FR)) {
-            this.isTerminated = true;
-        } else if (errorContent.includes(TOO_MANY_REQUEST_ERROR_MESSAGE) || errorContent.includes(TOO_MANY_REQUEST_ERROR_MESSAGE_FR)) {
-            //add contact to black list and set terminated the task
-            log("handle error: will save to black list db");
-            await this.saveToBlackList()
-        } else if (errorContent.includes(CAPTCHA_ERROR_MESSAGE) || errorContent.includes(CAPTCHA_ERROR_MESSAGE_FR)) {
-            this.isTerminated = true;
-        }
-
-    }
-
     async removeDebugFlag() {
         await delay(1000)
         await this.device.shell("am clear-debug-app --persistent com.android.chrome")
@@ -680,12 +644,55 @@ class CommandorPage {
         console.log("will reset browser")
         await this.device.shell("pm clear com.android.chrome")
         await delay(1000)
-        await this.device.shell("pm am start -n com.android.chrome/com.google.android.apps.chrome.Main")
+        await this.device.shell("am start -n com.android.chrome/com.google.android.apps.chrome.Main")
         await delay(1000)
+        // await this.skipWelcomePage()
         this.isTerminated = true
     }
 
 
+    async checkResultWithOcr() {
+        console.log("checkResultWithOcr() called.")
+        await delay(2000);
+        let ocrChecker = new OCRChecker(this.device, this.contact);
+        let checkResult = await ocrChecker.get_result();
+        while (checkResult === OCRResult.RECHECK) {
+            await delay(4000)
+            logWithDevice("will recheck OCR", this.device)
+            checkResult = await ocrChecker.get_result();
+        }
+        while (checkResult === OCRResult.SLIDING_CAPTCHA) {
+            await this.slidingCaptcha(async () => {
+                checkResult = await ocrChecker.get_result();
+                await this.checkResultWithOcr()
+            })
+            await delay(5000)
+        }
+        switch (checkResult) {
+            case OCRResult.SUCCESS:
+                //         reconnect to page and get url
+                await this.connect_to_browser(OCRResult.SUCCESS);
+                break;
+            case OCRResult.RECAPTCHA_ERROR:
+                this.isTerminated = true;
+                break;
+            case OCRResult.TO_REFRESH:
+                await this.connect_to_browser(OCRResult.TO_REFRESH)
+                break;
+            case OCRResult.BLOCKED:
+                await this.resetBrowser();
+                break;
+        }
+    }
+
+    async enableDisableAirPlanMode() {
+        logWithDevice("will enable aireplan mode", this.device)
+        await this.device.shell("cmd connectivity airplane-mode enable")
+        await this.device.shell("adb shell settings put global airplane_mode_on 1")
+        await delay(1000)
+        await this.device.shell("adb shell settings put global airplane_mode_on 0")
+        await this.device.shell("cmd connectivity airplane-mode disable")
+    }
 }
 
 module.exports = CommandorPage

src/workers/OCRChecker.js

@@ -23,6 +23,8 @@ const CAPTCHA_ERROR_MESSAGE_FR = "La vérification du captcha a échoué"
 const BLOCKED_MSG_EN = "have been blocked"
 const BLOCKED_MSG_FR = "avez été bloqué"
 const CHECKING_MSG_FR = "Verifying"
+const ERR_CACHE_MISS = "ERR_CACHE_MISS"
+const SLIDING_CAPTCHA_FR = "Glissez vers la droite"
 
 class OCRChecker {
 
@@ -47,6 +49,12 @@ class OCRChecker {
             return OCRResult.RECAPTCHA_ERROR
         } else if (result.includes(BLOCKED_MSG_EN) || result.includes(BLOCKED_MSG_FR)) {
             return OCRResult.BLOCKED
+        } else if (result.includes(ERR_CACHE_MISS)) {
+            return OCRResult.TO_REFRESH
+        } else if (result.includes(CHECKING_MSG_FR)) {
+            return OCRResult.RECHECK
+        } else if (result.includes(SLIDING_CAPTCHA_FR)) {
+            return OCRResult.SLIDING_CAPTCHA
         }
     }
 

start.js

@@ -1,6 +1,6 @@
 const {startBookWithNumbers, scheduleBookWithNumbers} = require('./src/appointment')
 
 
-startBookWithNumbers(0, 2, "random", "./163_contacts.xlsx", true, false).then((r) => {
+startBookWithNumbers(70, 200, "random", "/Users/lpan/Desktop/gmail_contacts.xlsx", true, false).then((r) => {
     console.log(r)
 })