should use adb to refresh page

src/appointment.js

@@ -14,7 +14,7 @@ let collectionName = formatDate(new Date())
 // device_to_excludes = ["47e7e36b", "e30eb015"]
 // device_to_excludes = ["47e7e36b"]
 // device_to_excludes = ["J4AXB761H2322WJ"]
-device_to_excludes = ["e30eb015"]
+device_to_excludes = ["e30eb015", "07fbd156", "hi7ljr5xduyt9pfi"]
 attributedPorts = []
 const device_port_info = new Map();
 startPort = 9000
@@ -151,6 +151,21 @@ async function scheduleBookWithNumbers(startNumber, endNumber, selectedStore, pa
     })
 }
 
+function shuffle(array) {
+    let currentIndex = array.length, randomIndex;
+    // While there remain elements to shuffle.
+    while (currentIndex !== 0) {
+        // Pick a remaining element.
+        randomIndex = Math.floor(Math.random() * currentIndex);
+        currentIndex--;
+        // And swap it with the current element.
+        [array[currentIndex], array[randomIndex]] = [
+            array[randomIndex], array[currentIndex]];
+    }
+
+    return array;
+}
+
 async function startBookWithNumbers(startNumber, endNumber, selectedStore, pathToExcelFile = '/Users/lpan/Desktop/contact_all.xlsx', audioAnalyse = true, alertBeep = false) {
     console.log("startBookWithNumbers() called, with alertBeep:" + alertBeep)
     console.log("startBookWithNumbers() called, with audioAnalyse:" + audioAnalyse)
@@ -183,6 +198,8 @@ async function startBookWithNumbers(startNumber, endNumber, selectedStore, pathT
             const output = execSync(cmd, {encoding: 'utf-8'});  // the default is 'buffer'
             console.log('Output was:\n', output);
         }
+        // start chrome
+        device.shell("am start -n com.android.chrome/com.google.android.apps.chrome.Main")
         return attributedPort
     }
 
@@ -190,6 +207,7 @@ async function startBookWithNumbers(startNumber, endNumber, selectedStore, pathT
         filterAlreadyBookedContacts(contactList).then((listToBook) => {
             filterAlreadyAcceptedContacts(listToBook).then(notAcceptedContacts => {
                     filterBlacklistedContacts(notAcceptedContacts).then(listWithoutBlackContact => {
+
                         console.log("number of contacts to book:" + listWithoutBlackContact.length)
                         android.devices().then((devices) => {
                             if (devices.length === 0) {

src/models/OCRResult.js

@@ -3,6 +3,8 @@ const OCRResult = {
     TO_REFRESH: Symbol("TO_REFRESH"),
     BLOCKED: Symbol("BLOCKED"),
     SLIDING_CAPTCHA: Symbol("SLIDING_CAPTCHA"),
+    FILL_FIELD: Symbol("FILL_FIELD"),
+    NEED_TO_CLICK_LATE_BTN: Symbol("NEED_TO_CLICK_LATE_BTN"),
     RECHECK: Symbol("RECHECK"),
     RECAPTCHA_ERROR: Symbol("RECAPTCHA_ERROR"),
 }

src/workers/CommandorPage.js

@@ -72,27 +72,28 @@ class CommandorPage {
         this.port = port;
         this.isFillingFields = false;
         this.isTerminated = false;
-        this.audioAnalyse = audioAnalyse;
-        this.alertBeep = alertBeep;
         this.cguChecked = false;
         this.isNameInput = false;
         this.isEmailFilled = false;
         this.isCountryChoosen = false;
         this.isPhoneInput = false;
+        this.firstStart = false;
     }
 
 
     async connect_to_browser(ocrResult) {
         console.log("connect_to_browser() called");
-        if (!this.browser.isConnected()) {
+        console.log("browser.isConnected: " + this.browser.isConnected());
+        // if (!this.browser.isConnected()) {
         this.browser = await puppeteer.connect({
             browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
             headless: false, defaultViewport: null
         })
-        }
+        // }
         let pages = await this.browser.pages();
         // 0 is the last page
         this.page = pages[0];
+        // this.page = pages[pages.length - 1];
         switch (ocrResult) {
             case OCRResult.SUCCESS:
                 //         get url and push to server
@@ -102,12 +103,23 @@ class CommandorPage {
             case OCRResult.TO_REFRESH:
                 logWithDevice("will reload page", this.device)
                 await this.page.reload();
-                await delay(1000);
+                await delay(10000);
+                if (!await this.checkIfSuccessful()) {
                     await this.checkResultWithOcr();
+                }
                 break;
         }
+    }
+
+    async checkIfSuccessful() {
         let content = await this.page.content();
-        console.log(content);
+        if (content.includes(MESSAGE_URL_VALIDATION_FR) || content.includes(MESSAGE_URL_VALIDATION_EN)) {
+            log("successful");
+            await this.push_message_to_queue(PublishType.SUCCESS);
+            return true;
+        } else {
+            return false
+        }
     }
 
     async loadPage() {
@@ -128,27 +140,24 @@ class CommandorPage {
                 this.onResponse(response)
             })
         } catch (e) {
+            await this.checkResultWithOcr()
             console.log(e)
-            this.isTerminated = true
+            // this.isTerminated = true
         }
+        if (this.page !== undefined) {
 
             try {
-            console.log("will open google")
+                logWithDevice("will open google", this.device)
+                if (!this.isTerminated) {
                     const item = searchTexts[Math.floor(Math.random() * searchTexts.length)];
                     await this.page.goto("https://www.google.com/search?q=" + item + "&lr=lang_en", {timeout: 90 * 1000});
+                }
                 // await this.page.goto(RDV_URL, {timeout: 90 * 1000});
             } catch (e) {
                 log(e)
                 this.isTerminated = true
             }
-        // try {
+            await this.acceptCookies();
-        //     if (this.page.url().includes("google")) {
-        //         this.page.focus('button >> nth=3')
-        //         this.page.click()
-        //     }
-        // } catch (e) {
-        //     log(e)
-        // }
             try {
                 const [button] = await this.page.$x("//a[contains(., 'rendezvousparis')]");
                 console.log("button is " + button)
@@ -156,32 +165,41 @@ class CommandorPage {
                     console.log("will click on the button")
                     await button.click();
                 }
-            // await this.page.$eval(':nth-match(:text("rendezvousparis"), 1)', el => {
+                if (this.page.url() === RDV_URL) {
-            //     el.click()
+                    if (!this.isFillingFields)
-            // });
+                        await this.fillFields(this.page, false);
+                }
             } catch (e) {
                 log(e)
                 // if (!this.page.url().includes(RDV_URL)) {
                 //     this.isTerminated = true;
                 // }
             }
+        } else {
+            this.isTerminated = true;
+        }
 
         let cancel
         const intervalTask = setInterval(() => {
             if (this.isTerminated) {
-                log(this.device.model() + ":request terminated, will close device");
+                log(this.device.model() + ":request terminated, send cancel()");
-                if (this.browser !== undefined)
+                // if (this.browser !== undefined)
-                    this.browser.close();
+                //     try {
+                //         log(this.device.model() + ":trying to disconnect browser");
+                //
+                //         if (this.browser.isConnected()) {
+                //             this.browser.disconnect();
+                //         }
+                //     } catch (e) {
+                //         console.log(e);
+                //     }
+
                 // this.page.close()
                 // this.device.close()
                 clearInterval(intervalTask)
                 cancel()
-                return this.browser
+                // return this.browser
             } else {
-                // if (this.page.url() === RDV_URL) {
-                //     if (!this.isFillingFields)
-                //         this.fillFields(this.page);
-                // }
             }
         }, 10 * 1000)//interval of 10 seconds
 
@@ -193,8 +211,22 @@ class CommandorPage {
         }).then(log)
     }
 
+    async acceptCookies() {
+        await this.page.evaluate(() => {
+            let buttons = document.getElementsByTagName('button');
+            for (let i = 0; i < buttons.length; i++) {
+                let button = buttons[i];
+                let txt = button.textContent || button.innerText;
+                if (txt.includes("Tout accepter")) {
+                    button.click()
+                }
+                console.log(txt);
+            }
+        })
+    }
+
     async chooseCountry(page) {
-        if (!page.isClosed()) {
+        if (this.browser.isConnected()) {
             try {
                 if (!this.isCountryChoosen) {
                     await page.focus(COUNTRY_ID);
@@ -214,7 +246,7 @@ class CommandorPage {
 
     async fillEmail(page) {
         try {
-            if (!page.isClosed()) {
+            if (this.browser.isConnected()) {
                 if (!this.isEmailFilled) {
                     await page.focus(EMAIL_ID);
                     await delay(getRandomWaitTime())
@@ -229,7 +261,7 @@ class CommandorPage {
 
     async inputPhoneNumber(page) {
         try {
-            if (!page.isClosed()) {
+            if (this.browser.isConnected()) {
                 if (!this.isPhoneInput) {
                     await page.focus(PHONE_NUMBER);
                     await page.keyboard.type("" + this.contact.phoneNumber);
@@ -245,7 +277,9 @@ class CommandorPage {
 
     async inputName(page) {
         try {
-            if (!page.isClosed()) {
+            logWithDevice("input name called with this.browser.isConnected=" + this.browser.isConnected(), this.device)
+            logWithDevice("input name called with this.isNameInput=" + this.isNameInput, this.device)
+            if (this.browser.isConnected()) {
                 if (!this.isNameInput) {
                     await page.focus(LAST_NAME);
                     await delay(getRandomWaitTime());
@@ -264,7 +298,7 @@ class CommandorPage {
 
     async inputPassportId(page) {
         try {
-            if (!page.isClosed()) {
+            if (this.browser.isConnected()) {
                 if (!this.isPasspordInput) {
                     await page.focus(PASSPORT_ID);
                     await delay(getRandomWaitTime());
@@ -281,7 +315,7 @@ class CommandorPage {
 
     async checkCGU(page) {
         try {
-            if (!page.isClosed()) {
+            if (this.browser.isConnected()) {
                 if (!this.cguChecked) {
                     await page.focus(CGU_ID);
                     await page.evaluate(() => {
@@ -309,7 +343,7 @@ class CommandorPage {
 
     async chooseStore(page) {
         try {
-            if (!page.isClosed()) {
+            if (this.browser.isConnected()) {
                 if (this.selectedStore !== "random") {
                     page.focus(PREFER_STORE);
                     await delay(1000)
@@ -326,17 +360,19 @@ class CommandorPage {
     }
 
 
-    async fillFields(page) {
+    async fillFields(page, airePlanMode) {
-        log("fillFields called for contact:" + this.contact.mail)
+        log("fillFields called for contact: " + this.contact.mail)
-        await this.enableDisableAirPlanMode()
+        log("this.isFillingFields: " + this.isFillingFields);
         if (!this.isFillingFields) {
+            if (airePlanMode) {
+                await this.enableDisableAirPlanMode();
+            }
             this.isFillingFields = true;
             await this.chooseStore(page);
             await this.inputName(page);
             await this.chooseCountry(page);
             await this.inputPhoneNumber(page)
             await this.fillEmail(page)
-            // await this.inputPhoneNumber(page)
             await this.inputPassportId(page)
             await this.checkCGU(page)
             await this.resolveCaptcha(page)
@@ -364,9 +400,11 @@ class CommandorPage {
                         await this.page.evaluate(() => {
                             document.getElementsByClassName("btn")[0].click();
                         })
-                        // this.browser.disconnect();
+                        if (this.firstStart) {
-                        // await this.checkResultWithOcr();
+                            logWithDevice("will disconnect browser", this.device);
-
+                            this.browser.disconnect();
+                            await this.checkResultWithOcr();
+                        }
                     } catch (e) {
                         log(e);
                     }
@@ -387,8 +425,7 @@ class CommandorPage {
         let pageContent = await page.content()
         let hasCaptcha = pageContent.includes("g-recaptcha-response")
         if (hasCaptcha) {
-            // await this.browser.disconnect()
+            // await this.clickCheckbox()
-            await this.clickCheckbox()
             await delay(1000)
             this.captchaSolver = new SolveCaptcha(page);
             await this.captchaSolver.start((solution) => {
@@ -396,6 +433,7 @@ class CommandorPage {
                 if (solution !== ERROR_CAPTCHA_UNSOLVABLE && solution !== TWO_CAPTCHA_CONNECTION_FAILED) {
                     try {
                         if (!page.isClosed()) {
+                            if (this.browser.isConnected()) {
                                 page.evaluate((solution) => {
                                     let element = document.getElementById("g-recaptcha-response");
                                     if (element != null)
@@ -403,6 +441,7 @@ class CommandorPage {
                                 }, solution)
                                 this.clickValid();
                             }
+                        }
                     } catch (e) {
                         log(e)
                         this.isTerminated = true;
@@ -416,14 +455,6 @@ class CommandorPage {
         }
     }
 
-    async isBlocked() {
-        let iframeHandler = await this.page.$("body > iframe");
-        let captcha_container = await iframeHandler.$(CAPTCHA_CONTAINER)
-        let html = await captcha_container.innerHTML()
-        console.log("audio_tag: " + html);
-        return html.includes("You have been blocked") || html.includes("Vous avez été bloqué") || html.includes("封锁")
-    }
-
     async onPageLoad(currentPage) {
         logWithDevice("onPageLoad called with url " + this.page.url(), this.device)
         try {
@@ -445,14 +476,12 @@ class CommandorPage {
                 logWithDevice("ERR_NETWORK_CHANGED, will reload page", this.device);
                 await delay(2000)
                 await this.page.reload()
-            }
+            } else if (content.includes("CACHE_MISS")) {
-                // else if (content.includes("CACHE_MISS")) {
+                logWithDevice("CACHE_MISS, will reload page", this.device);
-                //     logWithDevice("CACHE_MISS, will reload page", this.device);
+                await delay(2000)
-                //     await delay(2000)
+                // await this.setUpCookies()
-                //     // await this.setUpCookies()
+                await this.page.reload()
-                //     await this.page.reload()
+            } else if (content.includes("ERR_TIMED_OUT")) {
-            // }
-            else if (content.includes("ERR_TIMED_OUT")) {
                 logWithDevice("ERR_TIMED_OUT, will reload page", this.device);
                 await delay(2000)
                 await this.page.reload()
@@ -462,8 +491,7 @@ class CommandorPage {
                 await this.page.reload()
             } else {
                 if (this.page.url() === RDV_URL) {
-                    await this.fillFields(this.page);
+                    await this.fillFields(this.page, true);
-                    await this.saveCookies()
                     // if (this.isFillingFields)
                     //     await this.getErrors()
                 } else {
@@ -484,42 +512,9 @@ class CommandorPage {
         }
     }
 
-    async saveCookies() {
-        log("saveCookies() called.")
-        // try {
-        //     let cookies = await this.page.context().cookies();
-        //     let cookiesInJson = [];
-        //     cookies.forEach((cookie) => {
-        //         cookiesInJson.push(JSON.stringify(cookie))
-        //     })
-        //     await require("fs").writeFileSync(this.contact.mail + '.txt', cookiesInJson.join('\n'));
-        // } catch (e) {
-        //     console.log(e)
-        // }
-    }
-
-    async setUpCookies() {
-        // load cookies from file
-        // try {
-        //     logWithDevice("will add cookies", this.device)
-        //     const fs = require('fs');
-        //     // let cookiesFile = fs.readFileSync('vampuka_fisherleyba@aol.com.txt', 'utf8');
-        //     let cookiesFile = fs.readFileSync(this.contact.mail + '.txt', 'utf8');
-        //     let cookiesList = cookiesFile.split("\n")
-        //     let cookiesListToAdd = []
-        //     cookiesList.forEach((cookie) => {
-        //         let cookieToAdd = JSON.parse(cookie)
-        //         // if (cookieToAdd.name === "datadome") {
-        //         cookiesListToAdd.push(cookieToAdd)
-        //         // }
-        //     })
-        //     await this.context.addCookies(cookiesListToAdd)
-        // } catch (e) {
-        //     console.log(e)
-        // }
-    }
 
     async slidingCaptcha(onResult) {
+        logWithDevice("slidingCaptcha", this.device);
         let slidingCaptchaSolver = new SlidingCaptchaSolver(this.device);
         await slidingCaptchaSolver.solve(this.page, async (isSuccessful) => {
             console.log("check isAlwaysBlocked")
@@ -527,38 +522,6 @@ class CommandorPage {
         })
     }
 
-    async checkAudioBtn() {
-        let isBlocked = await this.isBlocked()
-        if (!isBlocked) {
-            //try to sliding capthca at first
-            let slidingCaptchaSolver = new SlidingCaptchaSolver(this.device);
-            await slidingCaptchaSolver.solve(this.page, async (isSuccessful) => {
-                console.log("check isAlwaysBlocked")
-                let isAlwaysBlocked = await this.isBlocked();
-                if (isAlwaysBlocked) {
-                    await this.resolveByAudio();
-                }
-            })
-        } else {
-            log("audioBtn not found")
-            console.log("audioBtn not found")
-            logWithDevice("we are blocked", this.device)
-            await this.resetBrowser()
-        }
-    }
-
-    async resolveByAudio() {
-        // let audioBtn = await this.page.frameLocator("iframe").locator("#captcha__audio__button");
-        // log("audioBtn found")
-        // audioBtn.click()
-        // let captchaSolver = new GeoCaptchaSolver(this.page, this.device, this.isTerminated)
-        // await captchaSolver.solve((isSuccessful) => {
-        //     if (!isSuccessful) {
-        //         this.isTerminated = true
-        //     }
-        // })
-    }
-
     async onResponse(response) {
         let rex = new RegExp(REGEX_RDV_URL)
         // log("onResponse with url:" + response.url())
@@ -566,7 +529,6 @@ class CommandorPage {
         if (rex.test(response.url())) {
             logWithDevice("rdv url found:" + response.url(), this.device)
             // save cookies
-            await this.saveCookies();
             await this.push_message_to_db(PublishType.SUCCESS, response.url())
         }
         // if (response.url().includes("geo.captcha-delivery.com/interstitial")) {
@@ -579,6 +541,7 @@ class CommandorPage {
         let url = this.page.url();
         logWithDevice("successful url is " + url, this.device)
         await this.push_message_to_db(publishType, url)
+        this.firstStart = false;
     }
 
     async push_message_to_db(publishType, url) {
@@ -596,17 +559,13 @@ class CommandorPage {
         this.isTerminated = true
     }
 
-    async saveToBlackList() {
-        await this.mongoManager.saveBlackListToDb(new BlackListContactPojo(this.contact))
-        // await this.resetBrowser()
-        this.isTerminated = true
-    }
-
     async deleteFromBlackList() {
         await this.mongoManager.removeFromBlackList(this.contact)
     }
 
     async getErrors() {
+        logWithDevice("getErrors", this.device);
+        await this.checkResultWithOcr();
         if (this.page.url() === BLANK_URL) {
             this.isTerminated = true;
         } else {
@@ -623,6 +582,8 @@ class CommandorPage {
     }
 
     async clickCheckbox() {
+        logWithDevice("clickCheckbox", this.device);
+
         try {
             // let errorItem = await this.page.click("#recaptcha-anchor > div.recaptcha-checkbox-border")
             await this.page.waitForSelector("iframe")
@@ -641,13 +602,20 @@ class CommandorPage {
     }
 
     async resetBrowser() {
-        console.log("will reset browser")
+        logWithDevice("resetBrowser", this.device);
+        this.isFillingFields = false;
+        this.cguChecked = false;
+        this.isNameInput = false;
+        this.isEmailFilled = false;
+        this.isCountryChoosen = false;
+        this.isPhoneInput = false;
+        this.browser.disconnect();
         await this.device.shell("pm clear com.android.chrome")
         await delay(1000)
         await this.device.shell("am start -n com.android.chrome/com.google.android.apps.chrome.Main")
-        await delay(1000)
+        await delay(5000)
-        // await this.skipWelcomePage()
+        await this.checkResultWithOcr();
-        this.isTerminated = true
+
     }
 
 
@@ -656,6 +624,7 @@ class CommandorPage {
         await delay(2000);
         let ocrChecker = new OCRChecker(this.device, this.contact);
         let checkResult = await ocrChecker.get_result();
+        console.log(checkResult);
         while (checkResult === OCRResult.RECHECK) {
             await delay(4000)
             logWithDevice("will recheck OCR", this.device)
@@ -666,9 +635,12 @@ class CommandorPage {
                 checkResult = await ocrChecker.get_result();
                 await this.checkResultWithOcr()
             })
-            await delay(5000)
+            await delay(20 * 1000)
         }
         switch (checkResult) {
+            case OCRResult.FILL_FIELD:
+                await this.fillFields(this.page)
+                break;
             case OCRResult.SUCCESS:
                 //         reconnect to page and get url
                 await this.connect_to_browser(OCRResult.SUCCESS);
@@ -682,9 +654,20 @@ class CommandorPage {
             case OCRResult.BLOCKED:
                 await this.resetBrowser();
                 break;
+            case OCRResult.NEED_TO_CLICK_LATE_BTN:
+                await this.tapLaterBtn()
+                this.firstStart = true;
+                await this.loadPage()
+                break;
         }
     }
 
+    async tapLaterBtn() {
+        logWithDevice("tapLaterBtn", this.device)
+        this.device.shell("input tap " + 385 + " " + 2050)
+        await delay(2000);
+    }
+
     async enableDisableAirPlanMode() {
         logWithDevice("will enable aireplan mode", this.device)
         await this.device.shell("cmd connectivity airplane-mode enable")

src/workers/OCRChecker.js

@@ -25,6 +25,9 @@ const BLOCKED_MSG_FR = "avez été bloqué"
 const CHECKING_MSG_FR = "Verifying"
 const ERR_CACHE_MISS = "ERR_CACHE_MISS"
 const SLIDING_CAPTCHA_FR = "Glissez vers la droite"
+const MESSAGE_FILL_FIELD_FR = "Demande de rendez-vous pour"
+const WELCOME_MESSAGE_FR = "Bienvenue dans Chrome"
+const PAGE_OPTIMIZATION_CHROME_FR = "Oui, j'accepte"
 
 class OCRChecker {
 
@@ -55,6 +58,10 @@ class OCRChecker {
             return OCRResult.RECHECK
         } else if (result.includes(SLIDING_CAPTCHA_FR)) {
             return OCRResult.SLIDING_CAPTCHA
+        } else if (result.includes(MESSAGE_FILL_FIELD_FR)) {
+            return OCRResult.FILL_FIELD
+        } else if (result.includes(WELCOME_MESSAGE_FR)) {
+            return OCRResult.NEED_TO_CLICK_LATE_BTN
         }
     }