From fd5d5253e533a3629d2e92ec711d815bb99cb2cc Mon Sep 17 00:00:00 2001 From: PAN Lei Date: Mon, 22 May 2023 21:52:39 +0200 Subject: [PATCH] should use adb to refresh page --- src/appointment.js | 20 ++- src/models/OCRResult.js | 2 + src/workers/CommandorPage.js | 321 +++++++++++++++++------------------ src/workers/OCRChecker.js | 7 + 4 files changed, 180 insertions(+), 170 deletions(-) diff --git a/src/appointment.js b/src/appointment.js index 53d4549..96d3b40 100644 --- a/src/appointment.js +++ b/src/appointment.js @@ -14,7 +14,7 @@ let collectionName = formatDate(new Date()) // device_to_excludes = ["47e7e36b", "e30eb015"] // device_to_excludes = ["47e7e36b"] // device_to_excludes = ["J4AXB761H2322WJ"] -device_to_excludes = ["e30eb015"] +device_to_excludes = ["e30eb015", "07fbd156", "hi7ljr5xduyt9pfi"] attributedPorts = [] const device_port_info = new Map(); startPort = 9000 @@ -151,6 +151,21 @@ async function scheduleBookWithNumbers(startNumber, endNumber, selectedStore, pa }) } +function shuffle(array) { + let currentIndex = array.length, randomIndex; + // While there remain elements to shuffle. + while (currentIndex !== 0) { + // Pick a remaining element. + randomIndex = Math.floor(Math.random() * currentIndex); + currentIndex--; + // And swap it with the current element. + [array[currentIndex], array[randomIndex]] = [ + array[randomIndex], array[currentIndex]]; + } + + return array; +} + async function startBookWithNumbers(startNumber, endNumber, selectedStore, pathToExcelFile = '/Users/lpan/Desktop/contact_all.xlsx', audioAnalyse = true, alertBeep = false) { console.log("startBookWithNumbers() called, with alertBeep:" + alertBeep) console.log("startBookWithNumbers() called, with audioAnalyse:" + audioAnalyse) @@ -183,6 +198,8 @@ async function startBookWithNumbers(startNumber, endNumber, selectedStore, pathT const output = execSync(cmd, {encoding: 'utf-8'}); // the default is 'buffer' console.log('Output was:\n', output); } + // start chrome + device.shell("am start -n com.android.chrome/com.google.android.apps.chrome.Main") return attributedPort } @@ -190,6 +207,7 @@ async function startBookWithNumbers(startNumber, endNumber, selectedStore, pathT filterAlreadyBookedContacts(contactList).then((listToBook) => { filterAlreadyAcceptedContacts(listToBook).then(notAcceptedContacts => { filterBlacklistedContacts(notAcceptedContacts).then(listWithoutBlackContact => { + console.log("number of contacts to book:" + listWithoutBlackContact.length) android.devices().then((devices) => { if (devices.length === 0) { diff --git a/src/models/OCRResult.js b/src/models/OCRResult.js index e3258f2..fb156aa 100644 --- a/src/models/OCRResult.js +++ b/src/models/OCRResult.js @@ -3,6 +3,8 @@ const OCRResult = { TO_REFRESH: Symbol("TO_REFRESH"), BLOCKED: Symbol("BLOCKED"), SLIDING_CAPTCHA: Symbol("SLIDING_CAPTCHA"), + FILL_FIELD: Symbol("FILL_FIELD"), + NEED_TO_CLICK_LATE_BTN: Symbol("NEED_TO_CLICK_LATE_BTN"), RECHECK: Symbol("RECHECK"), RECAPTCHA_ERROR: Symbol("RECAPTCHA_ERROR"), } diff --git a/src/workers/CommandorPage.js b/src/workers/CommandorPage.js index 0f88081..84ccecc 100644 --- a/src/workers/CommandorPage.js +++ b/src/workers/CommandorPage.js @@ -72,27 +72,28 @@ class CommandorPage { this.port = port; this.isFillingFields = false; this.isTerminated = false; - this.audioAnalyse = audioAnalyse; - this.alertBeep = alertBeep; this.cguChecked = false; this.isNameInput = false; this.isEmailFilled = false; this.isCountryChoosen = false; this.isPhoneInput = false; + this.firstStart = false; } async connect_to_browser(ocrResult) { console.log("connect_to_browser() called"); - if (!this.browser.isConnected()) { - this.browser = await puppeteer.connect({ - browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser", - headless: false, defaultViewport: null - }) - } + console.log("browser.isConnected: " + this.browser.isConnected()); + // if (!this.browser.isConnected()) { + this.browser = await puppeteer.connect({ + browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser", + headless: false, defaultViewport: null + }) + // } let pages = await this.browser.pages(); // 0 is the last page this.page = pages[0]; + // this.page = pages[pages.length - 1]; switch (ocrResult) { case OCRResult.SUCCESS: // get url and push to server @@ -102,12 +103,23 @@ class CommandorPage { case OCRResult.TO_REFRESH: logWithDevice("will reload page", this.device) await this.page.reload(); - await delay(1000); - await this.checkResultWithOcr(); + await delay(10000); + if (!await this.checkIfSuccessful()) { + await this.checkResultWithOcr(); + } break; } + } + + async checkIfSuccessful() { let content = await this.page.content(); - console.log(content); + if (content.includes(MESSAGE_URL_VALIDATION_FR) || content.includes(MESSAGE_URL_VALIDATION_EN)) { + log("successful"); + await this.push_message_to_queue(PublishType.SUCCESS); + return true; + } else { + return false + } } async loadPage() { @@ -128,60 +140,66 @@ class CommandorPage { this.onResponse(response) }) } catch (e) { + await this.checkResultWithOcr() console.log(e) - this.isTerminated = true + // this.isTerminated = true } + if (this.page !== undefined) { - try { - console.log("will open google") - const item = searchTexts[Math.floor(Math.random() * searchTexts.length)]; - await this.page.goto("https://www.google.com/search?q=" + item + "&lr=lang_en", {timeout: 90 * 1000}); - // await this.page.goto(RDV_URL, {timeout: 90 * 1000}); - } catch (e) { - log(e) - this.isTerminated = true - } - // try { - // if (this.page.url().includes("google")) { - // this.page.focus('button >> nth=3') - // this.page.click() - // } - // } catch (e) { - // log(e) - // } - try { - const [button] = await this.page.$x("//a[contains(., 'rendezvousparis')]"); - console.log("button is " + button) - if (button) { - console.log("will click on the button") - await button.click(); + try { + logWithDevice("will open google", this.device) + if (!this.isTerminated) { + const item = searchTexts[Math.floor(Math.random() * searchTexts.length)]; + await this.page.goto("https://www.google.com/search?q=" + item + "&lr=lang_en", {timeout: 90 * 1000}); + } + // await this.page.goto(RDV_URL, {timeout: 90 * 1000}); + } catch (e) { + log(e) + this.isTerminated = true } - // await this.page.$eval(':nth-match(:text("rendezvousparis"), 1)', el => { - // el.click() - // }); - } catch (e) { - log(e) - // if (!this.page.url().includes(RDV_URL)) { - // this.isTerminated = true; - // } + await this.acceptCookies(); + try { + const [button] = await this.page.$x("//a[contains(., 'rendezvousparis')]"); + console.log("button is " + button) + if (button) { + console.log("will click on the button") + await button.click(); + } + if (this.page.url() === RDV_URL) { + if (!this.isFillingFields) + await this.fillFields(this.page, false); + } + } catch (e) { + log(e) + // if (!this.page.url().includes(RDV_URL)) { + // this.isTerminated = true; + // } + } + } else { + this.isTerminated = true; } let cancel const intervalTask = setInterval(() => { if (this.isTerminated) { - log(this.device.model() + ":request terminated, will close device"); - if (this.browser !== undefined) - this.browser.close(); + log(this.device.model() + ":request terminated, send cancel()"); + // if (this.browser !== undefined) + // try { + // log(this.device.model() + ":trying to disconnect browser"); + // + // if (this.browser.isConnected()) { + // this.browser.disconnect(); + // } + // } catch (e) { + // console.log(e); + // } + // this.page.close() // this.device.close() clearInterval(intervalTask) cancel() - return this.browser + // return this.browser } else { - // if (this.page.url() === RDV_URL) { - // if (!this.isFillingFields) - // this.fillFields(this.page); - // } } }, 10 * 1000)//interval of 10 seconds @@ -193,8 +211,22 @@ class CommandorPage { }).then(log) } + async acceptCookies() { + await this.page.evaluate(() => { + let buttons = document.getElementsByTagName('button'); + for (let i = 0; i < buttons.length; i++) { + let button = buttons[i]; + let txt = button.textContent || button.innerText; + if (txt.includes("Tout accepter")) { + button.click() + } + console.log(txt); + } + }) + } + async chooseCountry(page) { - if (!page.isClosed()) { + if (this.browser.isConnected()) { try { if (!this.isCountryChoosen) { await page.focus(COUNTRY_ID); @@ -214,7 +246,7 @@ class CommandorPage { async fillEmail(page) { try { - if (!page.isClosed()) { + if (this.browser.isConnected()) { if (!this.isEmailFilled) { await page.focus(EMAIL_ID); await delay(getRandomWaitTime()) @@ -229,7 +261,7 @@ class CommandorPage { async inputPhoneNumber(page) { try { - if (!page.isClosed()) { + if (this.browser.isConnected()) { if (!this.isPhoneInput) { await page.focus(PHONE_NUMBER); await page.keyboard.type("" + this.contact.phoneNumber); @@ -245,7 +277,9 @@ class CommandorPage { async inputName(page) { try { - if (!page.isClosed()) { + logWithDevice("input name called with this.browser.isConnected=" + this.browser.isConnected(), this.device) + logWithDevice("input name called with this.isNameInput=" + this.isNameInput, this.device) + if (this.browser.isConnected()) { if (!this.isNameInput) { await page.focus(LAST_NAME); await delay(getRandomWaitTime()); @@ -264,7 +298,7 @@ class CommandorPage { async inputPassportId(page) { try { - if (!page.isClosed()) { + if (this.browser.isConnected()) { if (!this.isPasspordInput) { await page.focus(PASSPORT_ID); await delay(getRandomWaitTime()); @@ -281,7 +315,7 @@ class CommandorPage { async checkCGU(page) { try { - if (!page.isClosed()) { + if (this.browser.isConnected()) { if (!this.cguChecked) { await page.focus(CGU_ID); await page.evaluate(() => { @@ -309,7 +343,7 @@ class CommandorPage { async chooseStore(page) { try { - if (!page.isClosed()) { + if (this.browser.isConnected()) { if (this.selectedStore !== "random") { page.focus(PREFER_STORE); await delay(1000) @@ -326,17 +360,19 @@ class CommandorPage { } - async fillFields(page) { - log("fillFields called for contact:" + this.contact.mail) - await this.enableDisableAirPlanMode() + async fillFields(page, airePlanMode) { + log("fillFields called for contact: " + this.contact.mail) + log("this.isFillingFields: " + this.isFillingFields); if (!this.isFillingFields) { + if (airePlanMode) { + await this.enableDisableAirPlanMode(); + } this.isFillingFields = true; await this.chooseStore(page); await this.inputName(page); await this.chooseCountry(page); await this.inputPhoneNumber(page) await this.fillEmail(page) - // await this.inputPhoneNumber(page) await this.inputPassportId(page) await this.checkCGU(page) await this.resolveCaptcha(page) @@ -364,9 +400,11 @@ class CommandorPage { await this.page.evaluate(() => { document.getElementsByClassName("btn")[0].click(); }) - // this.browser.disconnect(); - // await this.checkResultWithOcr(); - + if (this.firstStart) { + logWithDevice("will disconnect browser", this.device); + this.browser.disconnect(); + await this.checkResultWithOcr(); + } } catch (e) { log(e); } @@ -387,8 +425,7 @@ class CommandorPage { let pageContent = await page.content() let hasCaptcha = pageContent.includes("g-recaptcha-response") if (hasCaptcha) { - // await this.browser.disconnect() - await this.clickCheckbox() + // await this.clickCheckbox() await delay(1000) this.captchaSolver = new SolveCaptcha(page); await this.captchaSolver.start((solution) => { @@ -396,12 +433,14 @@ class CommandorPage { if (solution !== ERROR_CAPTCHA_UNSOLVABLE && solution !== TWO_CAPTCHA_CONNECTION_FAILED) { try { if (!page.isClosed()) { - page.evaluate((solution) => { - let element = document.getElementById("g-recaptcha-response"); - if (element != null) - document.getElementById("g-recaptcha-response").innerHTML = solution; - }, solution) - this.clickValid(); + if (this.browser.isConnected()) { + page.evaluate((solution) => { + let element = document.getElementById("g-recaptcha-response"); + if (element != null) + document.getElementById("g-recaptcha-response").innerHTML = solution; + }, solution) + this.clickValid(); + } } } catch (e) { log(e) @@ -416,14 +455,6 @@ class CommandorPage { } } - async isBlocked() { - let iframeHandler = await this.page.$("body > iframe"); - let captcha_container = await iframeHandler.$(CAPTCHA_CONTAINER) - let html = await captcha_container.innerHTML() - console.log("audio_tag: " + html); - return html.includes("You have been blocked") || html.includes("Vous avez été bloqué") || html.includes("封锁") - } - async onPageLoad(currentPage) { logWithDevice("onPageLoad called with url " + this.page.url(), this.device) try { @@ -445,14 +476,12 @@ class CommandorPage { logWithDevice("ERR_NETWORK_CHANGED, will reload page", this.device); await delay(2000) await this.page.reload() - } - // else if (content.includes("CACHE_MISS")) { - // logWithDevice("CACHE_MISS, will reload page", this.device); - // await delay(2000) - // // await this.setUpCookies() - // await this.page.reload() - // } - else if (content.includes("ERR_TIMED_OUT")) { + } else if (content.includes("CACHE_MISS")) { + logWithDevice("CACHE_MISS, will reload page", this.device); + await delay(2000) + // await this.setUpCookies() + await this.page.reload() + } else if (content.includes("ERR_TIMED_OUT")) { logWithDevice("ERR_TIMED_OUT, will reload page", this.device); await delay(2000) await this.page.reload() @@ -462,8 +491,7 @@ class CommandorPage { await this.page.reload() } else { if (this.page.url() === RDV_URL) { - await this.fillFields(this.page); - await this.saveCookies() + await this.fillFields(this.page, true); // if (this.isFillingFields) // await this.getErrors() } else { @@ -484,42 +512,9 @@ class CommandorPage { } } - async saveCookies() { - log("saveCookies() called.") - // try { - // let cookies = await this.page.context().cookies(); - // let cookiesInJson = []; - // cookies.forEach((cookie) => { - // cookiesInJson.push(JSON.stringify(cookie)) - // }) - // await require("fs").writeFileSync(this.contact.mail + '.txt', cookiesInJson.join('\n')); - // } catch (e) { - // console.log(e) - // } - } - - async setUpCookies() { - // load cookies from file - // try { - // logWithDevice("will add cookies", this.device) - // const fs = require('fs'); - // // let cookiesFile = fs.readFileSync('vampuka_fisherleyba@aol.com.txt', 'utf8'); - // let cookiesFile = fs.readFileSync(this.contact.mail + '.txt', 'utf8'); - // let cookiesList = cookiesFile.split("\n") - // let cookiesListToAdd = [] - // cookiesList.forEach((cookie) => { - // let cookieToAdd = JSON.parse(cookie) - // // if (cookieToAdd.name === "datadome") { - // cookiesListToAdd.push(cookieToAdd) - // // } - // }) - // await this.context.addCookies(cookiesListToAdd) - // } catch (e) { - // console.log(e) - // } - } async slidingCaptcha(onResult) { + logWithDevice("slidingCaptcha", this.device); let slidingCaptchaSolver = new SlidingCaptchaSolver(this.device); await slidingCaptchaSolver.solve(this.page, async (isSuccessful) => { console.log("check isAlwaysBlocked") @@ -527,38 +522,6 @@ class CommandorPage { }) } - async checkAudioBtn() { - let isBlocked = await this.isBlocked() - if (!isBlocked) { - //try to sliding capthca at first - let slidingCaptchaSolver = new SlidingCaptchaSolver(this.device); - await slidingCaptchaSolver.solve(this.page, async (isSuccessful) => { - console.log("check isAlwaysBlocked") - let isAlwaysBlocked = await this.isBlocked(); - if (isAlwaysBlocked) { - await this.resolveByAudio(); - } - }) - } else { - log("audioBtn not found") - console.log("audioBtn not found") - logWithDevice("we are blocked", this.device) - await this.resetBrowser() - } - } - - async resolveByAudio() { - // let audioBtn = await this.page.frameLocator("iframe").locator("#captcha__audio__button"); - // log("audioBtn found") - // audioBtn.click() - // let captchaSolver = new GeoCaptchaSolver(this.page, this.device, this.isTerminated) - // await captchaSolver.solve((isSuccessful) => { - // if (!isSuccessful) { - // this.isTerminated = true - // } - // }) - } - async onResponse(response) { let rex = new RegExp(REGEX_RDV_URL) // log("onResponse with url:" + response.url()) @@ -566,7 +529,6 @@ class CommandorPage { if (rex.test(response.url())) { logWithDevice("rdv url found:" + response.url(), this.device) // save cookies - await this.saveCookies(); await this.push_message_to_db(PublishType.SUCCESS, response.url()) } // if (response.url().includes("geo.captcha-delivery.com/interstitial")) { @@ -579,6 +541,7 @@ class CommandorPage { let url = this.page.url(); logWithDevice("successful url is " + url, this.device) await this.push_message_to_db(publishType, url) + this.firstStart = false; } async push_message_to_db(publishType, url) { @@ -596,17 +559,13 @@ class CommandorPage { this.isTerminated = true } - async saveToBlackList() { - await this.mongoManager.saveBlackListToDb(new BlackListContactPojo(this.contact)) - // await this.resetBrowser() - this.isTerminated = true - } - async deleteFromBlackList() { await this.mongoManager.removeFromBlackList(this.contact) } async getErrors() { + logWithDevice("getErrors", this.device); + await this.checkResultWithOcr(); if (this.page.url() === BLANK_URL) { this.isTerminated = true; } else { @@ -623,6 +582,8 @@ class CommandorPage { } async clickCheckbox() { + logWithDevice("clickCheckbox", this.device); + try { // let errorItem = await this.page.click("#recaptcha-anchor > div.recaptcha-checkbox-border") await this.page.waitForSelector("iframe") @@ -641,13 +602,20 @@ class CommandorPage { } async resetBrowser() { - console.log("will reset browser") + logWithDevice("resetBrowser", this.device); + this.isFillingFields = false; + this.cguChecked = false; + this.isNameInput = false; + this.isEmailFilled = false; + this.isCountryChoosen = false; + this.isPhoneInput = false; + this.browser.disconnect(); await this.device.shell("pm clear com.android.chrome") await delay(1000) await this.device.shell("am start -n com.android.chrome/com.google.android.apps.chrome.Main") - await delay(1000) - // await this.skipWelcomePage() - this.isTerminated = true + await delay(5000) + await this.checkResultWithOcr(); + } @@ -656,6 +624,7 @@ class CommandorPage { await delay(2000); let ocrChecker = new OCRChecker(this.device, this.contact); let checkResult = await ocrChecker.get_result(); + console.log(checkResult); while (checkResult === OCRResult.RECHECK) { await delay(4000) logWithDevice("will recheck OCR", this.device) @@ -666,9 +635,12 @@ class CommandorPage { checkResult = await ocrChecker.get_result(); await this.checkResultWithOcr() }) - await delay(5000) + await delay(20 * 1000) } switch (checkResult) { + case OCRResult.FILL_FIELD: + await this.fillFields(this.page) + break; case OCRResult.SUCCESS: // reconnect to page and get url await this.connect_to_browser(OCRResult.SUCCESS); @@ -682,9 +654,20 @@ class CommandorPage { case OCRResult.BLOCKED: await this.resetBrowser(); break; + case OCRResult.NEED_TO_CLICK_LATE_BTN: + await this.tapLaterBtn() + this.firstStart = true; + await this.loadPage() + break; } } + async tapLaterBtn() { + logWithDevice("tapLaterBtn", this.device) + this.device.shell("input tap " + 385 + " " + 2050) + await delay(2000); + } + async enableDisableAirPlanMode() { logWithDevice("will enable aireplan mode", this.device) await this.device.shell("cmd connectivity airplane-mode enable") diff --git a/src/workers/OCRChecker.js b/src/workers/OCRChecker.js index 54f36f1..abf39f4 100644 --- a/src/workers/OCRChecker.js +++ b/src/workers/OCRChecker.js @@ -25,6 +25,9 @@ const BLOCKED_MSG_FR = "avez été bloqué" const CHECKING_MSG_FR = "Verifying" const ERR_CACHE_MISS = "ERR_CACHE_MISS" const SLIDING_CAPTCHA_FR = "Glissez vers la droite" +const MESSAGE_FILL_FIELD_FR = "Demande de rendez-vous pour" +const WELCOME_MESSAGE_FR = "Bienvenue dans Chrome" +const PAGE_OPTIMIZATION_CHROME_FR = "Oui, j'accepte" class OCRChecker { @@ -55,6 +58,10 @@ class OCRChecker { return OCRResult.RECHECK } else if (result.includes(SLIDING_CAPTCHA_FR)) { return OCRResult.SLIDING_CAPTCHA + } else if (result.includes(MESSAGE_FILL_FIELD_FR)) { + return OCRResult.FILL_FIELD + } else if (result.includes(WELCOME_MESSAGE_FR)) { + return OCRResult.NEED_TO_CLICK_LATE_BTN } }