diff --git a/assets/templates/welcome_later_btn.png b/assets/templates/welcome_later_btn.png new file mode 100644 index 0000000..78d0c28 Binary files /dev/null and b/assets/templates/welcome_later_btn.png differ diff --git a/src/appointment.js b/src/appointment.js index ecc1326..53d4549 100644 --- a/src/appointment.js +++ b/src/appointment.js @@ -14,7 +14,7 @@ let collectionName = formatDate(new Date()) // device_to_excludes = ["47e7e36b", "e30eb015"] // device_to_excludes = ["47e7e36b"] // device_to_excludes = ["J4AXB761H2322WJ"] -device_to_excludes = [] +device_to_excludes = ["e30eb015"] attributedPorts = [] const device_port_info = new Map(); startPort = 9000 @@ -171,12 +171,16 @@ async function startBookWithNumbers(startNumber, endNumber, selectedStore, pathT // get attributed port let attributedPort = device_port_info[device.serial()] if (attributedPort) { - const output = execSync('adb -s ' + device.serial() + " forward tcp:" + attributedPort + " localabstract:chrome_devtools_remote", {encoding: 'utf-8'}); // the default is 'buffer' + let cmd = 'adb -s ' + device.serial() + " forward tcp:" + attributedPort + " localabstract:chrome_devtools_remote"; + console.log("cmd is " + cmd); + const output = execSync(cmd, {encoding: 'utf-8'}); // the default is 'buffer' console.log('Output was:\n', output); } else { attributedPort = startPort; startPort++; - const output = execSync('adb -s ' + device.serial() + " forward tcp:" + attributedPort + " localabstract:chrome_devtools_remote", {encoding: 'utf-8'}); // the default is 'buffer' + let cmd = 'adb -s ' + device.serial() + " forward tcp:" + attributedPort + " localabstract:chrome_devtools_remote"; + console.log("cmd is " + cmd); + const output = execSync(cmd, {encoding: 'utf-8'}); // the default is 'buffer' console.log('Output was:\n', output); } return attributedPort diff --git a/src/findTextWithOpenCV.js b/src/findTextWithOpenCV.js new file mode 100644 index 0000000..c6e4baa --- /dev/null +++ b/src/findTextWithOpenCV.js @@ -0,0 +1,107 @@ +const path = require('path'); +const fs = require('fs'); +const { cv, drawBlueRect } = require('opencv4nodejs/ut'); +// const { extractResults } = require('./dnn/ssdUtils'); + +if (!cv.xmodules.dnn) { + throw new Error('exiting: opencv4nodejs compiled without dnn module'); +} + +const modelPath = path.resolve(__dirname, + '../data/text-models/frozen_east_text_detection.pb'); +const imgPath = path.resolve(__dirname, '../data/text-data/detection.png'); + +if (!fs.existsSync(modelPath)) { + console.log('could not find EAST model'); + console.log('download the model from: https://github.com/oyyd/frozen_east_text_detection.pb/blob/71415464412c55bb1d135fcdeda498e29a67effa/frozen_east_text_detection.pb?raw=true' + + ' or create a .pb model from https://github.com/argman/EAST'); + throw new Error('exiting: could not find EAST model'); +} + +const MIN_CONFIDENCE = 0.5; +const NMS_THRESHOLD = 0.4; +const SIZE = 320; + +function decode(scores, geometry, confThreshold) { + const [numRows, numCols] = scores.sizes.slice(2); + const boxes = []; + const confidences = []; + + for (let y = 0; y < numRows; y += 1) { + for (let x = 0; x < numCols; x += 1) { + const score = scores.at([0, 0, y, x]); + + if (score < MIN_CONFIDENCE) { + continue; + } + + const offsetX = x * 4; + const offsetY = y * 4; + const angle = geometry.at([0, 4, y, x]); + const cos = Math.cos(angle); + const sin = Math.sin(angle); + + const h = geometry.at([0, 0, y, x]) + geometry.at([0, 2, y, x]); + const w = geometry.at([0, 1, y, x]) + geometry.at([0, 3, y, x]); + + const endX = offsetX + (cos * geometry.at([0, 1, y, x])) + (sin * geometry.at([0, 2, y, x])); + const endY = offsetY - (sin * geometry.at([0, 1, y, x])) + (cos * geometry.at([0, 2, y, x])); + const startX = endX - w; + const startY = endY - h; + + boxes.push(new cv.Rect( + startX, + startY, + endX - startX, + endY - startY, + )); + confidences.push(score); + } + } + + return [boxes, confidences]; +} + +function detection(modelAbsPath, imgAbsPath) { + const net = cv.readNetFromTensorflow(modelPath); + const img = cv.imread(imgAbsPath); + const [imgHeight, imgWidth] = img.sizes; + const widthRatio = imgWidth / SIZE; + const heightRatio = imgHeight / SIZE; + + const inputBlob = cv.blobFromImage(img, 1, + new cv.Size(SIZE, SIZE), new cv.Vec3(123.68, 116.78, 103.94), true, false); + + net.setInput(inputBlob); + + const outBlobNames = [ + 'feature_fusion/Conv_7/Sigmoid', + 'feature_fusion/concat_3', + ]; + + const [scores, geometry] = net.forward(outBlobNames); + const [boxes, confidences] = decode(scores, geometry, MIN_CONFIDENCE); + + const indices = cv.NMSBoxes( + boxes, + confidences, MIN_CONFIDENCE, NMS_THRESHOLD + ); + + indices.forEach((i) => { + const rect = boxes[i]; + const imgRect = new cv.Rect( + rect.x * widthRatio, + rect.y * heightRatio, + rect.width * widthRatio, + rect.height * heightRatio, + ) + drawBlueRect(img, imgRect); + }); + + cv.imshowWait('EAST text detection', img); +} + +detection( + modelPath, + imgPath +); \ No newline at end of file diff --git a/src/openCvTest.js b/src/openCvTest.js index 5f8f0be..addcd79 100644 --- a/src/openCvTest.js +++ b/src/openCvTest.js @@ -12,8 +12,8 @@ const cv = require('opencv4nodejs'); const findWaldo = async () => { // Load images - const originalMat = await cv.imreadAsync(`/Users/lpan/Downloads/Screenshot_2023-05-19-14-09-11-82_40deb401b9ffe8e1df2f1cc5ba480b12.jpg`); - const waldoMat = await cv.imreadAsync(`${__dirname}/../assets/templates/valid_btn_template.png`); + const originalMat = await cv.imreadAsync(`/Users/lpan/Downloads/Screenshot_2023-05-20-15-20-22-860_com.android.chrome.jpg`); + const waldoMat = await cv.imreadAsync(`${__dirname}/../assets/templates/welcome_later_btn.png`); // Match template (the brightest locations indicate the highest match) const matched = originalMat.matchTemplate(waldoMat, 3); diff --git a/src/workers/CommandorPage.js b/src/workers/CommandorPage.js index 047089f..0f88081 100644 --- a/src/workers/CommandorPage.js +++ b/src/workers/CommandorPage.js @@ -11,7 +11,6 @@ const { } = require('electron') const GeoCaptchaSolver = require("./GeoCaptchaSolver"); const SlidingCaptchaSolver = require("./SlidingCaptchaSolver"); -const {de} = require("yarn/lib/cli"); const OCRChecker = require("./OCRChecker"); // const RDV_URL = "http://192.168.0.44:8000/test_appointment.html" const RDV_URL = "https://rendezvousparis.hermes.com/client/register"; @@ -34,12 +33,6 @@ const TIME_OUT = 60 * 1000 * 4//4 mins const EMPTY_RESPONSE_ERROR = "ERR_EMPTY_RESPONSE" const MESSAGE_URL_VALIDATION_FR = "Nous avons envoyé un lien par e-mail." const MESSAGE_URL_VALIDATION_EN = "Please click on the link we sent by email" -const DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today." -const DOUBLE_REQUEST_ERROR_MESSAGE_FR = "Une demande avec les données saisies a déjà été validée aujourd’hui." -const TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests" -const TOO_MANY_REQUEST_ERROR_MESSAGE_FR = "Suite à un trop grand nombre de demandes" -const CAPTCHA_ERROR_MESSAGE = "Error verifying captcha, please try again" -const CAPTCHA_ERROR_MESSAGE_FR = "La vérification du captcha a échoué" const REGEX_RDV_URL = "https:\/\/rendezvousparis\.hermes\.com\/client\/register\/[A-Z0-9]+" const DEFAULT_STORE = 'faubourg'; @@ -90,12 +83,16 @@ class CommandorPage { async connect_to_browser(ocrResult) { - this.browser = await puppeteer.connect({ - browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser", - headless: false, defaultViewport: null - }) + console.log("connect_to_browser() called"); + if (!this.browser.isConnected()) { + this.browser = await puppeteer.connect({ + browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser", + headless: false, defaultViewport: null + }) + } let pages = await this.browser.pages(); - this.page = pages[pages.length - 1] + // 0 is the last page + this.page = pages[0]; switch (ocrResult) { case OCRResult.SUCCESS: // get url and push to server @@ -105,10 +102,9 @@ class CommandorPage { case OCRResult.TO_REFRESH: logWithDevice("will reload page", this.device) await this.page.reload(); - logWithDevice("will disconnect browser", this.device) - await this.browser.disconnect() + await delay(1000); + await this.checkResultWithOcr(); break; - } let content = await this.page.content(); console.log(content); @@ -116,7 +112,7 @@ class CommandorPage { async loadPage() { // Connect to the device. - log("loadPage() called, with port:" + this.port); + logWithDevice("loadPage() called, with port:" + this.port, this.device); try { this.browser = await puppeteer.connect({ browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser", @@ -145,14 +141,14 @@ class CommandorPage { log(e) this.isTerminated = true } - try { - if (this.page.url().includes("google")) { - this.page.focus('button >> nth=3') - this.page.click() - } - } catch (e) { - log(e) - } + // try { + // if (this.page.url().includes("google")) { + // this.page.focus('button >> nth=3') + // this.page.click() + // } + // } catch (e) { + // log(e) + // } try { const [button] = await this.page.$x("//a[contains(., 'rendezvousparis')]"); console.log("button is " + button) @@ -165,9 +161,9 @@ class CommandorPage { // }); } catch (e) { log(e) - if (!this.page.url().includes(RDV_URL)) { - this.isTerminated = true; - } + // if (!this.page.url().includes(RDV_URL)) { + // this.isTerminated = true; + // } } let cancel @@ -182,10 +178,10 @@ class CommandorPage { cancel() return this.browser } else { - if (this.page.url() === RDV_URL) { - if (!this.isFillingFields) - this.fillFields(this.page); - } + // if (this.page.url() === RDV_URL) { + // if (!this.isFillingFields) + // this.fillFields(this.page); + // } } }, 10 * 1000)//interval of 10 seconds @@ -331,7 +327,8 @@ class CommandorPage { async fillFields(page) { - log("fillFields called") + log("fillFields called for contact:" + this.contact.mail) + await this.enableDisableAirPlanMode() if (!this.isFillingFields) { this.isFillingFields = true; await this.chooseStore(page); @@ -364,24 +361,12 @@ class CommandorPage { // remove debug flag // const validElement = await page.$('.btn'); console.log("will click on valid button"); - console.log("will click on valid button"); - console.log("will click on valid button"); await this.page.evaluate(() => { document.getElementsByClassName("btn")[0].click(); }) // this.browser.disconnect(); - // await delay(2000); - // let ocrChecker = new OCRChecker(this.device, this.contact); - // let checkResult = await ocrChecker.get_result(); - // switch (checkResult) { - // case OCRResult.SUCCESS: - // // reconnect to page and get url - // await this.connect_to_browser(OCRResult.SUCCESS); - // break; - // case OCRResult.BLOCKED: - // await this.resetBrowser(); - // break; - // } + // await this.checkResultWithOcr(); + } catch (e) { log(e); } @@ -440,24 +425,17 @@ class CommandorPage { } async onPageLoad(currentPage) { + logWithDevice("onPageLoad called with url " + this.page.url(), this.device) try { - let content = await currentPage.content(); + let content = await this.page.content(); let captcha_url = "geo.captcha-delivery.com/captcha"; if (content.toString().includes(captcha_url)) { - if (this.audioAnalyse) { - await this.checkAudioBtn(); - } - if (this.alertBeep) { - for (let i = 0; i < 15; i++) { - await delay(1000) - shell.beep() - } - } + await this.checkResultWithOcr() logWithDevice("发现datadome", this.device); } else if (content.includes("502 Bad Gateway")) { logWithDevice("502 Bad Gateway found", this.device) await this.page.reload() - } else if (currentPage.url().includes("sorry")) { + } else if (this.page.url().includes("sorry")) { await this.resetBrowser() } else if (content.includes("PROXY_CONNECTION_FAILED")) { logWithDevice("PROXY_CONNECTION_FAILED, will reload page", this.device); @@ -467,12 +445,14 @@ class CommandorPage { logWithDevice("ERR_NETWORK_CHANGED, will reload page", this.device); await delay(2000) await this.page.reload() - } else if (content.includes("CACHE_MISS")) { - logWithDevice("CACHE_MISS, will reload page", this.device); - await delay(2000) - // await this.setUpCookies() - await this.page.reload() - } else if (content.includes("ERR_TIMED_OUT")) { + } + // else if (content.includes("CACHE_MISS")) { + // logWithDevice("CACHE_MISS, will reload page", this.device); + // await delay(2000) + // // await this.setUpCookies() + // await this.page.reload() + // } + else if (content.includes("ERR_TIMED_OUT")) { logWithDevice("ERR_TIMED_OUT, will reload page", this.device); await delay(2000) await this.page.reload() @@ -481,7 +461,7 @@ class CommandorPage { await delay(2000) await this.page.reload() } else { - if (currentPage.url() === RDV_URL) { + if (this.page.url() === RDV_URL) { await this.fillFields(this.page); await this.saveCookies() // if (this.isFillingFields) @@ -539,6 +519,14 @@ class CommandorPage { // } } + async slidingCaptcha(onResult) { + let slidingCaptchaSolver = new SlidingCaptchaSolver(this.device); + await slidingCaptchaSolver.solve(this.page, async (isSuccessful) => { + console.log("check isAlwaysBlocked") + onResult(isSuccessful) + }) + } + async checkAudioBtn() { let isBlocked = await this.isBlocked() if (!isBlocked) { @@ -589,6 +577,7 @@ class CommandorPage { async push_message_to_queue(publishType) { let url = this.page.url(); + logWithDevice("successful url is " + url, this.device) await this.push_message_to_db(publishType, url) } @@ -640,36 +629,11 @@ class CommandorPage { let elementHandle = await this.page.$('[title="reCAPTCHA"]') const iframe = await elementHandle.contentFrame() await iframe.click("#recaptcha-anchor > div.recaptcha-checkbox-border") - // // .getByRole('checkbox', {name: 'I\'m not a robot'}) - // if (enCheckbox) { - // enCheckbox.click() - // } else { - // let frCheckbox = await this.page.$('[title="reCAPTCHA"]').getByRole('checkbox', {name: 'Je ne suis pas un robot'}) - // if (frCheckbox) { - // frCheckbox.click() - // } else { - // console.log("recaptcha checkbox not found") - // } - // } } catch (e) { log(e); } } - async handleError(errorContent) { - log("handle error:" + errorContent); - if (errorContent.includes(DOUBLE_REQUEST_ERROR_MESSAGE) || errorContent.includes(DOUBLE_REQUEST_ERROR_MESSAGE_FR)) { - this.isTerminated = true; - } else if (errorContent.includes(TOO_MANY_REQUEST_ERROR_MESSAGE) || errorContent.includes(TOO_MANY_REQUEST_ERROR_MESSAGE_FR)) { - //add contact to black list and set terminated the task - log("handle error: will save to black list db"); - await this.saveToBlackList() - } else if (errorContent.includes(CAPTCHA_ERROR_MESSAGE) || errorContent.includes(CAPTCHA_ERROR_MESSAGE_FR)) { - this.isTerminated = true; - } - - } - async removeDebugFlag() { await delay(1000) await this.device.shell("am clear-debug-app --persistent com.android.chrome") @@ -680,12 +644,55 @@ class CommandorPage { console.log("will reset browser") await this.device.shell("pm clear com.android.chrome") await delay(1000) - await this.device.shell("pm am start -n com.android.chrome/com.google.android.apps.chrome.Main") + await this.device.shell("am start -n com.android.chrome/com.google.android.apps.chrome.Main") await delay(1000) + // await this.skipWelcomePage() this.isTerminated = true } + async checkResultWithOcr() { + console.log("checkResultWithOcr() called.") + await delay(2000); + let ocrChecker = new OCRChecker(this.device, this.contact); + let checkResult = await ocrChecker.get_result(); + while (checkResult === OCRResult.RECHECK) { + await delay(4000) + logWithDevice("will recheck OCR", this.device) + checkResult = await ocrChecker.get_result(); + } + while (checkResult === OCRResult.SLIDING_CAPTCHA) { + await this.slidingCaptcha(async () => { + checkResult = await ocrChecker.get_result(); + await this.checkResultWithOcr() + }) + await delay(5000) + } + switch (checkResult) { + case OCRResult.SUCCESS: + // reconnect to page and get url + await this.connect_to_browser(OCRResult.SUCCESS); + break; + case OCRResult.RECAPTCHA_ERROR: + this.isTerminated = true; + break; + case OCRResult.TO_REFRESH: + await this.connect_to_browser(OCRResult.TO_REFRESH) + break; + case OCRResult.BLOCKED: + await this.resetBrowser(); + break; + } + } + + async enableDisableAirPlanMode() { + logWithDevice("will enable aireplan mode", this.device) + await this.device.shell("cmd connectivity airplane-mode enable") + await this.device.shell("adb shell settings put global airplane_mode_on 1") + await delay(1000) + await this.device.shell("adb shell settings put global airplane_mode_on 0") + await this.device.shell("cmd connectivity airplane-mode disable") + } } module.exports = CommandorPage \ No newline at end of file diff --git a/src/workers/OCRChecker.js b/src/workers/OCRChecker.js index 2226052..54f36f1 100644 --- a/src/workers/OCRChecker.js +++ b/src/workers/OCRChecker.js @@ -23,6 +23,8 @@ const CAPTCHA_ERROR_MESSAGE_FR = "La vérification du captcha a échoué" const BLOCKED_MSG_EN = "have been blocked" const BLOCKED_MSG_FR = "avez été bloqué" const CHECKING_MSG_FR = "Verifying" +const ERR_CACHE_MISS = "ERR_CACHE_MISS" +const SLIDING_CAPTCHA_FR = "Glissez vers la droite" class OCRChecker { @@ -47,6 +49,12 @@ class OCRChecker { return OCRResult.RECAPTCHA_ERROR } else if (result.includes(BLOCKED_MSG_EN) || result.includes(BLOCKED_MSG_FR)) { return OCRResult.BLOCKED + } else if (result.includes(ERR_CACHE_MISS)) { + return OCRResult.TO_REFRESH + } else if (result.includes(CHECKING_MSG_FR)) { + return OCRResult.RECHECK + } else if (result.includes(SLIDING_CAPTCHA_FR)) { + return OCRResult.SLIDING_CAPTCHA } } diff --git a/start.js b/start.js index 359a044..fc0d87e 100644 --- a/start.js +++ b/start.js @@ -1,6 +1,6 @@ const {startBookWithNumbers, scheduleBookWithNumbers} = require('./src/appointment') -startBookWithNumbers(0, 2, "random", "./163_contacts.xlsx", true, false).then((r) => { +startBookWithNumbers(70, 200, "random", "/Users/lpan/Desktop/gmail_contacts.xlsx", true, false).then((r) => { console.log(r) })