From 4de3bf606c94e384208fcf82e8075b48ef1b6c07 Mon Sep 17 00:00:00 2001 From: Lei PAN Date: Fri, 26 May 2023 23:38:06 +0200 Subject: [PATCH] use jimp --- package.json | 1 + src/models/OCRResult.js | 1 + src/workers/CommandorPage.js | 159 ++++++++++++++++++----------------- src/workers/OCRChecker.js | 72 ++++++++++------ 4 files changed, 128 insertions(+), 105 deletions(-) diff --git a/package.json b/package.json index c3183e8..5815948 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "@nodegui/nodegui": "^0.57.1", "alert": "^5.1.1", "axios": "^0.27.2", + "jimp": "^0.22.8", "mongodb": "^4.9.1", "mongoose": "^6.5.4", "node-schedule": "^2.1.0", diff --git a/src/models/OCRResult.js b/src/models/OCRResult.js index 13dfef4..9d13f06 100644 --- a/src/models/OCRResult.js +++ b/src/models/OCRResult.js @@ -10,6 +10,7 @@ const OCRResult = { CONFIRM_RESEND_FORM: Symbol("CONFIRM_RESEND_FORM"), RECHECK: Symbol("RECHECK"), CLOSED: Symbol("CLOSED"), + TERMINAED: Symbol("TERMINAED"), RECAPTCHA_ERROR: Symbol("RECAPTCHA_ERROR"), } module.exports = OCRResult \ No newline at end of file diff --git a/src/workers/CommandorPage.js b/src/workers/CommandorPage.js index abf6c5c..c27d1b4 100644 --- a/src/workers/CommandorPage.js +++ b/src/workers/CommandorPage.js @@ -5,14 +5,9 @@ const appointmentLogger = require("../utiles/LoggerUtils") const OCRResult = require("../models/OCRResult"); const PublishType = require("../models/PublishType"); const puppeteer = require('puppeteer'); - -const { - shell -} = require('electron') const GeoCaptchaSolver = require("./GeoCaptchaSolver"); const SlidingCaptchaSolver = require("./SlidingCaptchaSolver"); const OCRChecker = require("./OCRChecker"); -const {browser} = require("yarn/lib/cli"); const {disconnect} = require("mongoose"); const {execSync} = require("child_process"); // const RDV_URL = "http://192.168.0.44:8000/test_appointment.html" @@ -73,6 +68,7 @@ class CommandorPage { this.selectedStore = selectedStore; this.choosedStore = selectedStore; this.port = port; + this.ocrChecker = new OCRChecker(this.device, this.contact); // this.browserPackageName = "com.brave.browser"; this.browserPackageName = "com.android.chrome"; this.isFillingFields = false; @@ -88,34 +84,45 @@ class CommandorPage { async connect_to_browser(ocrResult) { console.log("connect_to_browser() called"); - // console.log("browser.isConnected: " + this.browser.isConnected()); + console.log("browser.isConnected: " + this.browser.isConnected()); // if (!this.browser.isConnected()) { - // this.browser = await puppeteer.connect({ - // browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser", - // headless: false, defaultViewport: null - // }) + this.browser = await puppeteer.connect({ + browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser", + headless: false, defaultViewport: null + }) // } - // let pages = await this.browser.pages(); + let pages = await this.browser.pages(); // 0 is the last page + pages.forEach((currentPage) => { + let url = currentPage.url() + if (url.includes(RDV_URL)) { + this.page = currentPage + } + }) + // this.page = pages[pages.length - 1]; // this.page = pages[0]; + logWithDevice("pageUrl is " + this.page.url(), this.device) // this.page = pages[pages.length - 1]; switch (ocrResult) { case OCRResult.SUCCESS: // get url and push to server - logWithDevice("will save success appointment", this.device) - await this.push_message_to_queue(PublishType.SUCCESS); + await this.push_message_to_queue(PublishType.SUCCESS) break; case OCRResult.TO_REFRESH: logWithDevice("will reload page", this.device) - // await this.page.reload(); + // if (this.browser.isConnected()) { + // await this.page.reload(); + // await this.checkIfSuccessful() + // } else { // await delay(10000); // if (!await this.checkIfSuccessful()) { - let cmd = "input swipe 382 482 382 1482" + let cmd = "input swipe 382 482 382 1582" logWithDevice("will send cmd:" + cmd, this.device) this.device.shell(cmd); - await delay(2000); + await delay(3000); await this.checkResultWithOcr(); // } + // } break; } } @@ -191,18 +198,7 @@ class CommandorPage { const intervalTask = setInterval(() => { if (this.isTerminated) { log(this.device.model() + ":request terminated, send cancel()"); - // if (this.browser !== undefined) - // try { - // log(this.device.model() + ":trying to disconnect browser"); - // - // if (this.browser.isConnected()) { - // this.browser.disconnect(); - // } - // } catch (e) { - // console.log(e); - // } - - // this.page.close() + this.resetBrowser(); clearInterval(intervalTask) cancel() // return this.browser @@ -552,11 +548,6 @@ class CommandorPage { logWithDevice("ERR_NETWORK_CHANGED, will reload page", this.device); await delay(2000) await this.page.reload() - } else if (content.includes("CACHE_MISS")) { - logWithDevice("CACHE_MISS, will reload page", this.device); - await delay(2000) - // await this.setUpCookies() - await this.page.reload() } else if (content.includes("ERR_TIMED_OUT")) { logWithDevice("ERR_TIMED_OUT, will reload page", this.device); await delay(2000) @@ -613,7 +604,7 @@ class CommandorPage { let url = this.page.url(); logWithDevice("successful url is " + url, this.device) await this.push_message_to_db(publishType, url) - this.firstStart = false; + // this.firstStart = false; } async push_message_to_db(publishType, url) { @@ -695,10 +686,6 @@ class CommandorPage { } - canContinue() { - return this.browser.isConnected() - } - disconnectBrowser() { try { logWithDevice("will disconnect browser", this.device); @@ -711,52 +698,61 @@ class CommandorPage { async checkResultWithOcr() { console.log("checkResultWithOcr() called.") await delay(2000); - let ocrChecker = new OCRChecker(this.device, this.contact); - let checkResult = await ocrChecker.get_result(); + let checkResult = await this.ocrChecker.get_result(); console.log(checkResult); while (checkResult === OCRResult.RECHECK) { logWithDevice("will recheck OCR", this.device) await delay(4000) logWithDevice("will recheck OCR", this.device) - checkResult = await ocrChecker.get_result(); + checkResult = await this.ocrChecker.get_result(); } while (checkResult === OCRResult.SLIDING_CAPTCHA) { await this.slidingCaptcha(async () => { - checkResult = await ocrChecker.get_result(); - await this.checkResultWithOcr() + checkResult = await this.ocrChecker.get_result(); + // await this.checkResultWithOcr() }) - await delay(20 * 1000) + await delay(10 * 1000) } switch (checkResult) { + case OCRResult.TERMINAED: + this.isTerminated = true; + break; case OCRResult.FILL_FIELD: console.log("browser.isConnected: " + this.browser.isConnected()); while (!this.browser.isConnected()) { logWithDevice("trying to connect to browser", this.device) - // let cmd = 'adb -s ' + this.device.serial() + " forward tcp:" + this.port + " localabstract:chrome_devtools_remote"; - // console.log("cmd is " + cmd); - // const output = execSync(cmd, {encoding: 'utf-8'}); // the default is 'buffer' - // console.log('Output was:\n', output); - this.browser = await puppeteer.connect({ - browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser", - headless: false, defaultViewport: null - }) - let pages = await this.browser.pages(); - pages.forEach((currentPage) => { - if (currentPage.url() === RDV_URL) { - this.page = currentPage; - } - }) - await this.page.bringToFront(); - // this.page = pages; - // this.page.await - await delay(5 * 1000); + try { + this.browser = await puppeteer.connect({ + browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser", + headless: false, defaultViewport: null + }) + let pages = await this.browser.pages(); + pages.forEach((currentPage) => { + if (currentPage.url() === RDV_URL) { + this.page = currentPage; + } + }) + await this.page.bringToFront(); + // this.page = pages; + // this.page.await + await delay(2 * 1000); + } catch (e) { + console.log(e) + this.isTerminated = true + } + } await this.fillFields(this.page) break; case OCRResult.SUCCESS: // reconnect to page and get url if (!this.isTerminated) { - await this.connect_to_browser(OCRResult.SUCCESS); + await this.connect_to_browser(OCRResult.SUCCESS) + // if (!this.isTerminated) { + // logWithDevice("will save success appointment", this.device) + // await this.push_message_to_queue(PublishType.SUCCESS); + // await this.closePage() + // } } break; case OCRResult.RECAPTCHA_ERROR: @@ -773,8 +769,12 @@ class CommandorPage { break; case OCRResult.CONFIRM_RESEND_FORM: logWithDevice("CONFIRM_RESEND_FORM", this.device) - this.device.shell("input tap " + 871 + " " + 1544) + if (this.device.model() === "CPH2219") { + this.device.shell("input tap " + 870 + " " + 1532) + } else + this.device.shell("input tap " + 884 + " " + 1543) await delay(2000); + await this.checkResultWithOcr(); break; case OCRResult.PAGE_OPTIMIZATION: logWithDevice("PAGE_OPTIMIZATION", this.device) @@ -782,7 +782,7 @@ class CommandorPage { await delay(2000); this.device.shell("input tap " + 800 + " " + 2215) await delay(1000); - checkResult = ocrChecker.get_result(); + checkResult = this.ocrChecker.get_result(); break case OCRResult.NEED_TO_CLICK_LATE_BTN: await this.tapLaterBtn() @@ -795,27 +795,28 @@ class CommandorPage { break; default: await delay(5000); - checkResult = ocrChecker.get_result(); + checkResult = this.ocrChecker.get_result(); } if (checkResult === undefined) { await delay(5000); - checkResult = ocrChecker.get_result(); + checkResult = this.ocrChecker.get_result(); } } async closePage() { - while (!this.browser.isConnected()) { - logWithDevice("trying to connect to browser", this.device) - this.browser = await puppeteer.connect({ - browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser", - headless: false, defaultViewport: null - }) - await delay(2 * 1000); - } - console.log("will close all pages"); - let pages = await this.browser.pages(); - await pages[0].close() - console.log("page closed"); + await this.resetBrowser(); + // while (!this.browser.isConnected()) { + // logWithDevice("trying to connect to browser", this.device) + // this.browser = await puppeteer.connect({ + // browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser", + // headless: false, defaultViewport: null + // }) + // await delay(2 * 1000); + // } + // console.log("will close all pages"); + // let pages = await this.browser.pages(); + // await pages[0].close() + // console.log("page closed"); // pages.forEach((currentPage) => { // currentPage.close(); // }) diff --git a/src/workers/OCRChecker.js b/src/workers/OCRChecker.js index 8c8bd5d..02fe808 100644 --- a/src/workers/OCRChecker.js +++ b/src/workers/OCRChecker.js @@ -1,6 +1,7 @@ const {v4: uuidv4} = require("uuid"); const tesseract = require("node-tesseract-ocr"); const OCRResult = require("../models/OCRResult"); +const Jimp = require('jimp'); function delay(delayInMs) { return new Promise(resolve => { @@ -30,9 +31,18 @@ const WELCOME_MESSAGE_FR = "Bienvenue dans Chrome" const PAGE_OPTIMIZATION_CHROME_FR = "Vous pouvez changer d'avis a tout moment dans" const PAGE_OPTIMIZATION_CHROME_FR_2 = "Vous pouvez modifier vos options a tout moment" const PAGE_OPTIMIZATION_CHROME_FR_3 = "Vous pouvez effectuer des modifications" +const PAGE_OPTIMIZATION_CHROME_FR_4 = "de nouvelles fonctionnalités" +const PAGE_OPTIMIZATION_CHROME_FR_5 = "Avec la mesure des performance" const ONLINE_APPOINTMENT = "Online Appointment" const CONFIRM_RESEND_FORM_FR = "Confirmer le nouvel envoi" const CLOSED_MESSAGE_FR = "Depuis plus de 130 ans" +const ABOUT_BLANK = " about:blank" + +async function convertImageToWhiteBlack(image_path) { + const image = await Jimp.read(image_path); + image.grayscale().write(image_path + ".wb"); + return image_path + ".wb"; +} class OCRChecker { @@ -46,36 +56,46 @@ class OCRChecker { return this.contact.passportNumber + "_" + uuid + ".png" } + async get_result() { let fileName = await this.take_screen_shot() - let result = await tesseract - .recognize(fileName, config) - console.log(result) - if (result.includes(MESSAGE_URL_VALIDATION_EN) || result.includes(MESSAGE_URL_VALIDATION_FR)) { - return OCRResult.SUCCESS - } else if (result.includes(CAPTCHA_ERROR_MESSAGE) || result.includes(CAPTCHA_ERROR_MESSAGE_FR)) { - return OCRResult.RECAPTCHA_ERROR - } else if (result.includes(BLOCKED_MSG_EN) || result.includes(BLOCKED_MSG_FR)) { - return OCRResult.BLOCKED - } else if (result.includes(ERR_CACHE_MISS)) { - return OCRResult.TO_REFRESH - } else if (result.includes(CHECKING_MSG_FR)) { + let screenShot = await convertImageToWhiteBlack(fileName); + try { + let result = await tesseract + .recognize(screenShot, config) + console.log(result) + if (result.includes(MESSAGE_URL_VALIDATION_EN) || result.includes(MESSAGE_URL_VALIDATION_FR)) { + return OCRResult.SUCCESS + } else if (result.includes(CAPTCHA_ERROR_MESSAGE) || result.includes(CAPTCHA_ERROR_MESSAGE_FR)) { + return OCRResult.RECAPTCHA_ERROR + } else if (result.includes(BLOCKED_MSG_EN) || result.includes(BLOCKED_MSG_FR)) { + return OCRResult.BLOCKED + } else if (result.includes(ERR_CACHE_MISS)) { + return OCRResult.TO_REFRESH + } else if (result.includes(CHECKING_MSG_FR)) { + return OCRResult.RECHECK + } else if (result.includes(SLIDING_CAPTCHA_FR)) { + return OCRResult.SLIDING_CAPTCHA + } else if (result.includes(MESSAGE_FILL_FIELD_FR)) { + return OCRResult.FILL_FIELD + } else if (result.includes(WELCOME_MESSAGE_FR)) { + return OCRResult.NEED_TO_CLICK_LATE_BTN + } else if (result.toLowerCase().includes(ONLINE_APPOINTMENT.toLowerCase())) { + return OCRResult.ONLINE_APPOINTMENT + } else if (result.includes(PAGE_OPTIMIZATION_CHROME_FR) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_2) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_3) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_4) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_5)) { + return OCRResult.PAGE_OPTIMIZATION + } else if (result.includes(CONFIRM_RESEND_FORM_FR)) { + return OCRResult.CONFIRM_RESEND_FORM + } else if (result.includes(CLOSED_MESSAGE_FR)) { + return OCRResult.CLOSED + } else if (result.includes(ABOUT_BLANK)) { + return OCRResult.TERMINAED + } + } catch (e) { + console.log(e) return OCRResult.RECHECK - } else if (result.includes(SLIDING_CAPTCHA_FR)) { - return OCRResult.SLIDING_CAPTCHA - } else if (result.includes(MESSAGE_FILL_FIELD_FR)) { - return OCRResult.FILL_FIELD - } else if (result.includes(WELCOME_MESSAGE_FR)) { - return OCRResult.NEED_TO_CLICK_LATE_BTN - } else if (result.toLowerCase().includes(ONLINE_APPOINTMENT.toLowerCase())) { - return OCRResult.ONLINE_APPOINTMENT - } else if (result.includes(PAGE_OPTIMIZATION_CHROME_FR) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_2) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_3)) { - return OCRResult.PAGE_OPTIMIZATION - } else if (result.includes(CONFIRM_RESEND_FORM_FR)) { - return OCRResult.CONFIRM_RESEND_FORM - } else if (result.includes(CLOSED_MESSAGE_FR)) { - return OCRResult.CLOSED } + } async take_screen_shot() {