panleicim/puppeteerjs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use jimp

Browse Source
This commit is contained in:
Lei PAN
2023-05-26 23:38:06 +02:00
parent b42d257ab9
commit 4de3bf606c
4 changed files with 128 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/models/OCRResult.js
+1
View File
@@ -10,6 +10,7 @@ const OCRResult = {
CONFIRM_RESEND_FORM: Symbol("CONFIRM_RESEND_FORM"),
RECHECK: Symbol("RECHECK"),
CLOSED: Symbol("CLOSED"),
TERMINAED: Symbol("TERMINAED"),
RECAPTCHA_ERROR: Symbol("RECAPTCHA_ERROR"),
}
module.exports = OCRResult
src/workers/CommandorPage.js
+80 -79
View File
@@ -5,14 +5,9 @@ const appointmentLogger = require("../utiles/LoggerUtils")
const OCRResult = require("../models/OCRResult");
const PublishType = require("../models/PublishType");
const puppeteer = require('puppeteer');
const {
shell
} = require('electron')
const GeoCaptchaSolver = require("./GeoCaptchaSolver");
const SlidingCaptchaSolver = require("./SlidingCaptchaSolver");
const OCRChecker = require("./OCRChecker");
const {browser} = require("yarn/lib/cli");
const {disconnect} = require("mongoose");
const {execSync} = require("child_process");
// const RDV_URL = "http://192.168.0.44:8000/test_appointment.html"
@@ -73,6 +68,7 @@ class CommandorPage {
this.selectedStore = selectedStore;
this.choosedStore = selectedStore;
this.port = port;
this.ocrChecker = new OCRChecker(this.device, this.contact);
// this.browserPackageName = "com.brave.browser";
this.browserPackageName = "com.android.chrome";
this.isFillingFields = false;
@@ -88,34 +84,45 @@ class CommandorPage {
async connect_to_browser(ocrResult) {
console.log("connect_to_browser() called");
// console.log("browser.isConnected: " + this.browser.isConnected());
console.log("browser.isConnected: " + this.browser.isConnected());
// if (!this.browser.isConnected()) {
// this.browser = await puppeteer.connect({
// browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
// headless: false, defaultViewport: null
// })
this.browser = await puppeteer.connect({
browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
headless: false, defaultViewport: null
})
// }
// let pages = await this.browser.pages();
let pages = await this.browser.pages();
// 0 is the last page
pages.forEach((currentPage) => {
let url = currentPage.url()
if (url.includes(RDV_URL)) {
this.page = currentPage
}
})
// this.page = pages[pages.length - 1];
// this.page = pages[0];
logWithDevice("pageUrl is " + this.page.url(), this.device)
// this.page = pages[pages.length - 1];
switch (ocrResult) {
case OCRResult.SUCCESS:
// get url and push to server
logWithDevice("will save success appointment", this.device)
await this.push_message_to_queue(PublishType.SUCCESS);
await this.push_message_to_queue(PublishType.SUCCESS)
break;
case OCRResult.TO_REFRESH:
logWithDevice("will reload page", this.device)
// await this.page.reload();
// if (this.browser.isConnected()) {
// await this.page.reload();
// await this.checkIfSuccessful()
// } else {
// await delay(10000);
// if (!await this.checkIfSuccessful()) {
let cmd = "input swipe 382 482 382 1482"
let cmd = "input swipe 382 482 382 1582"
logWithDevice("will send cmd:" + cmd, this.device)
this.device.shell(cmd);
await delay(2000);
await delay(3000);
await this.checkResultWithOcr();
// }
// }
break;
}
}
@@ -191,18 +198,7 @@ class CommandorPage {
const intervalTask = setInterval(() => {
if (this.isTerminated) {
log(this.device.model() + ":request terminated, send cancel()");
// if (this.browser !== undefined)
// try {
// log(this.device.model() + ":trying to disconnect browser");
//
// if (this.browser.isConnected()) {
// this.browser.disconnect();
// }
// } catch (e) {
// console.log(e);
// }
// this.page.close()
this.resetBrowser();
clearInterval(intervalTask)
cancel()
// return this.browser
@@ -552,11 +548,6 @@ class CommandorPage {
logWithDevice("ERR_NETWORK_CHANGED, will reload page", this.device);
await delay(2000)
await this.page.reload()
} else if (content.includes("CACHE_MISS")) {
logWithDevice("CACHE_MISS, will reload page", this.device);
await delay(2000)
// await this.setUpCookies()
await this.page.reload()
} else if (content.includes("ERR_TIMED_OUT")) {
logWithDevice("ERR_TIMED_OUT, will reload page", this.device);
await delay(2000)
@@ -613,7 +604,7 @@ class CommandorPage {
let url = this.page.url();
logWithDevice("successful url is " + url, this.device)
await this.push_message_to_db(publishType, url)
this.firstStart = false;
// this.firstStart = false;
}
async push_message_to_db(publishType, url) {
@@ -695,10 +686,6 @@ class CommandorPage {
}
canContinue() {
return this.browser.isConnected()
}
disconnectBrowser() {
try {
logWithDevice("will disconnect browser", this.device);
@@ -711,52 +698,61 @@ class CommandorPage {
async checkResultWithOcr() {
console.log("checkResultWithOcr() called.")
await delay(2000);
let ocrChecker = new OCRChecker(this.device, this.contact);
let checkResult = await ocrChecker.get_result();
let checkResult = await this.ocrChecker.get_result();
console.log(checkResult);
while (checkResult === OCRResult.RECHECK) {
logWithDevice("will recheck OCR", this.device)
await delay(4000)
logWithDevice("will recheck OCR", this.device)
checkResult = await ocrChecker.get_result();
checkResult = await this.ocrChecker.get_result();
}
while (checkResult === OCRResult.SLIDING_CAPTCHA) {
await this.slidingCaptcha(async () => {
checkResult = await ocrChecker.get_result();
await this.checkResultWithOcr()
checkResult = await this.ocrChecker.get_result();
// await this.checkResultWithOcr()
})
await delay(20 * 1000)
await delay(10 * 1000)
}
switch (checkResult) {
case OCRResult.TERMINAED:
this.isTerminated = true;
break;
case OCRResult.FILL_FIELD:
console.log("browser.isConnected: " + this.browser.isConnected());
while (!this.browser.isConnected()) {
logWithDevice("trying to connect to browser", this.device)
// let cmd = 'adb -s ' + this.device.serial() + " forward tcp:" + this.port + " localabstract:chrome_devtools_remote";
// console.log("cmd is " + cmd);
// const output = execSync(cmd, {encoding: 'utf-8'}); // the default is 'buffer'
// console.log('Output was:\n', output);
this.browser = await puppeteer.connect({
browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
headless: false, defaultViewport: null
})
let pages = await this.browser.pages();
pages.forEach((currentPage) => {
if (currentPage.url() === RDV_URL) {
this.page = currentPage;
}
})
await this.page.bringToFront();
// this.page = pages;
// this.page.await
await delay(5 * 1000);
try {
this.browser = await puppeteer.connect({
browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
headless: false, defaultViewport: null
})
let pages = await this.browser.pages();
pages.forEach((currentPage) => {
if (currentPage.url() === RDV_URL) {
this.page = currentPage;
}
})
await this.page.bringToFront();
// this.page = pages;
// this.page.await
await delay(2 * 1000);
} catch (e) {
console.log(e)
this.isTerminated = true
}
}
await this.fillFields(this.page)
break;
case OCRResult.SUCCESS:
// reconnect to page and get url
if (!this.isTerminated) {
await this.connect_to_browser(OCRResult.SUCCESS);
await this.connect_to_browser(OCRResult.SUCCESS)
// if (!this.isTerminated) {
// logWithDevice("will save success appointment", this.device)
// await this.push_message_to_queue(PublishType.SUCCESS);
// await this.closePage()
// }
}
break;
case OCRResult.RECAPTCHA_ERROR:
@@ -773,8 +769,12 @@ class CommandorPage {
break;
case OCRResult.CONFIRM_RESEND_FORM:
logWithDevice("CONFIRM_RESEND_FORM", this.device)
this.device.shell("input tap " + 871 + " " + 1544)
if (this.device.model() === "CPH2219") {
this.device.shell("input tap " + 870 + " " + 1532)
} else
this.device.shell("input tap " + 884 + " " + 1543)
await delay(2000);
await this.checkResultWithOcr();
break;
case OCRResult.PAGE_OPTIMIZATION:
logWithDevice("PAGE_OPTIMIZATION", this.device)
@@ -782,7 +782,7 @@ class CommandorPage {
await delay(2000);
this.device.shell("input tap " + 800 + " " + 2215)
await delay(1000);
checkResult = ocrChecker.get_result();
checkResult = this.ocrChecker.get_result();
break
case OCRResult.NEED_TO_CLICK_LATE_BTN:
await this.tapLaterBtn()
@@ -795,27 +795,28 @@ class CommandorPage {
break;
default:
await delay(5000);
checkResult = ocrChecker.get_result();
checkResult = this.ocrChecker.get_result();
}
if (checkResult === undefined) {
await delay(5000);
checkResult = ocrChecker.get_result();
checkResult = this.ocrChecker.get_result();
}
}
async closePage() {
while (!this.browser.isConnected()) {
logWithDevice("trying to connect to browser", this.device)
this.browser = await puppeteer.connect({
browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
headless: false, defaultViewport: null
})
await delay(2 * 1000);
}
console.log("will close all pages");
let pages = await this.browser.pages();
await pages[0].close()
console.log("page closed");
await this.resetBrowser();
// while (!this.browser.isConnected()) {
// logWithDevice("trying to connect to browser", this.device)
// this.browser = await puppeteer.connect({
// browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
// headless: false, defaultViewport: null
// })
// await delay(2 * 1000);
// }
// console.log("will close all pages");
// let pages = await this.browser.pages();
// await pages[0].close()
// console.log("page closed");
// pages.forEach((currentPage) => {
// currentPage.close();
// })
src/workers/OCRChecker.js
+46 -26
View File
@@ -1,6 +1,7 @@
const {v4: uuidv4} = require("uuid");
const tesseract = require("node-tesseract-ocr");
const OCRResult = require("../models/OCRResult");
const Jimp = require('jimp');
function delay(delayInMs) {
return new Promise(resolve => {
@@ -30,9 +31,18 @@ const WELCOME_MESSAGE_FR = "Bienvenue dans Chrome"
const PAGE_OPTIMIZATION_CHROME_FR = "Vous pouvez changer d'avis a tout moment dans"
const PAGE_OPTIMIZATION_CHROME_FR_2 = "Vous pouvez modifier vos options a tout moment"
const PAGE_OPTIMIZATION_CHROME_FR_3 = "Vous pouvez effectuer des modifications"
const PAGE_OPTIMIZATION_CHROME_FR_4 = "de nouvelles fonctionnalités"
const PAGE_OPTIMIZATION_CHROME_FR_5 = "Avec la mesure des performance"
const ONLINE_APPOINTMENT = "Online Appointment"
const CONFIRM_RESEND_FORM_FR = "Confirmer le nouvel envoi"
const CLOSED_MESSAGE_FR = "Depuis plus de 130 ans"
const ABOUT_BLANK = " about:blank"
async function convertImageToWhiteBlack(image_path) {
const image = await Jimp.read(image_path);
image.grayscale().write(image_path + ".wb");
return image_path + ".wb";
}
class OCRChecker {
@@ -46,36 +56,46 @@ class OCRChecker {
return this.contact.passportNumber + "_" + uuid + ".png"
}
async get_result() {
let fileName = await this.take_screen_shot()
let result = await tesseract
.recognize(fileName, config)
console.log(result)
if (result.includes(MESSAGE_URL_VALIDATION_EN) || result.includes(MESSAGE_URL_VALIDATION_FR)) {
return OCRResult.SUCCESS
} else if (result.includes(CAPTCHA_ERROR_MESSAGE) || result.includes(CAPTCHA_ERROR_MESSAGE_FR)) {
return OCRResult.RECAPTCHA_ERROR
} else if (result.includes(BLOCKED_MSG_EN) || result.includes(BLOCKED_MSG_FR)) {
return OCRResult.BLOCKED
} else if (result.includes(ERR_CACHE_MISS)) {
return OCRResult.TO_REFRESH
} else if (result.includes(CHECKING_MSG_FR)) {
let screenShot = await convertImageToWhiteBlack(fileName);
try {
let result = await tesseract
.recognize(screenShot, config)
console.log(result)
if (result.includes(MESSAGE_URL_VALIDATION_EN) || result.includes(MESSAGE_URL_VALIDATION_FR)) {
return OCRResult.SUCCESS
} else if (result.includes(CAPTCHA_ERROR_MESSAGE) || result.includes(CAPTCHA_ERROR_MESSAGE_FR)) {
return OCRResult.RECAPTCHA_ERROR
} else if (result.includes(BLOCKED_MSG_EN) || result.includes(BLOCKED_MSG_FR)) {
return OCRResult.BLOCKED
} else if (result.includes(ERR_CACHE_MISS)) {
return OCRResult.TO_REFRESH
} else if (result.includes(CHECKING_MSG_FR)) {
return OCRResult.RECHECK
} else if (result.includes(SLIDING_CAPTCHA_FR)) {
return OCRResult.SLIDING_CAPTCHA
} else if (result.includes(MESSAGE_FILL_FIELD_FR)) {
return OCRResult.FILL_FIELD
} else if (result.includes(WELCOME_MESSAGE_FR)) {
return OCRResult.NEED_TO_CLICK_LATE_BTN
} else if (result.toLowerCase().includes(ONLINE_APPOINTMENT.toLowerCase())) {
return OCRResult.ONLINE_APPOINTMENT
} else if (result.includes(PAGE_OPTIMIZATION_CHROME_FR) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_2) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_3) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_4) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_5)) {
return OCRResult.PAGE_OPTIMIZATION
} else if (result.includes(CONFIRM_RESEND_FORM_FR)) {
return OCRResult.CONFIRM_RESEND_FORM
} else if (result.includes(CLOSED_MESSAGE_FR)) {
return OCRResult.CLOSED
} else if (result.includes(ABOUT_BLANK)) {
return OCRResult.TERMINAED
}
} catch (e) {
console.log(e)
return OCRResult.RECHECK
} else if (result.includes(SLIDING_CAPTCHA_FR)) {
return OCRResult.SLIDING_CAPTCHA
} else if (result.includes(MESSAGE_FILL_FIELD_FR)) {
return OCRResult.FILL_FIELD
} else if (result.includes(WELCOME_MESSAGE_FR)) {
return OCRResult.NEED_TO_CLICK_LATE_BTN
} else if (result.toLowerCase().includes(ONLINE_APPOINTMENT.toLowerCase())) {
return OCRResult.ONLINE_APPOINTMENT
} else if (result.includes(PAGE_OPTIMIZATION_CHROME_FR) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_2) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_3)) {
return OCRResult.PAGE_OPTIMIZATION
} else if (result.includes(CONFIRM_RESEND_FORM_FR)) {
return OCRResult.CONFIRM_RESEND_FORM
} else if (result.includes(CLOSED_MESSAGE_FR)) {
return OCRResult.CLOSED
}
}
async take_screen_shot() {