panleicim/puppeteerjs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use jimp

Browse Source
This commit is contained in:
Lei PAN
2023-05-26 23:38:06 +02:00
parent b42d257ab9
commit 4de3bf606c
4 changed files with 128 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package.json
+1
View File
@@ -13,6 +13,7 @@
"@nodegui/nodegui": "^0.57.1", "@nodegui/nodegui": "^0.57.1",
"alert": "^5.1.1", "alert": "^5.1.1",
"axios": "^0.27.2", "axios": "^0.27.2",
"jimp": "^0.22.8",
"mongodb": "^4.9.1", "mongodb": "^4.9.1",
"mongoose": "^6.5.4", "mongoose": "^6.5.4",
"node-schedule": "^2.1.0", "node-schedule": "^2.1.0",
src/models/OCRResult.js
+1
View File
@@ -10,6 +10,7 @@ const OCRResult = {
CONFIRM_RESEND_FORM: Symbol("CONFIRM_RESEND_FORM"), CONFIRM_RESEND_FORM: Symbol("CONFIRM_RESEND_FORM"),
RECHECK: Symbol("RECHECK"), RECHECK: Symbol("RECHECK"),
CLOSED: Symbol("CLOSED"), CLOSED: Symbol("CLOSED"),
TERMINAED: Symbol("TERMINAED"),
RECAPTCHA_ERROR: Symbol("RECAPTCHA_ERROR"), RECAPTCHA_ERROR: Symbol("RECAPTCHA_ERROR"),
} }
module.exports = OCRResult module.exports = OCRResult
src/workers/CommandorPage.js
+80 -79
View File
@@ -5,14 +5,9 @@ const appointmentLogger = require("../utiles/LoggerUtils")
const OCRResult = require("../models/OCRResult"); const OCRResult = require("../models/OCRResult");
const PublishType = require("../models/PublishType"); const PublishType = require("../models/PublishType");
const puppeteer = require('puppeteer'); const puppeteer = require('puppeteer');
const {
shell
} = require('electron')
const GeoCaptchaSolver = require("./GeoCaptchaSolver"); const GeoCaptchaSolver = require("./GeoCaptchaSolver");
const SlidingCaptchaSolver = require("./SlidingCaptchaSolver"); const SlidingCaptchaSolver = require("./SlidingCaptchaSolver");
const OCRChecker = require("./OCRChecker"); const OCRChecker = require("./OCRChecker");
const {browser} = require("yarn/lib/cli");
const {disconnect} = require("mongoose"); const {disconnect} = require("mongoose");
const {execSync} = require("child_process"); const {execSync} = require("child_process");
// const RDV_URL = "http://192.168.0.44:8000/test_appointment.html" // const RDV_URL = "http://192.168.0.44:8000/test_appointment.html"
@@ -73,6 +68,7 @@ class CommandorPage {
this.selectedStore = selectedStore; this.selectedStore = selectedStore;
this.choosedStore = selectedStore; this.choosedStore = selectedStore;
this.port = port; this.port = port;
this.ocrChecker = new OCRChecker(this.device, this.contact);
// this.browserPackageName = "com.brave.browser"; // this.browserPackageName = "com.brave.browser";
this.browserPackageName = "com.android.chrome"; this.browserPackageName = "com.android.chrome";
this.isFillingFields = false; this.isFillingFields = false;
@@ -88,34 +84,45 @@ class CommandorPage {
async connect_to_browser(ocrResult) { async connect_to_browser(ocrResult) {
console.log("connect_to_browser() called"); console.log("connect_to_browser() called");
// console.log("browser.isConnected: " + this.browser.isConnected()); console.log("browser.isConnected: " + this.browser.isConnected());
// if (!this.browser.isConnected()) { // if (!this.browser.isConnected()) {
// this.browser = await puppeteer.connect({ this.browser = await puppeteer.connect({
// browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser", browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
// headless: false, defaultViewport: null headless: false, defaultViewport: null
// }) })
// } // }
// let pages = await this.browser.pages(); let pages = await this.browser.pages();
// 0 is the last page // 0 is the last page
pages.forEach((currentPage) => {
let url = currentPage.url()
if (url.includes(RDV_URL)) {
this.page = currentPage
}
})
// this.page = pages[pages.length - 1];
// this.page = pages[0]; // this.page = pages[0];
logWithDevice("pageUrl is " + this.page.url(), this.device)
// this.page = pages[pages.length - 1]; // this.page = pages[pages.length - 1];
switch (ocrResult) { switch (ocrResult) {
case OCRResult.SUCCESS: case OCRResult.SUCCESS:
// get url and push to server // get url and push to server
logWithDevice("will save success appointment", this.device) await this.push_message_to_queue(PublishType.SUCCESS)
await this.push_message_to_queue(PublishType.SUCCESS);
break; break;
case OCRResult.TO_REFRESH: case OCRResult.TO_REFRESH:
logWithDevice("will reload page", this.device) logWithDevice("will reload page", this.device)
// await this.page.reload(); // if (this.browser.isConnected()) {
// await this.page.reload();
// await this.checkIfSuccessful()
// } else {
// await delay(10000); // await delay(10000);
// if (!await this.checkIfSuccessful()) { // if (!await this.checkIfSuccessful()) {
let cmd = "input swipe 382 482 382 1482" let cmd = "input swipe 382 482 382 1582"
logWithDevice("will send cmd:" + cmd, this.device) logWithDevice("will send cmd:" + cmd, this.device)
this.device.shell(cmd); this.device.shell(cmd);
await delay(2000); await delay(3000);
await this.checkResultWithOcr(); await this.checkResultWithOcr();
// } // }
// }
break; break;
} }
} }
@@ -191,18 +198,7 @@ class CommandorPage {
const intervalTask = setInterval(() => { const intervalTask = setInterval(() => {
if (this.isTerminated) { if (this.isTerminated) {
log(this.device.model() + ":request terminated, send cancel()"); log(this.device.model() + ":request terminated, send cancel()");
// if (this.browser !== undefined) this.resetBrowser();
// try {
// log(this.device.model() + ":trying to disconnect browser");
//
// if (this.browser.isConnected()) {
// this.browser.disconnect();
// }
// } catch (e) {
// console.log(e);
// }
// this.page.close()
clearInterval(intervalTask) clearInterval(intervalTask)
cancel() cancel()
// return this.browser // return this.browser
@@ -552,11 +548,6 @@ class CommandorPage {
logWithDevice("ERR_NETWORK_CHANGED, will reload page", this.device); logWithDevice("ERR_NETWORK_CHANGED, will reload page", this.device);
await delay(2000) await delay(2000)
await this.page.reload() await this.page.reload()
} else if (content.includes("CACHE_MISS")) {
logWithDevice("CACHE_MISS, will reload page", this.device);
await delay(2000)
// await this.setUpCookies()
await this.page.reload()
} else if (content.includes("ERR_TIMED_OUT")) { } else if (content.includes("ERR_TIMED_OUT")) {
logWithDevice("ERR_TIMED_OUT, will reload page", this.device); logWithDevice("ERR_TIMED_OUT, will reload page", this.device);
await delay(2000) await delay(2000)
@@ -613,7 +604,7 @@ class CommandorPage {
let url = this.page.url(); let url = this.page.url();
logWithDevice("successful url is " + url, this.device) logWithDevice("successful url is " + url, this.device)
await this.push_message_to_db(publishType, url) await this.push_message_to_db(publishType, url)
this.firstStart = false; // this.firstStart = false;
} }
async push_message_to_db(publishType, url) { async push_message_to_db(publishType, url) {
@@ -695,10 +686,6 @@ class CommandorPage {
} }
canContinue() {
return this.browser.isConnected()
}
disconnectBrowser() { disconnectBrowser() {
try { try {
logWithDevice("will disconnect browser", this.device); logWithDevice("will disconnect browser", this.device);
@@ -711,52 +698,61 @@ class CommandorPage {
async checkResultWithOcr() { async checkResultWithOcr() {
console.log("checkResultWithOcr() called.") console.log("checkResultWithOcr() called.")
await delay(2000); await delay(2000);
let ocrChecker = new OCRChecker(this.device, this.contact); let checkResult = await this.ocrChecker.get_result();
let checkResult = await ocrChecker.get_result();
console.log(checkResult); console.log(checkResult);
while (checkResult === OCRResult.RECHECK) { while (checkResult === OCRResult.RECHECK) {
logWithDevice("will recheck OCR", this.device) logWithDevice("will recheck OCR", this.device)
await delay(4000) await delay(4000)
logWithDevice("will recheck OCR", this.device) logWithDevice("will recheck OCR", this.device)
checkResult = await ocrChecker.get_result(); checkResult = await this.ocrChecker.get_result();
} }
while (checkResult === OCRResult.SLIDING_CAPTCHA) { while (checkResult === OCRResult.SLIDING_CAPTCHA) {
await this.slidingCaptcha(async () => { await this.slidingCaptcha(async () => {
checkResult = await ocrChecker.get_result(); checkResult = await this.ocrChecker.get_result();
await this.checkResultWithOcr() // await this.checkResultWithOcr()
}) })
await delay(20 * 1000) await delay(10 * 1000)
} }
switch (checkResult) { switch (checkResult) {
case OCRResult.TERMINAED:
this.isTerminated = true;
break;
case OCRResult.FILL_FIELD: case OCRResult.FILL_FIELD:
console.log("browser.isConnected: " + this.browser.isConnected()); console.log("browser.isConnected: " + this.browser.isConnected());
while (!this.browser.isConnected()) { while (!this.browser.isConnected()) {
logWithDevice("trying to connect to browser", this.device) logWithDevice("trying to connect to browser", this.device)
// let cmd = 'adb -s ' + this.device.serial() + " forward tcp:" + this.port + " localabstract:chrome_devtools_remote"; try {
// console.log("cmd is " + cmd); this.browser = await puppeteer.connect({
// const output = execSync(cmd, {encoding: 'utf-8'}); // the default is 'buffer' browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
// console.log('Output was:\n', output); headless: false, defaultViewport: null
this.browser = await puppeteer.connect({ })
browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser", let pages = await this.browser.pages();
headless: false, defaultViewport: null pages.forEach((currentPage) => {
}) if (currentPage.url() === RDV_URL) {
let pages = await this.browser.pages(); this.page = currentPage;
pages.forEach((currentPage) => { }
if (currentPage.url() === RDV_URL) { })
this.page = currentPage; await this.page.bringToFront();
} // this.page = pages;
}) // this.page.await
await this.page.bringToFront(); await delay(2 * 1000);
// this.page = pages; } catch (e) {
// this.page.await console.log(e)
await delay(5 * 1000); this.isTerminated = true
}
} }
await this.fillFields(this.page) await this.fillFields(this.page)
break; break;
case OCRResult.SUCCESS: case OCRResult.SUCCESS:
// reconnect to page and get url // reconnect to page and get url
if (!this.isTerminated) { if (!this.isTerminated) {
await this.connect_to_browser(OCRResult.SUCCESS); await this.connect_to_browser(OCRResult.SUCCESS)
// if (!this.isTerminated) {
// logWithDevice("will save success appointment", this.device)
// await this.push_message_to_queue(PublishType.SUCCESS);
// await this.closePage()
// }
} }
break; break;
case OCRResult.RECAPTCHA_ERROR: case OCRResult.RECAPTCHA_ERROR:
@@ -773,8 +769,12 @@ class CommandorPage {
break; break;
case OCRResult.CONFIRM_RESEND_FORM: case OCRResult.CONFIRM_RESEND_FORM:
logWithDevice("CONFIRM_RESEND_FORM", this.device) logWithDevice("CONFIRM_RESEND_FORM", this.device)
this.device.shell("input tap " + 871 + " " + 1544) if (this.device.model() === "CPH2219") {
this.device.shell("input tap " + 870 + " " + 1532)
} else
this.device.shell("input tap " + 884 + " " + 1543)
await delay(2000); await delay(2000);
await this.checkResultWithOcr();
break; break;
case OCRResult.PAGE_OPTIMIZATION: case OCRResult.PAGE_OPTIMIZATION:
logWithDevice("PAGE_OPTIMIZATION", this.device) logWithDevice("PAGE_OPTIMIZATION", this.device)
@@ -782,7 +782,7 @@ class CommandorPage {
await delay(2000); await delay(2000);
this.device.shell("input tap " + 800 + " " + 2215) this.device.shell("input tap " + 800 + " " + 2215)
await delay(1000); await delay(1000);
checkResult = ocrChecker.get_result(); checkResult = this.ocrChecker.get_result();
break break
case OCRResult.NEED_TO_CLICK_LATE_BTN: case OCRResult.NEED_TO_CLICK_LATE_BTN:
await this.tapLaterBtn() await this.tapLaterBtn()
@@ -795,27 +795,28 @@ class CommandorPage {
break; break;
default: default:
await delay(5000); await delay(5000);
checkResult = ocrChecker.get_result(); checkResult = this.ocrChecker.get_result();
} }
if (checkResult === undefined) { if (checkResult === undefined) {
await delay(5000); await delay(5000);
checkResult = ocrChecker.get_result(); checkResult = this.ocrChecker.get_result();
} }
} }
async closePage() { async closePage() {
while (!this.browser.isConnected()) { await this.resetBrowser();
logWithDevice("trying to connect to browser", this.device) // while (!this.browser.isConnected()) {
this.browser = await puppeteer.connect({ // logWithDevice("trying to connect to browser", this.device)
browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser", // this.browser = await puppeteer.connect({
headless: false, defaultViewport: null // browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
}) // headless: false, defaultViewport: null
await delay(2 * 1000); // })
} // await delay(2 * 1000);
console.log("will close all pages"); // }
let pages = await this.browser.pages(); // console.log("will close all pages");
await pages[0].close() // let pages = await this.browser.pages();
console.log("page closed"); // await pages[0].close()
// console.log("page closed");
// pages.forEach((currentPage) => { // pages.forEach((currentPage) => {
// currentPage.close(); // currentPage.close();
// }) // })
src/workers/OCRChecker.js
+46 -26
View File
@@ -1,6 +1,7 @@
const {v4: uuidv4} = require("uuid"); const {v4: uuidv4} = require("uuid");
const tesseract = require("node-tesseract-ocr"); const tesseract = require("node-tesseract-ocr");
const OCRResult = require("../models/OCRResult"); const OCRResult = require("../models/OCRResult");
const Jimp = require('jimp');
function delay(delayInMs) { function delay(delayInMs) {
return new Promise(resolve => { return new Promise(resolve => {
@@ -30,9 +31,18 @@ const WELCOME_MESSAGE_FR = "Bienvenue dans Chrome"
const PAGE_OPTIMIZATION_CHROME_FR = "Vous pouvez changer d'avis a tout moment dans" const PAGE_OPTIMIZATION_CHROME_FR = "Vous pouvez changer d'avis a tout moment dans"
const PAGE_OPTIMIZATION_CHROME_FR_2 = "Vous pouvez modifier vos options a tout moment" const PAGE_OPTIMIZATION_CHROME_FR_2 = "Vous pouvez modifier vos options a tout moment"
const PAGE_OPTIMIZATION_CHROME_FR_3 = "Vous pouvez effectuer des modifications" const PAGE_OPTIMIZATION_CHROME_FR_3 = "Vous pouvez effectuer des modifications"
const PAGE_OPTIMIZATION_CHROME_FR_4 = "de nouvelles fonctionnalités"
const PAGE_OPTIMIZATION_CHROME_FR_5 = "Avec la mesure des performance"
const ONLINE_APPOINTMENT = "Online Appointment" const ONLINE_APPOINTMENT = "Online Appointment"
const CONFIRM_RESEND_FORM_FR = "Confirmer le nouvel envoi" const CONFIRM_RESEND_FORM_FR = "Confirmer le nouvel envoi"
const CLOSED_MESSAGE_FR = "Depuis plus de 130 ans" const CLOSED_MESSAGE_FR = "Depuis plus de 130 ans"
const ABOUT_BLANK = " about:blank"
async function convertImageToWhiteBlack(image_path) {
const image = await Jimp.read(image_path);
image.grayscale().write(image_path + ".wb");
return image_path + ".wb";
}
class OCRChecker { class OCRChecker {
@@ -46,36 +56,46 @@ class OCRChecker {
return this.contact.passportNumber + "_" + uuid + ".png" return this.contact.passportNumber + "_" + uuid + ".png"
} }
async get_result() { async get_result() {
let fileName = await this.take_screen_shot() let fileName = await this.take_screen_shot()
let result = await tesseract let screenShot = await convertImageToWhiteBlack(fileName);
.recognize(fileName, config) try {
console.log(result) let result = await tesseract
if (result.includes(MESSAGE_URL_VALIDATION_EN) || result.includes(MESSAGE_URL_VALIDATION_FR)) { .recognize(screenShot, config)
return OCRResult.SUCCESS console.log(result)
} else if (result.includes(CAPTCHA_ERROR_MESSAGE) || result.includes(CAPTCHA_ERROR_MESSAGE_FR)) { if (result.includes(MESSAGE_URL_VALIDATION_EN) || result.includes(MESSAGE_URL_VALIDATION_FR)) {
return OCRResult.RECAPTCHA_ERROR return OCRResult.SUCCESS
} else if (result.includes(BLOCKED_MSG_EN) || result.includes(BLOCKED_MSG_FR)) { } else if (result.includes(CAPTCHA_ERROR_MESSAGE) || result.includes(CAPTCHA_ERROR_MESSAGE_FR)) {
return OCRResult.BLOCKED return OCRResult.RECAPTCHA_ERROR
} else if (result.includes(ERR_CACHE_MISS)) { } else if (result.includes(BLOCKED_MSG_EN) || result.includes(BLOCKED_MSG_FR)) {
return OCRResult.TO_REFRESH return OCRResult.BLOCKED
} else if (result.includes(CHECKING_MSG_FR)) { } else if (result.includes(ERR_CACHE_MISS)) {
return OCRResult.TO_REFRESH
} else if (result.includes(CHECKING_MSG_FR)) {
return OCRResult.RECHECK
} else if (result.includes(SLIDING_CAPTCHA_FR)) {
return OCRResult.SLIDING_CAPTCHA
} else if (result.includes(MESSAGE_FILL_FIELD_FR)) {
return OCRResult.FILL_FIELD
} else if (result.includes(WELCOME_MESSAGE_FR)) {
return OCRResult.NEED_TO_CLICK_LATE_BTN
} else if (result.toLowerCase().includes(ONLINE_APPOINTMENT.toLowerCase())) {
return OCRResult.ONLINE_APPOINTMENT
} else if (result.includes(PAGE_OPTIMIZATION_CHROME_FR) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_2) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_3) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_4) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_5)) {
return OCRResult.PAGE_OPTIMIZATION
} else if (result.includes(CONFIRM_RESEND_FORM_FR)) {
return OCRResult.CONFIRM_RESEND_FORM
} else if (result.includes(CLOSED_MESSAGE_FR)) {
return OCRResult.CLOSED
} else if (result.includes(ABOUT_BLANK)) {
return OCRResult.TERMINAED
}
} catch (e) {
console.log(e)
return OCRResult.RECHECK return OCRResult.RECHECK
} else if (result.includes(SLIDING_CAPTCHA_FR)) {
return OCRResult.SLIDING_CAPTCHA
} else if (result.includes(MESSAGE_FILL_FIELD_FR)) {
return OCRResult.FILL_FIELD
} else if (result.includes(WELCOME_MESSAGE_FR)) {
return OCRResult.NEED_TO_CLICK_LATE_BTN
} else if (result.toLowerCase().includes(ONLINE_APPOINTMENT.toLowerCase())) {
return OCRResult.ONLINE_APPOINTMENT
} else if (result.includes(PAGE_OPTIMIZATION_CHROME_FR) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_2) || result.includes(PAGE_OPTIMIZATION_CHROME_FR_3)) {
return OCRResult.PAGE_OPTIMIZATION
} else if (result.includes(CONFIRM_RESEND_FORM_FR)) {
return OCRResult.CONFIRM_RESEND_FORM
} else if (result.includes(CLOSED_MESSAGE_FR)) {
return OCRResult.CLOSED
} }
} }
async take_screen_shot() { async take_screen_shot() {