panleicim/puppeteerjs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

need to use openCv to identifiy text

Browse Source
This commit is contained in:
PAN Lei
2023-05-22 00:57:12 +02:00
parent 0c99c6b36f
commit 48a7338928
7 changed files with 222 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/appointment.js
+7 -3
View File
@@ -14,7 +14,7 @@ let collectionName = formatDate(new Date())
// device_to_excludes = ["47e7e36b", "e30eb015"]
// device_to_excludes = ["47e7e36b"]
// device_to_excludes = ["J4AXB761H2322WJ"]
device_to_excludes = []
device_to_excludes = ["e30eb015"]
attributedPorts = []
const device_port_info = new Map();
startPort = 9000
@@ -171,12 +171,16 @@ async function startBookWithNumbers(startNumber, endNumber, selectedStore, pathT
// get attributed port
let attributedPort = device_port_info[device.serial()]
if (attributedPort) {
const output = execSync('adb -s ' + device.serial() + " forward tcp:" + attributedPort + " localabstract:chrome_devtools_remote", {encoding: 'utf-8'}); // the default is 'buffer'
let cmd = 'adb -s ' + device.serial() + " forward tcp:" + attributedPort + " localabstract:chrome_devtools_remote";
console.log("cmd is " + cmd);
const output = execSync(cmd, {encoding: 'utf-8'}); // the default is 'buffer'
console.log('Output was:\n', output);
} else {
attributedPort = startPort;
startPort++;
const output = execSync('adb -s ' + device.serial() + " forward tcp:" + attributedPort + " localabstract:chrome_devtools_remote", {encoding: 'utf-8'}); // the default is 'buffer'
let cmd = 'adb -s ' + device.serial() + " forward tcp:" + attributedPort + " localabstract:chrome_devtools_remote";
console.log("cmd is " + cmd);
const output = execSync(cmd, {encoding: 'utf-8'}); // the default is 'buffer'
console.log('Output was:\n', output);
}
return attributedPort
src/findTextWithOpenCV.js
+107
View File
@@ -0,0 +1,107 @@
const path = require('path');
const fs = require('fs');
const { cv, drawBlueRect } = require('opencv4nodejs/ut');
// const { extractResults } = require('./dnn/ssdUtils');
if (!cv.xmodules.dnn) {
throw new Error('exiting: opencv4nodejs compiled without dnn module');
}
const modelPath = path.resolve(__dirname,
'../data/text-models/frozen_east_text_detection.pb');
const imgPath = path.resolve(__dirname, '../data/text-data/detection.png');
if (!fs.existsSync(modelPath)) {
console.log('could not find EAST model');
console.log('download the model from: https://github.com/oyyd/frozen_east_text_detection.pb/blob/71415464412c55bb1d135fcdeda498e29a67effa/frozen_east_text_detection.pb?raw=true'
+ ' or create a .pb model from https://github.com/argman/EAST');
throw new Error('exiting: could not find EAST model');
}
const MIN_CONFIDENCE = 0.5;
const NMS_THRESHOLD = 0.4;
const SIZE = 320;
function decode(scores, geometry, confThreshold) {
const [numRows, numCols] = scores.sizes.slice(2);
const boxes = [];
const confidences = [];
for (let y = 0; y < numRows; y += 1) {
for (let x = 0; x < numCols; x += 1) {
const score = scores.at([0, 0, y, x]);
if (score < MIN_CONFIDENCE) {
continue;
}
const offsetX = x * 4;
const offsetY = y * 4;
const angle = geometry.at([0, 4, y, x]);
const cos = Math.cos(angle);
const sin = Math.sin(angle);
const h = geometry.at([0, 0, y, x]) + geometry.at([0, 2, y, x]);
const w = geometry.at([0, 1, y, x]) + geometry.at([0, 3, y, x]);
const endX = offsetX + (cos * geometry.at([0, 1, y, x])) + (sin * geometry.at([0, 2, y, x]));
const endY = offsetY - (sin * geometry.at([0, 1, y, x])) + (cos * geometry.at([0, 2, y, x]));
const startX = endX - w;
const startY = endY - h;
boxes.push(new cv.Rect(
startX,
startY,
endX - startX,
endY - startY,
));
confidences.push(score);
}
}
return [boxes, confidences];
}
function detection(modelAbsPath, imgAbsPath) {
const net = cv.readNetFromTensorflow(modelPath);
const img = cv.imread(imgAbsPath);
const [imgHeight, imgWidth] = img.sizes;
const widthRatio = imgWidth / SIZE;
const heightRatio = imgHeight / SIZE;
const inputBlob = cv.blobFromImage(img, 1,
new cv.Size(SIZE, SIZE), new cv.Vec3(123.68, 116.78, 103.94), true, false);
net.setInput(inputBlob);
const outBlobNames = [
'feature_fusion/Conv_7/Sigmoid',
'feature_fusion/concat_3',
];
const [scores, geometry] = net.forward(outBlobNames);
const [boxes, confidences] = decode(scores, geometry, MIN_CONFIDENCE);
const indices = cv.NMSBoxes(
boxes,
confidences, MIN_CONFIDENCE, NMS_THRESHOLD
);
indices.forEach((i) => {
const rect = boxes[i];
const imgRect = new cv.Rect(
rect.x * widthRatio,
rect.y * heightRatio,
rect.width * widthRatio,
rect.height * heightRatio,
)
drawBlueRect(img, imgRect);
});
cv.imshowWait('EAST text detection', img);
}
detection(
modelPath,
imgPath
);
src/openCvTest.js
+2 -2
View File
@@ -12,8 +12,8 @@ const cv = require('opencv4nodejs');
const findWaldo = async () => {
// Load images
const originalMat = await cv.imreadAsync(`/Users/lpan/Downloads/Screenshot_2023-05-19-14-09-11-82_40deb401b9ffe8e1df2f1cc5ba480b12.jpg`);
const waldoMat = await cv.imreadAsync(`${__dirname}/../assets/templates/valid_btn_template.png`);
const originalMat = await cv.imreadAsync(`/Users/lpan/Downloads/Screenshot_2023-05-20-15-20-22-860_com.android.chrome.jpg`);
const waldoMat = await cv.imreadAsync(`${__dirname}/../assets/templates/welcome_later_btn.png`);
// Match template (the brightest locations indicate the highest match)
const matched = originalMat.matchTemplate(waldoMat, 3);
src/workers/CommandorPage.js
+97 -90
View File
@@ -11,7 +11,6 @@ const {
} = require('electron')
const GeoCaptchaSolver = require("./GeoCaptchaSolver");
const SlidingCaptchaSolver = require("./SlidingCaptchaSolver");
const {de} = require("yarn/lib/cli");
const OCRChecker = require("./OCRChecker");
// const RDV_URL = "http://192.168.0.44:8000/test_appointment.html"
const RDV_URL = "https://rendezvousparis.hermes.com/client/register";
@@ -34,12 +33,6 @@ const TIME_OUT = 60 * 1000 * 4//4 mins
const EMPTY_RESPONSE_ERROR = "ERR_EMPTY_RESPONSE"
const MESSAGE_URL_VALIDATION_FR = "Nous avons envoyé un lien par e-mail."
const MESSAGE_URL_VALIDATION_EN = "Please click on the link we sent by email"
const DOUBLE_REQUEST_ERROR_MESSAGE = "A request with the same data has already been validated today."
const DOUBLE_REQUEST_ERROR_MESSAGE_FR = "Une demande avec les données saisies a déjà été validée aujourd’hui."
const TOO_MANY_REQUEST_ERROR_MESSAGE = "Due to a large number of requests"
const TOO_MANY_REQUEST_ERROR_MESSAGE_FR = "Suite à un trop grand nombre de demandes"
const CAPTCHA_ERROR_MESSAGE = "Error verifying captcha, please try again"
const CAPTCHA_ERROR_MESSAGE_FR = "La vérification du captcha a échoué"
const REGEX_RDV_URL = "https:\/\/rendezvousparis\.hermes\.com\/client\/register\/[A-Z0-9]+"
const DEFAULT_STORE = 'faubourg';
@@ -90,12 +83,16 @@ class CommandorPage {
async connect_to_browser(ocrResult) {
this.browser = await puppeteer.connect({
browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
headless: false, defaultViewport: null
})
console.log("connect_to_browser() called");
if (!this.browser.isConnected()) {
this.browser = await puppeteer.connect({
browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
headless: false, defaultViewport: null
})
}
let pages = await this.browser.pages();
this.page = pages[pages.length - 1]
// 0 is the last page
this.page = pages[0];
switch (ocrResult) {
case OCRResult.SUCCESS:
// get url and push to server
@@ -105,10 +102,9 @@ class CommandorPage {
case OCRResult.TO_REFRESH:
logWithDevice("will reload page", this.device)
await this.page.reload();
logWithDevice("will disconnect browser", this.device)
await this.browser.disconnect()
await delay(1000);
await this.checkResultWithOcr();
break;
}
let content = await this.page.content();
console.log(content);
@@ -116,7 +112,7 @@ class CommandorPage {
async loadPage() {
// Connect to the device.
log("loadPage() called, with port:" + this.port);
logWithDevice("loadPage() called, with port:" + this.port, this.device);
try {
this.browser = await puppeteer.connect({
browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
@@ -145,14 +141,14 @@ class CommandorPage {
log(e)
this.isTerminated = true
}
try {
if (this.page.url().includes("google")) {
this.page.focus('button >> nth=3')
this.page.click()
}
} catch (e) {
log(e)
}
// try {
// if (this.page.url().includes("google")) {
// this.page.focus('button >> nth=3')
// this.page.click()
// }
// } catch (e) {
// log(e)
// }
try {
const [button] = await this.page.$x("//a[contains(., 'rendezvousparis')]");
console.log("button is " + button)
@@ -165,9 +161,9 @@ class CommandorPage {
// });
} catch (e) {
log(e)
if (!this.page.url().includes(RDV_URL)) {
this.isTerminated = true;
}
// if (!this.page.url().includes(RDV_URL)) {
// this.isTerminated = true;
// }
}
let cancel
@@ -182,10 +178,10 @@ class CommandorPage {
cancel()
return this.browser
} else {
if (this.page.url() === RDV_URL) {
if (!this.isFillingFields)
this.fillFields(this.page);
}
// if (this.page.url() === RDV_URL) {
// if (!this.isFillingFields)
// this.fillFields(this.page);
// }
}
}, 10 * 1000)//interval of 10 seconds
@@ -331,7 +327,8 @@ class CommandorPage {
async fillFields(page) {
log("fillFields called")
log("fillFields called for contact:" + this.contact.mail)
await this.enableDisableAirPlanMode()
if (!this.isFillingFields) {
this.isFillingFields = true;
await this.chooseStore(page);
@@ -364,24 +361,12 @@ class CommandorPage {
// remove debug flag
// const validElement = await page.$('.btn');
console.log("will click on valid button");
console.log("will click on valid button");
console.log("will click on valid button");
await this.page.evaluate(() => {
document.getElementsByClassName("btn")[0].click();
})
// this.browser.disconnect();
// await delay(2000);
// let ocrChecker = new OCRChecker(this.device, this.contact);
// let checkResult = await ocrChecker.get_result();
// switch (checkResult) {
// case OCRResult.SUCCESS:
// // reconnect to page and get url
// await this.connect_to_browser(OCRResult.SUCCESS);
// break;
// case OCRResult.BLOCKED:
// await this.resetBrowser();
// break;
// }
// await this.checkResultWithOcr();
} catch (e) {
log(e);
}
@@ -440,24 +425,17 @@ class CommandorPage {
}
async onPageLoad(currentPage) {
logWithDevice("onPageLoad called with url " + this.page.url(), this.device)
try {
let content = await currentPage.content();
let content = await this.page.content();
let captcha_url = "geo.captcha-delivery.com/captcha";
if (content.toString().includes(captcha_url)) {
if (this.audioAnalyse) {
await this.checkAudioBtn();
}
if (this.alertBeep) {
for (let i = 0; i < 15; i++) {
await delay(1000)
shell.beep()
}
}
await this.checkResultWithOcr()
logWithDevice("发现datadome", this.device);
} else if (content.includes("502 Bad Gateway")) {
logWithDevice("502 Bad Gateway found", this.device)
await this.page.reload()
} else if (currentPage.url().includes("sorry")) {
} else if (this.page.url().includes("sorry")) {
await this.resetBrowser()
} else if (content.includes("PROXY_CONNECTION_FAILED")) {
logWithDevice("PROXY_CONNECTION_FAILED, will reload page", this.device);
@@ -467,12 +445,14 @@ class CommandorPage {
logWithDevice("ERR_NETWORK_CHANGED, will reload page", this.device);
await delay(2000)
await this.page.reload()
} else if (content.includes("CACHE_MISS")) {
logWithDevice("CACHE_MISS, will reload page", this.device);
await delay(2000)
// await this.setUpCookies()
await this.page.reload()
} else if (content.includes("ERR_TIMED_OUT")) {
}
// else if (content.includes("CACHE_MISS")) {
// logWithDevice("CACHE_MISS, will reload page", this.device);
// await delay(2000)
// // await this.setUpCookies()
// await this.page.reload()
// }
else if (content.includes("ERR_TIMED_OUT")) {
logWithDevice("ERR_TIMED_OUT, will reload page", this.device);
await delay(2000)
await this.page.reload()
@@ -481,7 +461,7 @@ class CommandorPage {
await delay(2000)
await this.page.reload()
} else {
if (currentPage.url() === RDV_URL) {
if (this.page.url() === RDV_URL) {
await this.fillFields(this.page);
await this.saveCookies()
// if (this.isFillingFields)
@@ -539,6 +519,14 @@ class CommandorPage {
// }
}
async slidingCaptcha(onResult) {
let slidingCaptchaSolver = new SlidingCaptchaSolver(this.device);
await slidingCaptchaSolver.solve(this.page, async (isSuccessful) => {
console.log("check isAlwaysBlocked")
onResult(isSuccessful)
})
}
async checkAudioBtn() {
let isBlocked = await this.isBlocked()
if (!isBlocked) {
@@ -589,6 +577,7 @@ class CommandorPage {
async push_message_to_queue(publishType) {
let url = this.page.url();
logWithDevice("successful url is " + url, this.device)
await this.push_message_to_db(publishType, url)
}
@@ -640,36 +629,11 @@ class CommandorPage {
let elementHandle = await this.page.$('[title="reCAPTCHA"]')
const iframe = await elementHandle.contentFrame()
await iframe.click("#recaptcha-anchor > div.recaptcha-checkbox-border")
// // .getByRole('checkbox', {name: 'I\'m not a robot'})
// if (enCheckbox) {
// enCheckbox.click()
// } else {
// let frCheckbox = await this.page.$('[title="reCAPTCHA"]').getByRole('checkbox', {name: 'Je ne suis pas un robot'})
// if (frCheckbox) {
// frCheckbox.click()
// } else {
// console.log("recaptcha checkbox not found")
// }
// }
} catch (e) {
log(e);
}
}
async handleError(errorContent) {
log("handle error:" + errorContent);
if (errorContent.includes(DOUBLE_REQUEST_ERROR_MESSAGE) || errorContent.includes(DOUBLE_REQUEST_ERROR_MESSAGE_FR)) {
this.isTerminated = true;
} else if (errorContent.includes(TOO_MANY_REQUEST_ERROR_MESSAGE) || errorContent.includes(TOO_MANY_REQUEST_ERROR_MESSAGE_FR)) {
//add contact to black list and set terminated the task
log("handle error: will save to black list db");
await this.saveToBlackList()
} else if (errorContent.includes(CAPTCHA_ERROR_MESSAGE) || errorContent.includes(CAPTCHA_ERROR_MESSAGE_FR)) {
this.isTerminated = true;
}
}
async removeDebugFlag() {
await delay(1000)
await this.device.shell("am clear-debug-app --persistent com.android.chrome")
@@ -680,12 +644,55 @@ class CommandorPage {
console.log("will reset browser")
await this.device.shell("pm clear com.android.chrome")
await delay(1000)
await this.device.shell("pm am start -n com.android.chrome/com.google.android.apps.chrome.Main")
await this.device.shell("am start -n com.android.chrome/com.google.android.apps.chrome.Main")
await delay(1000)
// await this.skipWelcomePage()
this.isTerminated = true
}
async checkResultWithOcr() {
console.log("checkResultWithOcr() called.")
await delay(2000);
let ocrChecker = new OCRChecker(this.device, this.contact);
let checkResult = await ocrChecker.get_result();
while (checkResult === OCRResult.RECHECK) {
await delay(4000)
logWithDevice("will recheck OCR", this.device)
checkResult = await ocrChecker.get_result();
}
while (checkResult === OCRResult.SLIDING_CAPTCHA) {
await this.slidingCaptcha(async () => {
checkResult = await ocrChecker.get_result();
await this.checkResultWithOcr()
})
await delay(5000)
}
switch (checkResult) {
case OCRResult.SUCCESS:
// reconnect to page and get url
await this.connect_to_browser(OCRResult.SUCCESS);
break;
case OCRResult.RECAPTCHA_ERROR:
this.isTerminated = true;
break;
case OCRResult.TO_REFRESH:
await this.connect_to_browser(OCRResult.TO_REFRESH)
break;
case OCRResult.BLOCKED:
await this.resetBrowser();
break;
}
}
async enableDisableAirPlanMode() {
logWithDevice("will enable aireplan mode", this.device)
await this.device.shell("cmd connectivity airplane-mode enable")
await this.device.shell("adb shell settings put global airplane_mode_on 1")
await delay(1000)
await this.device.shell("adb shell settings put global airplane_mode_on 0")
await this.device.shell("cmd connectivity airplane-mode disable")
}
}
module.exports = CommandorPage
src/workers/OCRChecker.js
+8
View File
@@ -23,6 +23,8 @@ const CAPTCHA_ERROR_MESSAGE_FR = "La vérification du captcha a échoué"
const BLOCKED_MSG_EN = "have been blocked"
const BLOCKED_MSG_FR = "avez été bloqué"
const CHECKING_MSG_FR = "Verifying"
const ERR_CACHE_MISS = "ERR_CACHE_MISS"
const SLIDING_CAPTCHA_FR = "Glissez vers la droite"
class OCRChecker {
@@ -47,6 +49,12 @@ class OCRChecker {
return OCRResult.RECAPTCHA_ERROR
} else if (result.includes(BLOCKED_MSG_EN) || result.includes(BLOCKED_MSG_FR)) {
return OCRResult.BLOCKED
} else if (result.includes(ERR_CACHE_MISS)) {
return OCRResult.TO_REFRESH
} else if (result.includes(CHECKING_MSG_FR)) {
return OCRResult.RECHECK
} else if (result.includes(SLIDING_CAPTCHA_FR)) {
return OCRResult.SLIDING_CAPTCHA
}
}