panleicim/puppeteerjs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

should use adb to refresh page

Browse Source
This commit is contained in:
PAN Lei
2023-05-22 21:52:39 +02:00
parent 48a7338928
commit fd5d5253e5
4 changed files with 180 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/appointment.js
+19 -1
View File
@@ -14,7 +14,7 @@ let collectionName = formatDate(new Date())
// device_to_excludes = ["47e7e36b", "e30eb015"]
// device_to_excludes = ["47e7e36b"]
// device_to_excludes = ["J4AXB761H2322WJ"]
device_to_excludes = ["e30eb015"]
device_to_excludes = ["e30eb015", "07fbd156", "hi7ljr5xduyt9pfi"]
attributedPorts = []
const device_port_info = new Map();
startPort = 9000
@@ -151,6 +151,21 @@ async function scheduleBookWithNumbers(startNumber, endNumber, selectedStore, pa
})
}
function shuffle(array) {
let currentIndex = array.length, randomIndex;
// While there remain elements to shuffle.
while (currentIndex !== 0) {
// Pick a remaining element.
randomIndex = Math.floor(Math.random() * currentIndex);
currentIndex--;
// And swap it with the current element.
[array[currentIndex], array[randomIndex]] = [
array[randomIndex], array[currentIndex]];
}
return array;
}
async function startBookWithNumbers(startNumber, endNumber, selectedStore, pathToExcelFile = '/Users/lpan/Desktop/contact_all.xlsx', audioAnalyse = true, alertBeep = false) {
console.log("startBookWithNumbers() called, with alertBeep:" + alertBeep)
console.log("startBookWithNumbers() called, with audioAnalyse:" + audioAnalyse)
@@ -183,6 +198,8 @@ async function startBookWithNumbers(startNumber, endNumber, selectedStore, pathT
const output = execSync(cmd, {encoding: 'utf-8'}); // the default is 'buffer'
console.log('Output was:\n', output);
}
// start chrome
device.shell("am start -n com.android.chrome/com.google.android.apps.chrome.Main")
return attributedPort
}
@@ -190,6 +207,7 @@ async function startBookWithNumbers(startNumber, endNumber, selectedStore, pathT
filterAlreadyBookedContacts(contactList).then((listToBook) => {
filterAlreadyAcceptedContacts(listToBook).then(notAcceptedContacts => {
filterBlacklistedContacts(notAcceptedContacts).then(listWithoutBlackContact => {
console.log("number of contacts to book:" + listWithoutBlackContact.length)
android.devices().then((devices) => {
if (devices.length === 0) {
src/models/OCRResult.js
+2
View File
@@ -3,6 +3,8 @@ const OCRResult = {
TO_REFRESH: Symbol("TO_REFRESH"),
BLOCKED: Symbol("BLOCKED"),
SLIDING_CAPTCHA: Symbol("SLIDING_CAPTCHA"),
FILL_FIELD: Symbol("FILL_FIELD"),
NEED_TO_CLICK_LATE_BTN: Symbol("NEED_TO_CLICK_LATE_BTN"),
RECHECK: Symbol("RECHECK"),
RECAPTCHA_ERROR: Symbol("RECAPTCHA_ERROR"),
}
src/workers/CommandorPage.js
+152 -169
View File
@@ -72,27 +72,28 @@ class CommandorPage {
this.port = port;
this.isFillingFields = false;
this.isTerminated = false;
this.audioAnalyse = audioAnalyse;
this.alertBeep = alertBeep;
this.cguChecked = false;
this.isNameInput = false;
this.isEmailFilled = false;
this.isCountryChoosen = false;
this.isPhoneInput = false;
this.firstStart = false;
}
async connect_to_browser(ocrResult) {
console.log("connect_to_browser() called");
if (!this.browser.isConnected()) {
this.browser = await puppeteer.connect({
browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
headless: false, defaultViewport: null
})
}
console.log("browser.isConnected: " + this.browser.isConnected());
// if (!this.browser.isConnected()) {
this.browser = await puppeteer.connect({
browserWSEndpoint: "ws://127.0.0.1:" + this.port + "/devtools/browser",
headless: false, defaultViewport: null
})
// }
let pages = await this.browser.pages();
// 0 is the last page
this.page = pages[0];
// this.page = pages[pages.length - 1];
switch (ocrResult) {
case OCRResult.SUCCESS:
// get url and push to server
@@ -102,12 +103,23 @@ class CommandorPage {
case OCRResult.TO_REFRESH:
logWithDevice("will reload page", this.device)
await this.page.reload();
await delay(1000);
await this.checkResultWithOcr();
await delay(10000);
if (!await this.checkIfSuccessful()) {
await this.checkResultWithOcr();
}
break;
}
}
async checkIfSuccessful() {
let content = await this.page.content();
console.log(content);
if (content.includes(MESSAGE_URL_VALIDATION_FR) || content.includes(MESSAGE_URL_VALIDATION_EN)) {
log("successful");
await this.push_message_to_queue(PublishType.SUCCESS);
return true;
} else {
return false
}
}
async loadPage() {
@@ -128,60 +140,66 @@ class CommandorPage {
this.onResponse(response)
})
} catch (e) {
await this.checkResultWithOcr()
console.log(e)
this.isTerminated = true
// this.isTerminated = true
}
if (this.page !== undefined) {
try {
console.log("will open google")
const item = searchTexts[Math.floor(Math.random() * searchTexts.length)];
await this.page.goto("https://www.google.com/search?q=" + item + "&lr=lang_en", {timeout: 90 * 1000});
// await this.page.goto(RDV_URL, {timeout: 90 * 1000});
} catch (e) {
log(e)
this.isTerminated = true
}
// try {
// if (this.page.url().includes("google")) {
// this.page.focus('button >> nth=3')
// this.page.click()
// }
// } catch (e) {
// log(e)
// }
try {
const [button] = await this.page.$x("//a[contains(., 'rendezvousparis')]");
console.log("button is " + button)
if (button) {
console.log("will click on the button")
await button.click();
try {
logWithDevice("will open google", this.device)
if (!this.isTerminated) {
const item = searchTexts[Math.floor(Math.random() * searchTexts.length)];
await this.page.goto("https://www.google.com/search?q=" + item + "&lr=lang_en", {timeout: 90 * 1000});
}
// await this.page.goto(RDV_URL, {timeout: 90 * 1000});
} catch (e) {
log(e)
this.isTerminated = true
}
// await this.page.$eval(':nth-match(:text("rendezvousparis"), 1)', el => {
// el.click()
// });
} catch (e) {
log(e)
// if (!this.page.url().includes(RDV_URL)) {
// this.isTerminated = true;
// }
await this.acceptCookies();
try {
const [button] = await this.page.$x("//a[contains(., 'rendezvousparis')]");
console.log("button is " + button)
if (button) {
console.log("will click on the button")
await button.click();
}
if (this.page.url() === RDV_URL) {
if (!this.isFillingFields)
await this.fillFields(this.page, false);
}
} catch (e) {
log(e)
// if (!this.page.url().includes(RDV_URL)) {
// this.isTerminated = true;
// }
}
} else {
this.isTerminated = true;
}
let cancel
const intervalTask = setInterval(() => {
if (this.isTerminated) {
log(this.device.model() + ":request terminated, will close device");
if (this.browser !== undefined)
this.browser.close();
log(this.device.model() + ":request terminated, send cancel()");
// if (this.browser !== undefined)
// try {
// log(this.device.model() + ":trying to disconnect browser");
//
// if (this.browser.isConnected()) {
// this.browser.disconnect();
// }
// } catch (e) {
// console.log(e);
// }
// this.page.close()
// this.device.close()
clearInterval(intervalTask)
cancel()
return this.browser
// return this.browser
} else {
// if (this.page.url() === RDV_URL) {
// if (!this.isFillingFields)
// this.fillFields(this.page);
// }
}
}, 10 * 1000)//interval of 10 seconds
@@ -193,8 +211,22 @@ class CommandorPage {
}).then(log)
}
async acceptCookies() {
await this.page.evaluate(() => {
let buttons = document.getElementsByTagName('button');
for (let i = 0; i < buttons.length; i++) {
let button = buttons[i];
let txt = button.textContent || button.innerText;
if (txt.includes("Tout accepter")) {
button.click()
}
console.log(txt);
}
})
}
async chooseCountry(page) {
if (!page.isClosed()) {
if (this.browser.isConnected()) {
try {
if (!this.isCountryChoosen) {
await page.focus(COUNTRY_ID);
@@ -214,7 +246,7 @@ class CommandorPage {
async fillEmail(page) {
try {
if (!page.isClosed()) {
if (this.browser.isConnected()) {
if (!this.isEmailFilled) {
await page.focus(EMAIL_ID);
await delay(getRandomWaitTime())
@@ -229,7 +261,7 @@ class CommandorPage {
async inputPhoneNumber(page) {
try {
if (!page.isClosed()) {
if (this.browser.isConnected()) {
if (!this.isPhoneInput) {
await page.focus(PHONE_NUMBER);
await page.keyboard.type("" + this.contact.phoneNumber);
@@ -245,7 +277,9 @@ class CommandorPage {
async inputName(page) {
try {
if (!page.isClosed()) {
logWithDevice("input name called with this.browser.isConnected=" + this.browser.isConnected(), this.device)
logWithDevice("input name called with this.isNameInput=" + this.isNameInput, this.device)
if (this.browser.isConnected()) {
if (!this.isNameInput) {
await page.focus(LAST_NAME);
await delay(getRandomWaitTime());
@@ -264,7 +298,7 @@ class CommandorPage {
async inputPassportId(page) {
try {
if (!page.isClosed()) {
if (this.browser.isConnected()) {
if (!this.isPasspordInput) {
await page.focus(PASSPORT_ID);
await delay(getRandomWaitTime());
@@ -281,7 +315,7 @@ class CommandorPage {
async checkCGU(page) {
try {
if (!page.isClosed()) {
if (this.browser.isConnected()) {
if (!this.cguChecked) {
await page.focus(CGU_ID);
await page.evaluate(() => {
@@ -309,7 +343,7 @@ class CommandorPage {
async chooseStore(page) {
try {
if (!page.isClosed()) {
if (this.browser.isConnected()) {
if (this.selectedStore !== "random") {
page.focus(PREFER_STORE);
await delay(1000)
@@ -326,17 +360,19 @@ class CommandorPage {
}
async fillFields(page) {
log("fillFields called for contact:" + this.contact.mail)
await this.enableDisableAirPlanMode()
async fillFields(page, airePlanMode) {
log("fillFields called for contact: " + this.contact.mail)
log("this.isFillingFields: " + this.isFillingFields);
if (!this.isFillingFields) {
if (airePlanMode) {
await this.enableDisableAirPlanMode();
}
this.isFillingFields = true;
await this.chooseStore(page);
await this.inputName(page);
await this.chooseCountry(page);
await this.inputPhoneNumber(page)
await this.fillEmail(page)
// await this.inputPhoneNumber(page)
await this.inputPassportId(page)
await this.checkCGU(page)
await this.resolveCaptcha(page)
@@ -364,9 +400,11 @@ class CommandorPage {
await this.page.evaluate(() => {
document.getElementsByClassName("btn")[0].click();
})
// this.browser.disconnect();
// await this.checkResultWithOcr();
if (this.firstStart) {
logWithDevice("will disconnect browser", this.device);
this.browser.disconnect();
await this.checkResultWithOcr();
}
} catch (e) {
log(e);
}
@@ -387,8 +425,7 @@ class CommandorPage {
let pageContent = await page.content()
let hasCaptcha = pageContent.includes("g-recaptcha-response")
if (hasCaptcha) {
// await this.browser.disconnect()
await this.clickCheckbox()
// await this.clickCheckbox()
await delay(1000)
this.captchaSolver = new SolveCaptcha(page);
await this.captchaSolver.start((solution) => {
@@ -396,12 +433,14 @@ class CommandorPage {
if (solution !== ERROR_CAPTCHA_UNSOLVABLE && solution !== TWO_CAPTCHA_CONNECTION_FAILED) {
try {
if (!page.isClosed()) {
page.evaluate((solution) => {
let element = document.getElementById("g-recaptcha-response");
if (element != null)
document.getElementById("g-recaptcha-response").innerHTML = solution;
}, solution)
this.clickValid();
if (this.browser.isConnected()) {
page.evaluate((solution) => {
let element = document.getElementById("g-recaptcha-response");
if (element != null)
document.getElementById("g-recaptcha-response").innerHTML = solution;
}, solution)
this.clickValid();
}
}
} catch (e) {
log(e)
@@ -416,14 +455,6 @@ class CommandorPage {
}
}
async isBlocked() {
let iframeHandler = await this.page.$("body > iframe");
let captcha_container = await iframeHandler.$(CAPTCHA_CONTAINER)
let html = await captcha_container.innerHTML()
console.log("audio_tag: " + html);
return html.includes("You have been blocked") || html.includes("Vous avez été bloqué") || html.includes("封锁")
}
async onPageLoad(currentPage) {
logWithDevice("onPageLoad called with url " + this.page.url(), this.device)
try {
@@ -445,14 +476,12 @@ class CommandorPage {
logWithDevice("ERR_NETWORK_CHANGED, will reload page", this.device);
await delay(2000)
await this.page.reload()
}
// else if (content.includes("CACHE_MISS")) {
// logWithDevice("CACHE_MISS, will reload page", this.device);
// await delay(2000)
// // await this.setUpCookies()
// await this.page.reload()
// }
else if (content.includes("ERR_TIMED_OUT")) {
} else if (content.includes("CACHE_MISS")) {
logWithDevice("CACHE_MISS, will reload page", this.device);
await delay(2000)
// await this.setUpCookies()
await this.page.reload()
} else if (content.includes("ERR_TIMED_OUT")) {
logWithDevice("ERR_TIMED_OUT, will reload page", this.device);
await delay(2000)
await this.page.reload()
@@ -462,8 +491,7 @@ class CommandorPage {
await this.page.reload()
} else {
if (this.page.url() === RDV_URL) {
await this.fillFields(this.page);
await this.saveCookies()
await this.fillFields(this.page, true);
// if (this.isFillingFields)
// await this.getErrors()
} else {
@@ -484,42 +512,9 @@ class CommandorPage {
}
}
async saveCookies() {
log("saveCookies() called.")
// try {
// let cookies = await this.page.context().cookies();
// let cookiesInJson = [];
// cookies.forEach((cookie) => {
// cookiesInJson.push(JSON.stringify(cookie))
// })
// await require("fs").writeFileSync(this.contact.mail + '.txt', cookiesInJson.join('\n'));
// } catch (e) {
// console.log(e)
// }
}
async setUpCookies() {
// load cookies from file
// try {
// logWithDevice("will add cookies", this.device)
// const fs = require('fs');
// // let cookiesFile = fs.readFileSync('vampuka_fisherleyba@aol.com.txt', 'utf8');
// let cookiesFile = fs.readFileSync(this.contact.mail + '.txt', 'utf8');
// let cookiesList = cookiesFile.split("\n")
// let cookiesListToAdd = []
// cookiesList.forEach((cookie) => {
// let cookieToAdd = JSON.parse(cookie)
// // if (cookieToAdd.name === "datadome") {
// cookiesListToAdd.push(cookieToAdd)
// // }
// })
// await this.context.addCookies(cookiesListToAdd)
// } catch (e) {
// console.log(e)
// }
}
async slidingCaptcha(onResult) {
logWithDevice("slidingCaptcha", this.device);
let slidingCaptchaSolver = new SlidingCaptchaSolver(this.device);
await slidingCaptchaSolver.solve(this.page, async (isSuccessful) => {
console.log("check isAlwaysBlocked")
@@ -527,38 +522,6 @@ class CommandorPage {
})
}
async checkAudioBtn() {
let isBlocked = await this.isBlocked()
if (!isBlocked) {
//try to sliding capthca at first
let slidingCaptchaSolver = new SlidingCaptchaSolver(this.device);
await slidingCaptchaSolver.solve(this.page, async (isSuccessful) => {
console.log("check isAlwaysBlocked")
let isAlwaysBlocked = await this.isBlocked();
if (isAlwaysBlocked) {
await this.resolveByAudio();
}
})
} else {
log("audioBtn not found")
console.log("audioBtn not found")
logWithDevice("we are blocked", this.device)
await this.resetBrowser()
}
}
async resolveByAudio() {
// let audioBtn = await this.page.frameLocator("iframe").locator("#captcha__audio__button");
// log("audioBtn found")
// audioBtn.click()
// let captchaSolver = new GeoCaptchaSolver(this.page, this.device, this.isTerminated)
// await captchaSolver.solve((isSuccessful) => {
// if (!isSuccessful) {
// this.isTerminated = true
// }
// })
}
async onResponse(response) {
let rex = new RegExp(REGEX_RDV_URL)
// log("onResponse with url:" + response.url())
@@ -566,7 +529,6 @@ class CommandorPage {
if (rex.test(response.url())) {
logWithDevice("rdv url found:" + response.url(), this.device)
// save cookies
await this.saveCookies();
await this.push_message_to_db(PublishType.SUCCESS, response.url())
}
// if (response.url().includes("geo.captcha-delivery.com/interstitial")) {
@@ -579,6 +541,7 @@ class CommandorPage {
let url = this.page.url();
logWithDevice("successful url is " + url, this.device)
await this.push_message_to_db(publishType, url)
this.firstStart = false;
}
async push_message_to_db(publishType, url) {
@@ -596,17 +559,13 @@ class CommandorPage {
this.isTerminated = true
}
async saveToBlackList() {
await this.mongoManager.saveBlackListToDb(new BlackListContactPojo(this.contact))
// await this.resetBrowser()
this.isTerminated = true
}
async deleteFromBlackList() {
await this.mongoManager.removeFromBlackList(this.contact)
}
async getErrors() {
logWithDevice("getErrors", this.device);
await this.checkResultWithOcr();
if (this.page.url() === BLANK_URL) {
this.isTerminated = true;
} else {
@@ -623,6 +582,8 @@ class CommandorPage {
}
async clickCheckbox() {
logWithDevice("clickCheckbox", this.device);
try {
// let errorItem = await this.page.click("#recaptcha-anchor > div.recaptcha-checkbox-border")
await this.page.waitForSelector("iframe")
@@ -641,13 +602,20 @@ class CommandorPage {
}
async resetBrowser() {
console.log("will reset browser")
logWithDevice("resetBrowser", this.device);
this.isFillingFields = false;
this.cguChecked = false;
this.isNameInput = false;
this.isEmailFilled = false;
this.isCountryChoosen = false;
this.isPhoneInput = false;
this.browser.disconnect();
await this.device.shell("pm clear com.android.chrome")
await delay(1000)
await this.device.shell("am start -n com.android.chrome/com.google.android.apps.chrome.Main")
await delay(1000)
// await this.skipWelcomePage()
this.isTerminated = true
await delay(5000)
await this.checkResultWithOcr();
}
@@ -656,6 +624,7 @@ class CommandorPage {
await delay(2000);
let ocrChecker = new OCRChecker(this.device, this.contact);
let checkResult = await ocrChecker.get_result();
console.log(checkResult);
while (checkResult === OCRResult.RECHECK) {
await delay(4000)
logWithDevice("will recheck OCR", this.device)
@@ -666,9 +635,12 @@ class CommandorPage {
checkResult = await ocrChecker.get_result();
await this.checkResultWithOcr()
})
await delay(5000)
await delay(20 * 1000)
}
switch (checkResult) {
case OCRResult.FILL_FIELD:
await this.fillFields(this.page)
break;
case OCRResult.SUCCESS:
// reconnect to page and get url
await this.connect_to_browser(OCRResult.SUCCESS);
@@ -682,9 +654,20 @@ class CommandorPage {
case OCRResult.BLOCKED:
await this.resetBrowser();
break;
case OCRResult.NEED_TO_CLICK_LATE_BTN:
await this.tapLaterBtn()
this.firstStart = true;
await this.loadPage()
break;
}
}
async tapLaterBtn() {
logWithDevice("tapLaterBtn", this.device)
this.device.shell("input tap " + 385 + " " + 2050)
await delay(2000);
}
async enableDisableAirPlanMode() {
logWithDevice("will enable aireplan mode", this.device)
await this.device.shell("cmd connectivity airplane-mode enable")
src/workers/OCRChecker.js
+7
View File
@@ -25,6 +25,9 @@ const BLOCKED_MSG_FR = "avez été bloqué"
const CHECKING_MSG_FR = "Verifying"
const ERR_CACHE_MISS = "ERR_CACHE_MISS"
const SLIDING_CAPTCHA_FR = "Glissez vers la droite"
const MESSAGE_FILL_FIELD_FR = "Demande de rendez-vous pour"
const WELCOME_MESSAGE_FR = "Bienvenue dans Chrome"
const PAGE_OPTIMIZATION_CHROME_FR = "Oui, j'accepte"
class OCRChecker {
@@ -55,6 +58,10 @@ class OCRChecker {
return OCRResult.RECHECK
} else if (result.includes(SLIDING_CAPTCHA_FR)) {
return OCRResult.SLIDING_CAPTCHA
} else if (result.includes(MESSAGE_FILL_FIELD_FR)) {
return OCRResult.FILL_FIELD
} else if (result.includes(WELCOME_MESSAGE_FR)) {
return OCRResult.NEED_TO_CLICK_LATE_BTN
}
}