'After opening multiple pages with Puppeteer I get blank pages
I apologize if I duplicated the topic, but everything I've tried so far doesn't help. I am trying to scraping some details data from each links from ads. After opened few links, I get each next time blank page without any data. Also, if I stop script and start it again I can't do it(because I received blank page on the start) and need to wait few minutes. This is code and can you please said me where I made mistake.
Thank you :)
const puppeteer = require("puppeteer-extra");
(async () => {
const browser = await puppeteer.launch({
headless: false,
args: [
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-infobars",
"--window-position=0,0",
"--ignore-certifcate-errors",
"--ignore-certifcate-errors-spki-list",
'--user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3312.0 Safari/537.36"',
],
});
const page = await browser.newPage();
await page.goto(
"https://www.ebay-kleinanzeigen.de/s-immobilien/01309/c195l23973"
);
const banner = (await page.$("#gdpr-banner-accept")) !== null;
if (banner) {
await Promise.all([page.click("#gdpr-banner-accept")]);
}
let isBtnDisabled = false;
while (!isBtnDisabled) {
const productsHandles = await page.$$(".ad-listitem.lazyload-item");
for (const producthandle of productsHandles) {
try {
link = await page.evaluate(
(el) => el.querySelector("a.ellipsis").href,
producthandle
);
const eachPage = await browser.newPage();
await eachPage.goto(link, {
waitUntil: "networkidle0",
});
await eachPage.waitForSelector("#viewad-price", {
visible: true,
});
const price = await eachPage.$eval(
"#viewad-price",
(price) => price.textContent
);
console.log(price);
eachPage.close();
} catch (error) {
console.log(error);
}
}
await page.waitForSelector("#srchrslt-pagination > div", { visible: true });
const is_disabled =
(await page.$(
"#srchrslt-pagination > div > div.pagination-nav > .pagination-next"
)) === null;
isBtnDisabled = is_disabled;
if (!is_disabled) {
await Promise.all([
page.click(".pagination-nav > .pagination-next"),
page.waitForNavigation({ waitUntil: "networkidle2" }),
]);
}
}
await browser.close();
})();
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|
