Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Development of PuppeteerPool recycleUserDataDirs option
Jan Curn15c8fe79b48e
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex 9a27620b5..0674b9edb 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -197,6 +197,8 @@ export default class PuppeteerPool {*/_launchInstance() {const id = this.browserCounter++;+ log.debug('PuppeteerPool: launching new instance', { id });+const browserPromise = this.launchPuppeteerFunction();const instance = new PuppeteerInstance(id, browserPromise);@@ -205,7 +207,7 @@ export default class PuppeteerPool {.then((browser) => {browser.on('disconnected', () => {// If instance.killed === true then we killed the instance so don't log it.- if (!instance.killed) log.error('PuppeteerPool: Puppeteer sent "disconnect" event. Crashed???', { id });+ if (!instance.killed) log.error('PuppeteerPool: Puppeteer sent "disconnect" event. Maybe it crashed???', { id });this._retireInstance(instance);});// This one is done manually in Puppeteerpool.newPage() so that it happens immediately.@@ -213,7 +215,13 @@ export default class PuppeteerPool {browser.on('targetdestroyed', () => {instance.activePages--;- if (instance.activePages === 0 && this.retiredInstances[id]) this._killInstance(instance);+ console.log('on targetdestroyed ' + id);++ if (instance.activePages === 0 && this.retiredInstances[id]) {+ // Run this in the next tick, otherwise page.close() for this page might fail+ // with "Protocol error (Target.closeTarget): Target closed."+ setTimeout(() => this._killInstance(instance), 0);+ }});instance.childProcess = browser.process();@@ -360,8 +368,9 @@ export default class PuppeteerPool {.then((page) => {page.once('error', (error) => {log.exception(error, 'PuppeteerPool: page crashed');- // Ignore errors from Page.close()- page.close();+ // Swallow errors from Page.close()+ page.close()+ .catch(err => log.debug('Page.close() failed', { errorMessage: err.message, id: instance.id }));});// TODO: log console messages page.on('console', message => log.debug(`Chrome console: ${message.text}`));diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.jsindex 78b99a9a3..c68ed1ed9 100644--- a/test/puppeteer_pool.js+++ b/test/puppeteer_pool.js@@ -206,14 +206,28 @@ describe('PuppeteerPool', () => {log.setLevel(log.LEVELS.DEBUG);const page1 = await pool.newPage();+ console.log('HERE1-1');await page1.goto('https://www.apify.com');+ console.log('HERE1-2');++ const htmlLength = await page1.evaluate(() => {+ return document.documentElement.innerHTML.length;+ });+ console.log('htmlLength: ' + htmlLength);+await page1.close();+ console.log('HERE1-3');+const page2 = await pool.newPage();++ console.log('HERE2-1');await page2.goto('https://www.apify.com');+ console.log('HERE2-2');await page2.close();+ console.log('HERE2-3');// Cleanup everything.- await pool.destroy();+ // await pool.destroy();});});