Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fixing Puppeteer pool - errors of page.goto() not catched
Marek Trunkate2f6ac03f8cc
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsindex 98a0d1096..7c63d7a20 100644--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@ -245,17 +245,19 @@ export default class PuppeteerCrawler {if (!this.isRunning) throw new Error('PuppeteerCrawler is stopped.'); // Pool will be destroyed.const page = await this.puppeteerPool.newPage();- const response = await this.gotoFunction({ page, request, puppeteerPool: this.puppeteerPool });-- const pageHandledOrTimedOutPromise = Promise.race([- this.handlePageFunction({ page, request, puppeteerPool: this.puppeteerPool, response }),- createTimeoutPromise(this.handlePageTimeoutSecs * 1000, 'PuppeteerCrawler: handlePageFunction timed out.'),- ]);try {+ const gotoPromise = this.gotoFunction({ page, request, puppeteerPool: this.puppeteerPool });+ const pageOperationsPromise = gotoPromise.then((response) => {+ return Promise.race([+ this.handlePageFunction({ page, request, puppeteerPool: this.puppeteerPool, response }),+ createTimeoutPromise(this.handlePageTimeoutSecs * 1000, 'PuppeteerCrawler: handlePageFunction timed out.'),+ ]);+ });+// rejectOnStopPromise rejects when .stop() is called or BasicCrawler throws.// All running pages are therefore terminated with an error to be reclaimed and retried.- return await Promise.race([pageHandledOrTimedOutPromise, this.rejectOnStopPromise]);+ return await Promise.race([pageOperationsPromise, this.rejectOnStopPromise]);} finally {try {await Promise.race([page.close(), createTimeoutPromise(PAGE_CLOSE_TIMEOUT_MILLIS, 'Operation timed out.')]);