Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Removed "pageCloseTimeoutMillis" from PuppeteerCrawler
Jan Curnbc1b1b36135f
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsindex 4acc4f037..4d5a7a0ba 100644--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@ -9,9 +9,10 @@ import { isPromise } from './utils';const DEFAULT_OPTIONS = {gotoFunction: ({ request, page }) => page.goto(request.url),pageOpsTimeoutMillis: 300000,- pageCloseTimeoutMillis: 30000,};+const PAGE_CLOSE_TIMEOUT_MILLIS = 30000;+/*** Provides a simple framework for parallel crawling of web pages* using headless Chrome with [Puppeteer](https://github.com/GoogleChrome/puppeteer).@@ -118,7 +119,6 @@ const DEFAULT_OPTIONS = {* @param {LaunchPuppeteerOptions} [options.launchPuppeteerOptions]* Options used by `Apify.launchPuppeteer()` to start new Puppeteer instances.* See `launchPuppeteerOptions` parameter of `PuppeteerPool`.- * @param {Number} [options.pageCloseTimeoutMillis=30000] Timeout for `page.close()` in milliseconds.*/export default class PuppeteerCrawler {constructor(opts) {@@ -132,7 +132,6 @@ export default class PuppeteerCrawler {handlePageFunction,gotoFunction,pageOpsTimeoutMillis,- pageCloseTimeoutMillis,// Autoscaled pool optionsmaxMemoryMbytes,@@ -160,12 +159,10 @@ export default class PuppeteerCrawler {checkParamOrThrow(handlePageFunction, 'opts.handlePageFunction', 'Function');checkParamOrThrow(handleFailedRequestFunction, 'opts.handleFailedRequestFunction', 'Maybe Function');checkParamOrThrow(gotoFunction, 'opts.gotoFunction', 'Function');- checkParamOrThrow(pageCloseTimeoutMillis, 'opts.pageCloseTimeoutMillis', 'Number');this.handlePageFunction = handlePageFunction;this.gotoFunction = gotoFunction;this.pageOpsTimeoutMillis = pageOpsTimeoutMillis;- this.pageCloseTimeoutMillis = pageCloseTimeoutMillis;this.puppeteerPool = new PuppeteerPool({maxOpenPagesPerInstance,@@ -235,8 +232,8 @@ export default class PuppeteerCrawler {.finally(() => {return Promise.try(() => page.close())- .timeout(this.pageCloseTimeoutMillis, 'PuppeteerCrawler: page.close() timed out.')- .catch(err => log.debug('PuppeteerCrawler: page.close() failed.', err));+ .timeout(PAGE_CLOSE_TIMEOUT_MILLIS, 'Operation timed out.')+ .catch(err => log.debug('PuppeteerCrawler: Page.close() failed.', { reason: err ? err.message : err }));});}}diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.jsindex 3357c8ca5..8dc0787a7 100644--- a/test/puppeteer_crawler.js+++ b/test/puppeteer_crawler.js@@ -59,51 +59,6 @@ describe('PuppeteerCrawler', () => {});});- it('should only log when pageCloseTimeoutMillis gets exceeded', async () => {- const sources = [- { url: 'http://example.com/?q=1' },- ];- const failed = [];- const errors = [];- const expectErrorMsgString = 'failed';- const expectErrorDataString = 'timed out';-- const logger = (message, data) => {- errors.push({ message, data });- };-- const originalLogger = log.debug;- log.debug = logger;-- const requestList = new Apify.RequestList({ sources });- const handlePageFunction = ({ page }) => {- page.close = () => {- return new Promise(() => {}); // This will never resolve.- };-- return Promise.resolve();- };-- const puppeteerCrawler = new Apify.PuppeteerCrawler({- requestList,- handlePageFunction,- handleFailedRequestFunction: ({ request }) => {- failed.push(request);- },- pageCloseTimeoutMillis: 1000,- });-- await requestList.initialize();- await puppeteerCrawler.run();-- expect(failed).to.have.lengthOf(0);- expect(errors.length).to.be.greaterThan(0);- expect(errors[0].message).to.include(expectErrorMsgString);- expect(errors[0].data.message).to.include(expectErrorDataString);-- log.debug = originalLogger;- });-it('should only log when page.close() rejects', async () => {const sources = [{ url: 'http://example.com/?q=1' },