Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix repeating requests on failed page.close()
Ondra Urban4210cb8bae38
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsindex 7587a9c12..4acc4f037 100644--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@ -4,7 +4,7 @@ import _ from 'underscore';import Promise from 'bluebird';import BasicCrawler from './basic_crawler';import PuppeteerPool from './puppeteer_pool';-import { isPromise, createTimeoutPromise } from './utils';+import { isPromise } from './utils';const DEFAULT_OPTIONS = {gotoFunction: ({ request, page }) => page.goto(request.url),@@ -230,19 +230,13 @@ export default class PuppeteerCrawler {return promise;});- return Promise- .race([- handlePagePromise,- createTimeoutPromise(this.pageOpsTimeoutMillis, 'PuppeteerCrawler: handlePageFunction timed out.'),- ])+ return handlePagePromise+ .timeout(this.pageOpsTimeoutMillis, 'PuppeteerCrawler: handlePageFunction timed out.').finally(() => {- if (!page) return;-return Promise- .race([- page.close(),- createTimeoutPromise(this.pageCloseTimeoutMillis, 'PuppeteerCrawler: page.close() timed out.'),- ]);+ .try(() => page.close())+ .timeout(this.pageCloseTimeoutMillis, 'PuppeteerCrawler: page.close() timed out.')+ .catch(err => log.debug('PuppeteerCrawler: page.close() failed.', err));});}}diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.jsindex 524197470..3357c8ca5 100644--- a/test/puppeteer_crawler.js+++ b/test/puppeteer_crawler.js@@ -1,5 +1,6 @@import chai, { expect } from 'chai';import chaiAsPromised from 'chai-as-promised';+import log from 'apify-shared/log';import 'babel-polyfill';import { ENV_VARS } from '../build/constants';import * as Apify from '../build/index';@@ -58,13 +59,22 @@ describe('PuppeteerCrawler', () => {});});- it('should fail when pageCloseTimeoutMillis gets exceeded', async () => {+ it('should only log when pageCloseTimeoutMillis gets exceeded', async () => {const sources = [{ url: 'http://example.com/?q=1' },];const failed = [];const errors = [];- const expectErrorMsgString = 'timed out';+ const expectErrorMsgString = 'failed';+ const expectErrorDataString = 'timed out';++ const logger = (message, data) => {+ errors.push({ message, data });+ };++ const originalLogger = log.debug;+ log.debug = logger;+const requestList = new Apify.RequestList({ sources });const handlePageFunction = ({ page }) => {page.close = () => {@@ -77,11 +87,8 @@ describe('PuppeteerCrawler', () => {const puppeteerCrawler = new Apify.PuppeteerCrawler({requestList,handlePageFunction,- handleFailedRequestFunction: ({ request, error }) => {- console.log(errors);-+ handleFailedRequestFunction: ({ request }) => {failed.push(request);- errors.push(error);},pageCloseTimeoutMillis: 1000,});@@ -89,16 +96,56 @@ describe('PuppeteerCrawler', () => {await requestList.initialize();await puppeteerCrawler.run();- expect(failed).to.have.lengthOf(1);- expect(failed[0].retryCount).to.be.eql(3);- expect(failed[0].errorMessages).to.have.lengthOf(4);- failed[0].errorMessages.forEach((error) => {- expect(error).to.be.a('string');- expect(error).to.include(expectErrorMsgString);- });- errors.forEach((error) => {- expect(error).to.be.an('error');- expect(error.message).to.include(expectErrorMsgString);+ expect(failed).to.have.lengthOf(0);+ expect(errors.length).to.be.greaterThan(0);+ expect(errors[0].message).to.include(expectErrorMsgString);+ expect(errors[0].data.message).to.include(expectErrorDataString);++ log.debug = originalLogger;+ });++ it('should only log when page.close() rejects', async () => {+ const sources = [+ { url: 'http://example.com/?q=1' },+ ];+ const failed = [];+ const errors = [];+ const expectErrorMsgString = 'failed';+ const expectErrorDataString = 'my_rejection';++ const logger = (message, data) => {+ errors.push({ message, data });+ };++ const originalLogger = log.debug;+ log.debug = logger;++ const requestList = new Apify.RequestList({ sources });+ const handlePageFunction = ({ page }) => {+ page.close = () => {+ return Promise.reject(new Error('my_rejection'));+ };++ return Promise.resolve();+ };++ const puppeteerCrawler = new Apify.PuppeteerCrawler({+ requestList,+ handlePageFunction,+ handleFailedRequestFunction: ({ request }) => {+ failed.push(request);+ },+ pageCloseTimeoutMillis: 1000,});++ await requestList.initialize();+ await puppeteerCrawler.run();++ expect(failed).to.have.lengthOf(0);+ expect(errors.length).to.be.greaterThan(0);+ expect(errors[0].message).to.include(expectErrorMsgString);+ expect(errors[0].data.message).to.include(expectErrorDataString);++ log.debug = originalLogger;});});