Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fix repeating requests on failed page.close()

Ondra Urban4210cb8bae38
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index 7587a9c12..4acc4f037 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -4,7 +4,7 @@ import _ from 'underscore';
import Promise from 'bluebird';
import BasicCrawler from './basic_crawler';
import PuppeteerPool from './puppeteer_pool';
-import { isPromise, createTimeoutPromise } from './utils';
+import { isPromise } from './utils';
const DEFAULT_OPTIONS = {
gotoFunction: ({ request, page }) => page.goto(request.url),
@@ -230,19 +230,13 @@ export default class PuppeteerCrawler {
return promise;
});
- return Promise
- .race([
- handlePagePromise,
- createTimeoutPromise(this.pageOpsTimeoutMillis, 'PuppeteerCrawler: handlePageFunction timed out.'),
- ])
+ return handlePagePromise
+ .timeout(this.pageOpsTimeoutMillis, 'PuppeteerCrawler: handlePageFunction timed out.')
.finally(() => {
- if (!page) return;
-
return Promise
- .race([
- page.close(),
- createTimeoutPromise(this.pageCloseTimeoutMillis, 'PuppeteerCrawler: page.close() timed out.'),
- ]);
+ .try(() => page.close())
+ .timeout(this.pageCloseTimeoutMillis, 'PuppeteerCrawler: page.close() timed out.')
+ .catch(err => log.debug('PuppeteerCrawler: page.close() failed.', err));
});
}
}
diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.js
index 524197470..3357c8ca5 100644
--- a/test/puppeteer_crawler.js
+++ b/test/puppeteer_crawler.js
@@ -1,5 +1,6 @@
import chai, { expect } from 'chai';
import chaiAsPromised from 'chai-as-promised';
+import log from 'apify-shared/log';
import 'babel-polyfill';
import { ENV_VARS } from '../build/constants';
import * as Apify from '../build/index';
@@ -58,13 +59,22 @@ describe('PuppeteerCrawler', () => {
});
});
- it('should fail when pageCloseTimeoutMillis gets exceeded', async () => {
+ it('should only log when pageCloseTimeoutMillis gets exceeded', async () => {
const sources = [
{ url: 'http://example.com/?q=1' },
];
const failed = [];
const errors = [];
- const expectErrorMsgString = 'timed out';
+ const expectErrorMsgString = 'failed';
+ const expectErrorDataString = 'timed out';
+
+ const logger = (message, data) => {
+ errors.push({ message, data });
+ };
+
+ const originalLogger = log.debug;
+ log.debug = logger;
+
const requestList = new Apify.RequestList({ sources });
const handlePageFunction = ({ page }) => {
page.close = () => {
@@ -77,11 +87,8 @@ describe('PuppeteerCrawler', () => {
const puppeteerCrawler = new Apify.PuppeteerCrawler({
requestList,
handlePageFunction,
- handleFailedRequestFunction: ({ request, error }) => {
- console.log(errors);
-
+ handleFailedRequestFunction: ({ request }) => {
failed.push(request);
- errors.push(error);
},
pageCloseTimeoutMillis: 1000,
});
@@ -89,16 +96,56 @@ describe('PuppeteerCrawler', () => {
await requestList.initialize();
await puppeteerCrawler.run();
- expect(failed).to.have.lengthOf(1);
- expect(failed[0].retryCount).to.be.eql(3);
- expect(failed[0].errorMessages).to.have.lengthOf(4);
- failed[0].errorMessages.forEach((error) => {
- expect(error).to.be.a('string');
- expect(error).to.include(expectErrorMsgString);
- });
- errors.forEach((error) => {
- expect(error).to.be.an('error');
- expect(error.message).to.include(expectErrorMsgString);
+ expect(failed).to.have.lengthOf(0);
+ expect(errors.length).to.be.greaterThan(0);
+ expect(errors[0].message).to.include(expectErrorMsgString);
+ expect(errors[0].data.message).to.include(expectErrorDataString);
+
+ log.debug = originalLogger;
+ });
+
+ it('should only log when page.close() rejects', async () => {
+ const sources = [
+ { url: 'http://example.com/?q=1' },
+ ];
+ const failed = [];
+ const errors = [];
+ const expectErrorMsgString = 'failed';
+ const expectErrorDataString = 'my_rejection';
+
+ const logger = (message, data) => {
+ errors.push({ message, data });
+ };
+
+ const originalLogger = log.debug;
+ log.debug = logger;
+
+ const requestList = new Apify.RequestList({ sources });
+ const handlePageFunction = ({ page }) => {
+ page.close = () => {
+ return Promise.reject(new Error('my_rejection'));
+ };
+
+ return Promise.resolve();
+ };
+
+ const puppeteerCrawler = new Apify.PuppeteerCrawler({
+ requestList,
+ handlePageFunction,
+ handleFailedRequestFunction: ({ request }) => {
+ failed.push(request);
+ },
+ pageCloseTimeoutMillis: 1000,
});
+
+ await requestList.initialize();
+ await puppeteerCrawler.run();
+
+ expect(failed).to.have.lengthOf(0);
+ expect(errors.length).to.be.greaterThan(0);
+ expect(errors[0].message).to.include(expectErrorMsgString);
+ expect(errors[0].data.message).to.include(expectErrorDataString);
+
+ log.debug = originalLogger;
});
});