Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Adding page.close() timeout to PuppeteerCrawler (#107)

Marek Trunkát4e3014699f18
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index 88c13bf49..712910fdc 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -2,13 +2,15 @@ import { checkParamOrThrow } from 'apify-client/build/utils';
import log from 'apify-shared/log';
import _ from 'underscore';
import Promise from 'bluebird';
+import { delayPromise } from 'apify-shared/utilities';
import BasicCrawler from './basic_crawler';
import PuppeteerPool from './puppeteer_pool';
-import { isPromise } from './utils';
+import { isPromise, createTimeoutPromise } from './utils';
const DEFAULT_OPTIONS = {
gotoFunction: ({ request, page }) => page.goto(request.url),
pageOpsTimeoutMillis: 300000,
+ pageCloseTimeoutMillis: 30000,
};
/**
@@ -112,6 +114,7 @@ const DEFAULT_OPTIONS = {
* @param {LaunchPuppeteerOptions} [options.launchPuppeteerOptions]
* Options used by `Apify.launchPuppeteer()` to start new Puppeteer instances.
* See `launchPuppeteerOptions` parameter of `PuppeteerPool`.
+ * @param {Number} [options.pageCloseTimeoutMillis=30000] Timeout for `page.close()` in milliseconds.
*/
export default class PuppeteerCrawler {
constructor(opts) {
@@ -125,6 +128,7 @@ export default class PuppeteerCrawler {
handlePageFunction,
gotoFunction,
pageOpsTimeoutMillis,
+ pageCloseTimeoutMillis,
// Autoscaled pool options
maxMemoryMbytes,
@@ -151,11 +155,12 @@ export default class PuppeteerCrawler {
checkParamOrThrow(handlePageFunction, 'opts.handlePageFunction', 'Function');
checkParamOrThrow(handleFailedRequestFunction, 'opts.handleFailedRequestFunction', 'Maybe Function');
checkParamOrThrow(gotoFunction, 'opts.gotoFunction', 'Function');
- checkParamOrThrow(pageOpsTimeoutMillis, 'opts.pageOpsTimeoutMillis', 'Number');
+ checkParamOrThrow(pageCloseTimeoutMillis, 'opts.pageCloseTimeoutMillis', 'Number');
this.handlePageFunction = handlePageFunction;
this.gotoFunction = gotoFunction;
this.pageOpsTimeoutMillis = pageOpsTimeoutMillis;
+ this.pageCloseTimeoutMillis = pageCloseTimeoutMillis;
this.puppeteerPool = new PuppeteerPool({
maxOpenPagesPerInstance,
@@ -219,15 +224,19 @@ export default class PuppeteerCrawler {
return promise;
});
- const timeoutPromise = new Promise(resolve => setTimeout(resolve, this.pageOpsTimeoutMillis));
-
return Promise
.race([
handlePagePromise,
- timeoutPromise.then(() => { throw new Error('PuppeteerCrawler: handlePageFunction timed out'); }),
+ createTimeoutPromise(this.pageOpsTimeoutMillis, 'PuppeteerCrawler: handlePageFunction timed out.'),
])
.finally(() => {
- if (page) return page.close();
+ if (!page) return;
+
+ return Promise
+ .race([
+ page.close(),
+ createTimeoutPromise(this.pageCloseTimeoutMillis, 'PuppeteerCrawler: page.close() timed out.'),
+ ]);
});
}
}
diff --git a/src/utils.js b/src/utils.js
index 4c42f9786..54cd72f28 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -269,7 +269,7 @@ export const getFirstKey = (dict) => {
/**
* Gets a typical path to Chrome executable, depending on the current operating system.
*
- * @returns {string}
+ * @return {string}
* @ignore
*/
export const getTypicalChromeExecutablePath = () => {
@@ -280,6 +280,18 @@ export const getTypicalChromeExecutablePath = () => {
}
};
+/**
+ * Creates a promise that after given time gets rejected with given error.
+ *
+ * @return {Promise<Error>}
+ * @ignore.
+ */
+export const createTimeoutPromise = (timeoutMillis, errorMessage) => {
+ return delayPromise(timeoutMillis).then(() => {
+ throw new Error(errorMessage);
+ });
+};
+
/**
* Returns `true` when code is running on Apify platform and `false` otherwise (for example locally).
*
@@ -292,7 +304,6 @@ export const getTypicalChromeExecutablePath = () => {
*/
export const isAtHome = () => !!process.env[ENV_VARS.IS_AT_HOME];
-
/**
* Returns a promise that resolves after a specific period of time. This is useful to implement waiting
* in your code, e.g. to prevent overloading of target website or to avoid bot detection.
diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.js
index 3f53986ca..6548c5f3a 100644
--- a/test/puppeteer_crawler.js
+++ b/test/puppeteer_crawler.js
@@ -59,7 +59,7 @@ describe('PuppeteerCrawler', () => {
});
});
- it('should fail when pageOpsTimeoutMillis gets exceeded', async () => {
+ it('should fail when pageCloseTimeoutMillis gets exceeded', async () => {
const sources = [
{ url: 'http://example.com/?q=1' },
];
@@ -67,18 +67,24 @@ describe('PuppeteerCrawler', () => {
const errors = [];
const expectErrorMsgString = 'timed out';
const requestList = new Apify.RequestList({ sources });
- const handlePageFunction = async () => {
- await delayPromise(1000);
+ const handlePageFunction = ({ page }) => {
+ page.close = () => {
+ return new Promise(() => {}); // This will never resolve.
+ };
+
+ return Promise.resolve();
};
const puppeteerCrawler = new Apify.PuppeteerCrawler({
requestList,
handlePageFunction,
handleFailedRequestFunction: ({ request, error }) => {
+ console.log(errors);
+
failed.push(request);
errors.push(error);
},
- pageOpsTimeoutMillis: 900,
+ pageCloseTimeoutMillis: 1000,
});
await requestList.initialize();