Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: implement abortable timeouts (#1245)

Martin Adámek6488b9440e8a
diff --git a/package.json b/package.json
index da9e0fe55..94755dd43 100644
--- a/package.json
+++ b/package.json
@@ -54,9 +54,10 @@
"@apify/log": "^1.1.1",
"@apify/ps-tree": "^1.1.4",
"@apify/storage-local": "^2.0.1",
+ "@apify/timeout": "^0.2.1",
"@apify/utilities": "^1.1.2",
"apify-client": "^2.0.2",
- "browser-pool": "^3.1.0-beta.0",
+ "browser-pool": "^3.1.0-beta.2",
"cheerio": "1.0.0-rc.10",
"content-type": "^1.0.4",
"got-scraping": "^3.2.5",
diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.js
index ac4022d77..142cf0d34 100644
--- a/src/autoscaling/autoscaled_pool.js
+++ b/src/autoscaling/autoscaled_pool.js
@@ -1,9 +1,9 @@
+import { addTimeoutToPromise } from '@apify/timeout';
import { betterSetInterval, betterClearInterval } from '@apify/utilities';
import ow from 'ow';
import Snapshotter, { SnapshotterOptions } from './snapshotter'; // eslint-disable-line import/named,no-unused-vars
import SystemStatus, { SystemStatusOptions } from './system_status'; // eslint-disable-line import/named,no-unused-vars
import defaultLog from '../utils_log';
-import { addTimeoutToPromise } from '../utils';
/**
* @typedef AutoscaledPoolOptions
@@ -440,7 +440,7 @@ class AutoscaledPool {
if (this.taskTimeoutMillis > 0) {
await addTimeoutToPromise(
- this.runTaskFunction(),
+ () => this.runTaskFunction(),
this.taskTimeoutMillis,
`runTaskFunction timed out after ${this.taskTimeoutMillis / 1000} seconds.`,
);
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.js
index 3d55681c6..382a9519b 100644
--- a/src/crawlers/basic_crawler.js
+++ b/src/crawlers/basic_crawler.js
@@ -2,11 +2,11 @@ import { ACTOR_EVENT_NAMES } from '@apify/consts';
import { cryptoRandomObjectId } from '@apify/utilities';
import ow, { ArgumentError } from 'ow';
import _ from 'underscore';
+import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
import AutoscaledPool from '../autoscaling/autoscaled_pool'; // eslint-disable-line import/no-duplicates
import events from '../events';
import { openSessionPool } from '../session_pool/session_pool'; // eslint-disable-line import/no-duplicates
import Statistics from './statistics';
-import { addTimeoutToPromise } from '../utils';
import defaultLog from '../utils_log';
import { validators } from '../validators';
@@ -471,12 +471,12 @@ export class BasicCrawler {
* Wrapper around _runTaskFunction to set a timeout on the entire function.
* Temporary solution to see whether this causes the 0 concurrency bug.
* @ignore
- * @protected
- * @interface
+ * @internal
+ * @private
*/
async timeoutForRunTaskFunction() {
await addTimeoutToPromise(
- this._runTaskFunction(),
+ () => this._runTaskFunction(),
this.handleRequestTimeoutMillis * 2,
`_runTaskFunction timed out after ${(this.handleRequestTimeoutMillis * 2) / 1000} seconds.`,
);
@@ -502,6 +502,8 @@ export class BasicCrawler {
request = await this._fetchNextRequest();
}
+ tryCancel();
+
if (!request) return;
// Reset loadedUrl so an old one is not carried over to retries.
@@ -521,11 +523,13 @@ export class BasicCrawler {
try {
await addTimeoutToPromise(
- this._handleRequestFunction(crawlingContext),
+ () => this._handleRequestFunction(crawlingContext),
this.handleRequestTimeoutMillis,
`handleRequestFunction timed out after ${this.handleRequestTimeoutMillis / 1000} seconds.`,
);
+ tryCancel();
await source.markRequestHandled(request);
+ tryCancel();
this.stats.finishJob(statisticsId);
this.handledRequestsCount++;
@@ -545,6 +549,8 @@ export class BasicCrawler {
} finally {
this.crawlingContexts.delete(crawlingContext.id);
}
+
+ tryCancel();
}
/**
diff --git a/src/crawlers/browser_crawler.js b/src/crawlers/browser_crawler.js
index ec2c09176..65a57bfc6 100644
--- a/src/crawlers/browser_crawler.js
+++ b/src/crawlers/browser_crawler.js
@@ -1,8 +1,8 @@
import ow from 'ow';
+import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
import { BrowserPool, BrowserController } from 'browser-pool'; // eslint-disable-line import/no-duplicates,no-unused-vars
import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS } from '../constants';
import EVENTS from '../session_pool/events'; // eslint-disable-line import/no-duplicates
-import { addTimeoutToPromise } from '../utils';
import { validators } from '../validators';
import {
throwOnBlockedRequest,
@@ -387,32 +387,38 @@ export default class BrowserCrawler extends BasicCrawler {
}
const page = await this.browserPool.newPage(newPageOptions);
+ tryCancel();
this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages);
if (this.useSessionPool) {
const sessionCookies = session.getPuppeteerCookies(request.url);
if (sessionCookies.length) {
await crawlingContext.browserController.setCookies(page, sessionCookies);
+ tryCancel();
}
}
try {
await this._handleNavigation(crawlingContext);
+ tryCancel();
await this._responseHandler(crawlingContext);
+ tryCancel();
// save cookies
// @TODO: Should we save the cookies also after/only the handle page?
if (this.persistCookiesPerSession) {
const cookies = await crawlingContext.browserController.getCookies(page);
+ tryCancel();
session.setPuppeteerCookies(cookies, request.loadedUrl);
}
await addTimeoutToPromise(
- this.handlePageFunction(crawlingContext),
+ () => this.handlePageFunction(crawlingContext),
this.handlePageTimeoutMillis,
`handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`,
);
+ tryCancel();
if (session) session.markGood();
} finally {
@@ -457,6 +463,8 @@ export default class BrowserCrawler extends BasicCrawler {
/** @type {*} */
const gotoOptions = { ...this.defaultGotoOptions };
await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
+ tryCancel();
+
try {
crawlingContext.response = await this._navigationHandler(crawlingContext, gotoOptions);
} catch (error) {
@@ -465,6 +473,7 @@ export default class BrowserCrawler extends BasicCrawler {
throw error;
}
+ tryCancel();
await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
}
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
index 3a16eca15..fb87a1b90 100644
--- a/src/crawlers/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -1,5 +1,6 @@
/* eslint-disable class-methods-use-this */
import { readStreamToString, concatStreamToBuffer } from '@apify/utilities';
+import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
import cheerio from 'cheerio';
import contentTypeParser from 'content-type';
import { DomHandler } from 'htmlparser2';
@@ -9,7 +10,7 @@ import ow from 'ow';
import util from 'util';
import { TimeoutError } from 'got-scraping';
import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS } from '../constants';
-import { addTimeoutToPromise, parseContentTypeFromResponse } from '../utils';
+import { parseContentTypeFromResponse } from '../utils';
import { requestAsBrowser } from '../utils_request';
import { diffCookies, mergeCookies } from './crawler_utils';
import { BasicCrawler } from './basic_crawler'; // eslint-disable-line import/no-duplicates
@@ -543,8 +544,10 @@ class CheerioCrawler extends BasicCrawler {
}
await this._handleNavigation(crawlingContext);
+ tryCancel();
const { dom, isXml, body, contentType, response } = await this._parseResponse(request, crawlingContext.response);
+ tryCancel();
if (this.useSessionPool) {
this._throwOnBlockedRequest(session, response.statusCode);
@@ -589,7 +592,7 @@ class CheerioCrawler extends BasicCrawler {
});
return addTimeoutToPromise(
- this.userProvidedHandler(crawlingContext),
+ () => this.userProvidedHandler(crawlingContext),
this.handlePageTimeoutMillis,
`handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`,
);
@@ -605,6 +608,7 @@ class CheerioCrawler extends BasicCrawler {
if (this.prepareRequestFunction) {
this.log.deprecated('Option "prepareRequestFunction" is deprecated. Use "preNavigationHooks" instead.');
await this.prepareRequestFunction(crawlingContext);
+ tryCancel();
}
const requestAsBrowserOptions = {};
@@ -616,20 +620,24 @@ class CheerioCrawler extends BasicCrawler {
const { request, session } = crawlingContext;
const cookieSnapshot = request.headers.Cookie ?? request.headers.cookie;
await this._executeHooks(this.preNavigationHooks, crawlingContext, requestAsBrowserOptions);
+ tryCancel();
const proxyUrl = crawlingContext.proxyInfo && crawlingContext.proxyInfo.url;
this._mergeRequestCookieDiff(request, cookieSnapshot, requestAsBrowserOptions);
crawlingContext.response = await addTimeoutToPromise(
- this._requestFunction({ request, session, proxyUrl, requestAsBrowserOptions }),
+ () => this._requestFunction({ request, session, proxyUrl, requestAsBrowserOptions }),
this.requestTimeoutMillis,
`request timed out after ${this.requestTimeoutMillis / 1000} seconds.`,
);
+ tryCancel();
await this._executeHooks(this.postNavigationHooks, crawlingContext, requestAsBrowserOptions);
+ tryCancel();
if (this.postResponseFunction) {
this.log.deprecated('Option "postResponseFunction" is deprecated. Use "postNavigationHooks" instead.');
await this.postResponseFunction(crawlingContext);
+ tryCancel();
}
}
diff --git a/src/utils.js b/src/utils.js
index 8c3d0fec8..8e6074c13 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -1,5 +1,4 @@
import psTree from '@apify/ps-tree';
-// eslint-disable-next-line import/no-extraneous-dependencies
import { execSync } from 'child_process';
import { ApifyClient } from 'apify-client';
import { version as apifyClientVersion } from 'apify-client/package.json';
@@ -9,7 +8,6 @@ import contentTypeParser from 'content-type';
import fs from 'fs';
import mime from 'mime-types';
import os from 'os';
-// eslint-disable-next-line import/no-extraneous-dependencies
import ow from 'ow';
import path from 'path';
import semver from 'semver';
@@ -361,29 +359,6 @@ export const getTypicalChromeExecutablePath = () => {
}
};
-/**
- * Wraps the provided Promise with another one that rejects with the given errorMessage
- * after the given timeoutMillis, unless the original promise resolves or rejects earlier.
- *
- * @param {Promise<*>} promise
- * @param {number} timeoutMillis
- * @param {string} errorMessage
- * @return {Promise<*>}
- * @ignore
- */
-export const addTimeoutToPromise = (promise, timeoutMillis, errorMessage) => {
- return new Promise((resolve, reject) => {
- ow(promise, ow.promise);
- ow(timeoutMillis, ow.number);
- ow(errorMessage, ow.string);
- const timeout = setTimeout(() => reject(new Error(errorMessage)), timeoutMillis);
- promise
- .then(resolve)
- .catch(reject)
- .finally(() => clearTimeout(timeout));
- });
-};
-
/**
* Returns `true` when code is running on Apify platform and `false` otherwise (for example locally).
*
diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.js
index 7d2062de7..b631d7472 100644
--- a/test/crawlers/puppeteer_crawler.test.js
+++ b/test/crawlers/puppeteer_crawler.test.js
@@ -221,6 +221,110 @@ describe('PuppeteerCrawler', () => {
expect(loadedUserAgent).toEqual(opts.userAgent);
});
+ test('timeout via preNavigationHooks will abort the page function as early as possible (gh #1216)', async () => {
+ const requestQueue = await Apify.openRequestQueue();
+ await requestQueue.addRequest({ url: 'http://www.example.com' });
+ const handlePageFunction = jest.fn();
+
+ const crawler = new Apify.PuppeteerCrawler({
+ requestQueue,
+ handlePageTimeoutSecs: 0.005,
+ navigationTimeoutSecs: 0.005,
+ preNavigationHooks: [
+ async () => {
+ await Apify.utils.sleep(20);
+ },
+ ],
+ handlePageFunction,
+ });
+
+ const logSpy = jest.spyOn(crawler.log, 'exception');
+ logSpy.mockImplementation(() => {});
+
+ await crawler.run();
+ await crawler.teardown();
+ await requestQueue.drop();
+
+ expect(handlePageFunction).not.toBeCalled();
+ const exceptions = logSpy.mock.calls.map((call) => [call[0].message, call[1], call[2].retryCount]);
+ expect(exceptions).toEqual([
+ [
+ 'Navigation timed out after 0.005 seconds.',
+ 'handleRequestFunction failed, reclaiming failed request back to the list or queue',
+ 1,
+ ],
+ [
+ 'Navigation timed out after 0.005 seconds.',
+ 'handleRequestFunction failed, reclaiming failed request back to the list or queue',
+ 2,
+ ],
+ [
+ 'Navigation timed out after 0.005 seconds.',
+ 'handleRequestFunction failed, reclaiming failed request back to the list or queue',
+ 3,
+ ],
+ [
+ 'Navigation timed out after 0.005 seconds.',
+ 'Request failed and reached maximum retries',
+ undefined,
+ ],
+ ]);
+ logSpy.mockRestore();
+ });
+
+ test('timeout in preLaunchHooks will abort the page function as early as possible (gh #1216)', async () => {
+ const requestQueue = await Apify.openRequestQueue();
+ await requestQueue.addRequest({ url: 'http://www.example.com' });
+ const handlePageFunction = jest.fn();
+
+ const crawler = new Apify.PuppeteerCrawler({
+ requestQueue,
+ navigationTimeoutSecs: 0.005,
+ browserPoolOptions: {
+ preLaunchHooks: [
+ async () => {
+ // Do some async work that's longer than navigationTimeoutSecs
+ await Apify.utils.sleep(20);
+ },
+ ],
+ },
+ handlePageFunction,
+ });
+
+ const logSpy = jest.spyOn(crawler.log, 'exception');
+ logSpy.mockImplementation(() => {});
+
+ await crawler.run();
+ await crawler.teardown();
+ await requestQueue.drop();
+
+ expect(handlePageFunction).not.toBeCalled();
+ const exceptions = logSpy.mock.calls.map((call) => [call[0].message, call[1], call[2].retryCount]);
+ expect(exceptions).toEqual([
+ [
+ 'Navigation timed out after 0.005 seconds.',
+ 'handleRequestFunction failed, reclaiming failed request back to the list or queue',
+ 1,
+ ],
+ [
+ 'Navigation timed out after 0.005 seconds.',
+ 'handleRequestFunction failed, reclaiming failed request back to the list or queue',
+ 2,
+ ],
+ [
+ 'Navigation timed out after 0.005 seconds.',
+ 'handleRequestFunction failed, reclaiming failed request back to the list or queue',
+ 3,
+ ],
+ [
+ 'Navigation timed out after 0.005 seconds.',
+ 'Request failed and reached maximum retries',
+ undefined,
+ ],
+ ]);
+ logSpy.mockRestore();
+ });
+
test('should set cookies assigned to session to page', async () => {
const cookies = [
{
diff --git a/test/utils.test.js b/test/utils.test.js
index 466da70da..0ddded45a 100644
--- a/test/utils.test.js
+++ b/test/utils.test.js
@@ -7,6 +7,7 @@ import os from 'os';
import cheerio from 'cheerio';
import semver from 'semver';
import { ENV_VARS } from '@apify/consts';
+import { addTimeoutToPromise } from '@apify/timeout';
import Apify from '../build/index';
import * as utils from '../build/utils';
import log from '../build/utils_log';
@@ -812,8 +813,8 @@ describe('utils.addTimeoutToPromise()', () => {
test('should timeout', async () => {
const clock = sinon.useFakeTimers();
try {
- const p = utils.addTimeoutToPromise(
- new Promise((r) => setTimeout(r, 500)),
+ const p = addTimeoutToPromise(
+ () => new Promise((r) => setTimeout(r, 500)),
100,
'Timed out.',
);
@@ -830,8 +831,8 @@ describe('utils.addTimeoutToPromise()', () => {
test('should not timeout too soon', async () => {
const clock = sinon.useFakeTimers();
try {
- const p = utils.addTimeoutToPromise(
- new Promise((r) => setTimeout(() => r('Done'), 100)),
+ const p = addTimeoutToPromise(
+ () => new Promise((r) => setTimeout(() => r('Done'), 100)),
500,
'Timed out.',
);