Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: implement abortable timeouts (#1245)
Martin Adámek6488b9440e8a
diff --git a/package.json b/package.jsonindex da9e0fe55..94755dd43 100644--- a/package.json+++ b/package.json@@ -54,9 +54,10 @@"@apify/log": "^1.1.1","@apify/ps-tree": "^1.1.4","@apify/storage-local": "^2.0.1",+ "@apify/timeout": "^0.2.1","@apify/utilities": "^1.1.2","apify-client": "^2.0.2",- "browser-pool": "^3.1.0-beta.0",+ "browser-pool": "^3.1.0-beta.2","cheerio": "1.0.0-rc.10","content-type": "^1.0.4","got-scraping": "^3.2.5",diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.jsindex ac4022d77..142cf0d34 100644--- a/src/autoscaling/autoscaled_pool.js+++ b/src/autoscaling/autoscaled_pool.js@@ -1,9 +1,9 @@+import { addTimeoutToPromise } from '@apify/timeout';import { betterSetInterval, betterClearInterval } from '@apify/utilities';import ow from 'ow';import Snapshotter, { SnapshotterOptions } from './snapshotter'; // eslint-disable-line import/named,no-unused-varsimport SystemStatus, { SystemStatusOptions } from './system_status'; // eslint-disable-line import/named,no-unused-varsimport defaultLog from '../utils_log';-import { addTimeoutToPromise } from '../utils';/*** @typedef AutoscaledPoolOptions@@ -440,7 +440,7 @@ class AutoscaledPool {if (this.taskTimeoutMillis > 0) {await addTimeoutToPromise(- this.runTaskFunction(),+ () => this.runTaskFunction(),this.taskTimeoutMillis,`runTaskFunction timed out after ${this.taskTimeoutMillis / 1000} seconds.`,);diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex 3d55681c6..382a9519b 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -2,11 +2,11 @@ import { ACTOR_EVENT_NAMES } from '@apify/consts';import { cryptoRandomObjectId } from '@apify/utilities';import ow, { ArgumentError } from 'ow';import _ from 'underscore';+import { addTimeoutToPromise, tryCancel } from '@apify/timeout';import AutoscaledPool from '../autoscaling/autoscaled_pool'; // eslint-disable-line import/no-duplicatesimport events from '../events';import { openSessionPool } from '../session_pool/session_pool'; // eslint-disable-line import/no-duplicatesimport Statistics from './statistics';-import { addTimeoutToPromise } from '../utils';import defaultLog from '../utils_log';import { validators } from '../validators';@@ -471,12 +471,12 @@ export class BasicCrawler {* Wrapper around _runTaskFunction to set a timeout on the entire function.* Temporary solution to see whether this causes the 0 concurrency bug.* @ignore- * @protected- * @interface+ * @internal+ * @private*/async timeoutForRunTaskFunction() {await addTimeoutToPromise(- this._runTaskFunction(),+ () => this._runTaskFunction(),this.handleRequestTimeoutMillis * 2,`_runTaskFunction timed out after ${(this.handleRequestTimeoutMillis * 2) / 1000} seconds.`,);@@ -502,6 +502,8 @@ export class BasicCrawler {request = await this._fetchNextRequest();}+ tryCancel();+if (!request) return;// Reset loadedUrl so an old one is not carried over to retries.@@ -521,11 +523,13 @@ export class BasicCrawler {try {await addTimeoutToPromise(- this._handleRequestFunction(crawlingContext),+ () => this._handleRequestFunction(crawlingContext),this.handleRequestTimeoutMillis,`handleRequestFunction timed out after ${this.handleRequestTimeoutMillis / 1000} seconds.`,);+ tryCancel();await source.markRequestHandled(request);+ tryCancel();this.stats.finishJob(statisticsId);this.handledRequestsCount++;@@ -545,6 +549,8 @@ export class BasicCrawler {} finally {this.crawlingContexts.delete(crawlingContext.id);}++ tryCancel();}/**diff --git a/src/crawlers/browser_crawler.js b/src/crawlers/browser_crawler.jsindex ec2c09176..65a57bfc6 100644--- a/src/crawlers/browser_crawler.js+++ b/src/crawlers/browser_crawler.js@@ -1,8 +1,8 @@import ow from 'ow';+import { addTimeoutToPromise, tryCancel } from '@apify/timeout';import { BrowserPool, BrowserController } from 'browser-pool'; // eslint-disable-line import/no-duplicates,no-unused-varsimport { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS } from '../constants';import EVENTS from '../session_pool/events'; // eslint-disable-line import/no-duplicates-import { addTimeoutToPromise } from '../utils';import { validators } from '../validators';import {throwOnBlockedRequest,@@ -387,32 +387,38 @@ export default class BrowserCrawler extends BasicCrawler {}const page = await this.browserPool.newPage(newPageOptions);+ tryCancel();this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages);if (this.useSessionPool) {const sessionCookies = session.getPuppeteerCookies(request.url);if (sessionCookies.length) {await crawlingContext.browserController.setCookies(page, sessionCookies);+ tryCancel();}}try {await this._handleNavigation(crawlingContext);+ tryCancel();await this._responseHandler(crawlingContext);+ tryCancel();// save cookies// @TODO: Should we save the cookies also after/only the handle page?if (this.persistCookiesPerSession) {const cookies = await crawlingContext.browserController.getCookies(page);+ tryCancel();session.setPuppeteerCookies(cookies, request.loadedUrl);}await addTimeoutToPromise(- this.handlePageFunction(crawlingContext),+ () => this.handlePageFunction(crawlingContext),this.handlePageTimeoutMillis,`handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`,);+ tryCancel();if (session) session.markGood();} finally {@@ -457,6 +463,8 @@ export default class BrowserCrawler extends BasicCrawler {/** @type {*} */const gotoOptions = { ...this.defaultGotoOptions };await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);+ tryCancel();+try {crawlingContext.response = await this._navigationHandler(crawlingContext, gotoOptions);} catch (error) {@@ -465,6 +473,7 @@ export default class BrowserCrawler extends BasicCrawler {throw error;}+ tryCancel();await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);}diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 3a16eca15..fb87a1b90 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -1,5 +1,6 @@/* eslint-disable class-methods-use-this */import { readStreamToString, concatStreamToBuffer } from '@apify/utilities';+import { addTimeoutToPromise, tryCancel } from '@apify/timeout';import cheerio from 'cheerio';import contentTypeParser from 'content-type';import { DomHandler } from 'htmlparser2';@@ -9,7 +10,7 @@ import ow from 'ow';import util from 'util';import { TimeoutError } from 'got-scraping';import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS } from '../constants';-import { addTimeoutToPromise, parseContentTypeFromResponse } from '../utils';+import { parseContentTypeFromResponse } from '../utils';import { requestAsBrowser } from '../utils_request';import { diffCookies, mergeCookies } from './crawler_utils';import { BasicCrawler } from './basic_crawler'; // eslint-disable-line import/no-duplicates@@ -543,8 +544,10 @@ class CheerioCrawler extends BasicCrawler {}await this._handleNavigation(crawlingContext);+ tryCancel();const { dom, isXml, body, contentType, response } = await this._parseResponse(request, crawlingContext.response);+ tryCancel();if (this.useSessionPool) {this._throwOnBlockedRequest(session, response.statusCode);@@ -589,7 +592,7 @@ class CheerioCrawler extends BasicCrawler {});return addTimeoutToPromise(- this.userProvidedHandler(crawlingContext),+ () => this.userProvidedHandler(crawlingContext),this.handlePageTimeoutMillis,`handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`,);@@ -605,6 +608,7 @@ class CheerioCrawler extends BasicCrawler {if (this.prepareRequestFunction) {this.log.deprecated('Option "prepareRequestFunction" is deprecated. Use "preNavigationHooks" instead.');await this.prepareRequestFunction(crawlingContext);+ tryCancel();}const requestAsBrowserOptions = {};@@ -616,20 +620,24 @@ class CheerioCrawler extends BasicCrawler {const { request, session } = crawlingContext;const cookieSnapshot = request.headers.Cookie ?? request.headers.cookie;await this._executeHooks(this.preNavigationHooks, crawlingContext, requestAsBrowserOptions);+ tryCancel();const proxyUrl = crawlingContext.proxyInfo && crawlingContext.proxyInfo.url;this._mergeRequestCookieDiff(request, cookieSnapshot, requestAsBrowserOptions);crawlingContext.response = await addTimeoutToPromise(- this._requestFunction({ request, session, proxyUrl, requestAsBrowserOptions }),+ () => this._requestFunction({ request, session, proxyUrl, requestAsBrowserOptions }),this.requestTimeoutMillis,`request timed out after ${this.requestTimeoutMillis / 1000} seconds.`,);+ tryCancel();await this._executeHooks(this.postNavigationHooks, crawlingContext, requestAsBrowserOptions);+ tryCancel();if (this.postResponseFunction) {this.log.deprecated('Option "postResponseFunction" is deprecated. Use "postNavigationHooks" instead.');await this.postResponseFunction(crawlingContext);+ tryCancel();}}diff --git a/src/utils.js b/src/utils.jsindex 8c3d0fec8..8e6074c13 100644--- a/src/utils.js+++ b/src/utils.js@@ -1,5 +1,4 @@import psTree from '@apify/ps-tree';-// eslint-disable-next-line import/no-extraneous-dependenciesimport { execSync } from 'child_process';import { ApifyClient } from 'apify-client';import { version as apifyClientVersion } from 'apify-client/package.json';@@ -9,7 +8,6 @@ import contentTypeParser from 'content-type';import fs from 'fs';import mime from 'mime-types';import os from 'os';-// eslint-disable-next-line import/no-extraneous-dependenciesimport ow from 'ow';import path from 'path';import semver from 'semver';@@ -361,29 +359,6 @@ export const getTypicalChromeExecutablePath = () => {}};-/**- * Wraps the provided Promise with another one that rejects with the given errorMessage- * after the given timeoutMillis, unless the original promise resolves or rejects earlier.- *- * @param {Promise<*>} promise- * @param {number} timeoutMillis- * @param {string} errorMessage- * @return {Promise<*>}- * @ignore- */-export const addTimeoutToPromise = (promise, timeoutMillis, errorMessage) => {- return new Promise((resolve, reject) => {- ow(promise, ow.promise);- ow(timeoutMillis, ow.number);- ow(errorMessage, ow.string);- const timeout = setTimeout(() => reject(new Error(errorMessage)), timeoutMillis);- promise- .then(resolve)- .catch(reject)- .finally(() => clearTimeout(timeout));- });-};-/*** Returns `true` when code is running on Apify platform and `false` otherwise (for example locally).*diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.jsindex 7d2062de7..b631d7472 100644--- a/test/crawlers/puppeteer_crawler.test.js+++ b/test/crawlers/puppeteer_crawler.test.js@@ -221,6 +221,110 @@ describe('PuppeteerCrawler', () => {expect(loadedUserAgent).toEqual(opts.userAgent);});+ test('timeout via preNavigationHooks will abort the page function as early as possible (gh #1216)', async () => {+ const requestQueue = await Apify.openRequestQueue();+ await requestQueue.addRequest({ url: 'http://www.example.com' });+ const handlePageFunction = jest.fn();++ const crawler = new Apify.PuppeteerCrawler({+ requestQueue,+ handlePageTimeoutSecs: 0.005,+ navigationTimeoutSecs: 0.005,+ preNavigationHooks: [+ async () => {+ await Apify.utils.sleep(20);+ },+ ],+ handlePageFunction,+ });++ const logSpy = jest.spyOn(crawler.log, 'exception');+ logSpy.mockImplementation(() => {});++ await crawler.run();+ await crawler.teardown();+ await requestQueue.drop();++ expect(handlePageFunction).not.toBeCalled();+ const exceptions = logSpy.mock.calls.map((call) => [call[0].message, call[1], call[2].retryCount]);+ expect(exceptions).toEqual([+ [+ 'Navigation timed out after 0.005 seconds.',+ 'handleRequestFunction failed, reclaiming failed request back to the list or queue',+ 1,+ ],+ [+ 'Navigation timed out after 0.005 seconds.',+ 'handleRequestFunction failed, reclaiming failed request back to the list or queue',+ 2,+ ],+ [+ 'Navigation timed out after 0.005 seconds.',+ 'handleRequestFunction failed, reclaiming failed request back to the list or queue',+ 3,+ ],+ [+ 'Navigation timed out after 0.005 seconds.',+ 'Request failed and reached maximum retries',+ undefined,+ ],+ ]);+ logSpy.mockRestore();+ });++ test('timeout in preLaunchHooks will abort the page function as early as possible (gh #1216)', async () => {+ const requestQueue = await Apify.openRequestQueue();+ await requestQueue.addRequest({ url: 'http://www.example.com' });+ const handlePageFunction = jest.fn();++ const crawler = new Apify.PuppeteerCrawler({+ requestQueue,+ navigationTimeoutSecs: 0.005,+ browserPoolOptions: {+ preLaunchHooks: [+ async () => {+ // Do some async work that's longer than navigationTimeoutSecs+ await Apify.utils.sleep(20);+ },+ ],+ },+ handlePageFunction,+ });++ const logSpy = jest.spyOn(crawler.log, 'exception');+ logSpy.mockImplementation(() => {});++ await crawler.run();+ await crawler.teardown();+ await requestQueue.drop();++ expect(handlePageFunction).not.toBeCalled();+ const exceptions = logSpy.mock.calls.map((call) => [call[0].message, call[1], call[2].retryCount]);+ expect(exceptions).toEqual([+ [+ 'Navigation timed out after 0.005 seconds.',+ 'handleRequestFunction failed, reclaiming failed request back to the list or queue',+ 1,+ ],+ [+ 'Navigation timed out after 0.005 seconds.',+ 'handleRequestFunction failed, reclaiming failed request back to the list or queue',+ 2,+ ],+ [+ 'Navigation timed out after 0.005 seconds.',+ 'handleRequestFunction failed, reclaiming failed request back to the list or queue',+ 3,+ ],+ [+ 'Navigation timed out after 0.005 seconds.',+ 'Request failed and reached maximum retries',+ undefined,+ ],+ ]);+ logSpy.mockRestore();+ });+test('should set cookies assigned to session to page', async () => {const cookies = [{diff --git a/test/utils.test.js b/test/utils.test.jsindex 466da70da..0ddded45a 100644--- a/test/utils.test.js+++ b/test/utils.test.js@@ -7,6 +7,7 @@ import os from 'os';import cheerio from 'cheerio';import semver from 'semver';import { ENV_VARS } from '@apify/consts';+import { addTimeoutToPromise } from '@apify/timeout';import Apify from '../build/index';import * as utils from '../build/utils';import log from '../build/utils_log';@@ -812,8 +813,8 @@ describe('utils.addTimeoutToPromise()', () => {test('should timeout', async () => {const clock = sinon.useFakeTimers();try {- const p = utils.addTimeoutToPromise(- new Promise((r) => setTimeout(r, 500)),+ const p = addTimeoutToPromise(+ () => new Promise((r) => setTimeout(r, 500)),100,'Timed out.',);@@ -830,8 +831,8 @@ describe('utils.addTimeoutToPromise()', () => {test('should not timeout too soon', async () => {const clock = sinon.useFakeTimers();try {- const p = utils.addTimeoutToPromise(- new Promise((r) => setTimeout(() => r('Done'), 100)),+ const p = addTimeoutToPromise(+ () => new Promise((r) => setTimeout(() => r('Done'), 100)),500,'Timed out.',);