Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: add timeouts with retries to `runTaskFunction()` (#1250)

Martin Adámeka2718628cc19
diff --git a/package.json b/package.json
index ae81fb360..e53176a3f 100644
--- a/package.json
+++ b/package.json
@@ -126,7 +126,7 @@
"sinon": "^12.0.0",
"sinon-stub-promise": "^4.0.0",
"ts-jest": "^27.0.5",
- "typescript": "4.5.2",
+ "typescript": "4.4.4",
"ua-parser-js": "^1.0.0",
"why-is-node-running": "^2.2.0"
}
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.js
index ce8d72ce2..9d260c488 100644
--- a/src/crawlers/basic_crawler.js
+++ b/src/crawlers/basic_crawler.js
@@ -2,7 +2,7 @@ import { ACTOR_EVENT_NAMES } from '@apify/consts';
import { cryptoRandomObjectId } from '@apify/utilities';
import ow, { ArgumentError } from 'ow';
import _ from 'underscore';
-import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
+import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
import AutoscaledPool from '../autoscaling/autoscaled_pool'; // eslint-disable-line import/no-duplicates
import events from '../events';
import { openSessionPool } from '../session_pool/session_pool'; // eslint-disable-line import/no-duplicates
@@ -289,7 +289,7 @@ export class BasicCrawler {
const basicCrawlerAutoscaledPoolConfiguration = {
minConcurrency,
maxConcurrency,
- runTaskFunction: this.timeoutForRunTaskFunction.bind(this),
+ runTaskFunction: this._runTaskFunction.bind(this),
isTaskReadyFunction: async () => {
if (isMaxPagesExceeded()) {
if (shouldLogMaxPagesExceeded) {
@@ -460,27 +460,8 @@ export class BasicCrawler {
await this.requestList.reclaimRequest(request);
return null;
}
- const [nextRequest] = await Promise.all([
- this.requestQueue.fetchNextRequest(),
- this.requestList.markRequestHandled(request),
- ]);
- return nextRequest;
- }
-
- /**
- * Wrapper around _runTaskFunction to set a timeout on the entire function.
- * Temporary solution to see whether this causes the 0 concurrency bug.
- * @ignore
- * @internal
- * @private
- */
- async timeoutForRunTaskFunction() {
- const timeout = 600e3; // 10 min last resort timeout for stuck runs
- await addTimeoutToPromise(
- () => this._runTaskFunction(),
- timeout,
- `_runTaskFunction timed out after ${(timeout) / 1000} seconds.`,
- );
+ await this.requestList.markRequestHandled(request);
+ return this.requestQueue.fetchNextRequest();
}
/**
@@ -497,11 +478,17 @@ export class BasicCrawler {
let request;
let session;
- if (this.useSessionPool) {
- [request, session] = await Promise.all([this._fetchNextRequest(), this.sessionPool.getSession()]);
- } else {
- request = await this._fetchNextRequest();
- }
+ await this._timeoutAndRetry(
+ async () => {
+ if (this.useSessionPool) {
+ [request, session] = await Promise.all([this._fetchNextRequest(), this.sessionPool.getSession()]);
+ } else {
+ request = await this._fetchNextRequest();
+ }
+ },
+ this.handleRequestTimeoutMillis,
+ `Fetching next request timed out after ${this.handleRequestTimeoutMillis * 1e3} seconds.`,
+ );
tryCancel();
@@ -529,7 +516,12 @@ export class BasicCrawler {
`handleRequestFunction timed out after ${this.handleRequestTimeoutMillis / 1000} seconds.`,
);
tryCancel();
- await source.markRequestHandled(request);
+
+ await this._timeoutAndRetry(
+ () => source.markRequestHandled(request),
+ this.handleRequestTimeoutMillis,
+ `Marking request ${request.url} as handled timed out after ${this.handleRequestTimeoutMillis * 1e3} seconds.`,
+ );
tryCancel();
this.stats.finishJob(statisticsId);
this.handledRequestsCount++;
@@ -538,7 +530,11 @@ export class BasicCrawler {
if (session) session.markGood();
} catch (err) {
try {
- await this._requestFunctionErrorHandler(err, crawlingContext, source);
+ await this._timeoutAndRetry(
+ () => this._requestFunctionErrorHandler(err, crawlingContext, source),
+ this.handleRequestTimeoutMillis,
+ `Handling request failure of ${request.url} timed out after ${this.handleRequestTimeoutMillis * 1e3} seconds.`,
+ );
} catch (secondaryError) {
this.log.exception(secondaryError, 'runTaskFunction error handler threw an exception. '
+ 'This places the crawler and its underlying storages into an unknown state and crawling will be terminated. '
@@ -554,6 +550,27 @@ export class BasicCrawler {
tryCancel();
}
+ /**
+ * Run async callback with given timeout and retry.
+ * @ignore
+ */
+ async _timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
+ try {
+ await addTimeoutToPromise(
+ handler,
+ timeout,
+ error,
+ );
+ } catch (e) {
+ if (e instanceof TimeoutError && retried <= maxRetries) {
+ this.log.warning(`${e.message} (retrying ${retried}/${maxRetries})`);
+ return this._timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
+ }
+
+ throw e;
+ }
+ }
+
/**
* Returns true if either RequestList or RequestQueue have a request ready for processing.
*