Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add `taskTimeoutSecs` to allow control over timeout of `AutoscaledPool` tasks (#1082)

Martin Adámekff6eb3f86d0c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bc85f98d4..6c5126df3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,6 +47,7 @@ await dataset.pushData({ myValue: 123 });
- Fix `maxUsageCount` of a `Session` not being persisted.
- Update `puppeteer` and `playwright` to match stable Chrome (90).
- Fix support for building TypeScript projects that depend on the SDK.
+- add `taskTimeoutSecs` to allow control over timeout of `AutoscaledPool` tasks
1.2.1 / 2021/05/14
====================
diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.js
index 44d063e60..caf340327 100644
--- a/src/autoscaling/autoscaled_pool.js
+++ b/src/autoscaling/autoscaled_pool.js
@@ -3,6 +3,7 @@ import ow from 'ow';
import Snapshotter, { SnapshotterOptions } from './snapshotter'; // eslint-disable-line import/named,no-unused-vars
import SystemStatus, { SystemStatusOptions } from './system_status'; // eslint-disable-line import/named,no-unused-vars
import defaultLog from '../utils_log';
+import { addTimeoutToPromise } from '../utils';
/**
* @typedef AutoscaledPoolOptions
@@ -52,6 +53,8 @@ import defaultLog from '../utils_log';
* Defines in seconds how often the pool should attempt to adjust the desired concurrency
* based on the latest system status. Setting it lower than 1 might have a severe impact on performance.
* We suggest using a value from 5 to 20.
+ * @property {number} [taskTimeoutSecs=60]
+ * Timeout in which the `runTaskFunction` needs to finish, given in seconds.
* @property {SnapshotterOptions} [snapshotterOptions]
* Options to be passed down to the {@link Snapshotter} constructor. This is useful for fine-tuning
* the snapshot intervals and history.
@@ -127,6 +130,7 @@ class AutoscaledPool {
maybeRunIntervalSecs: ow.optional.number,
loggingIntervalSecs: ow.any(ow.number, ow.nullOrUndefined),
autoscaleIntervalSecs: ow.optional.number,
+ taskTimeoutSecs: ow.optional.number,
systemStatusOptions: ow.optional.object,
snapshotterOptions: ow.optional.object,
log: ow.optional.object,
@@ -144,6 +148,7 @@ class AutoscaledPool {
scaleDownStepRatio = 0.05,
maybeRunIntervalSecs = 0.5,
loggingIntervalSecs = 60,
+ taskTimeoutSecs = 60,
autoscaleIntervalSecs = 10,
systemStatusOptions,
snapshotterOptions,
@@ -162,6 +167,7 @@ class AutoscaledPool {
this.maybeRunIntervalMillis = maybeRunIntervalSecs * 1000;
this.loggingIntervalMillis = loggingIntervalSecs * 1000;
this.autoscaleIntervalMillis = autoscaleIntervalSecs * 1000;
+ this.taskTimeoutMillis = taskTimeoutSecs * 1000;
this.runTaskFunction = runTaskFunction;
this.isFinishedFunction = isFinishedFunction;
this.isTaskReadyFunction = isTaskReadyFunction;
@@ -431,7 +437,13 @@ class AutoscaledPool {
// Execute the current task.
this.log.perf('Running a task.');
- await this.runTaskFunction();
+
+ await addTimeoutToPromise(
+ this.runTaskFunction(),
+ this.taskTimeoutMillis,
+ `runTaskFunction timed out after ${this.taskTimeoutMillis / 1000} seconds.`,
+ );
+
this.log.perf('Task finished.');
this._currentConcurrency--;
// Run task after the previous one finished.
diff --git a/test/autoscaling/autoscaled_pool.test.js b/test/autoscaling/autoscaled_pool.test.js
index e29d61c99..f502ac9c7 100644
--- a/test/autoscaling/autoscaled_pool.test.js
+++ b/test/autoscaling/autoscaled_pool.test.js
@@ -451,6 +451,26 @@ describe('AutoscaledPool', () => {
expect(results).toHaveLength(count);
results.forEach((r, i) => expect(r).toEqual(i));
});
+
+ test('should timeout after taskTimeoutSecs', async () => {
+ const runTaskFunction = async () => {
+ await new Promise((resolve) => setTimeout(resolve, 1e3));
+ return 1;
+ };
+
+ const pool = new AutoscaledPool({
+ minConcurrency: 1,
+ maxConcurrency: 1,
+ runTaskFunction,
+ taskTimeoutSecs: 0.1,
+ isFinishedFunction: async () => false,
+ isTaskReadyFunction: async () => true,
+ });
+
+ const now = Date.now();
+ await expect(pool.run()).rejects.toThrow('runTaskFunction timed out after 0.1 seconds.');
+ expect(Date.now() - now).toBeGreaterThanOrEqual(100);
+ });
});
/* eslint-enable no-underscore-dangle */