Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: disable the `taskTimeoutSecs` by default (#1083)

Martin Adámek94fa888aedb1
diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.js
index caf340327..ac4022d77 100644
--- a/src/autoscaling/autoscaled_pool.js
+++ b/src/autoscaling/autoscaled_pool.js
@@ -53,7 +53,7 @@ import { addTimeoutToPromise } from '../utils';
* Defines in seconds how often the pool should attempt to adjust the desired concurrency
* based on the latest system status. Setting it lower than 1 might have a severe impact on performance.
* We suggest using a value from 5 to 20.
- * @property {number} [taskTimeoutSecs=60]
+ * @property {number} [taskTimeoutSecs=0]
* Timeout in which the `runTaskFunction` needs to finish, given in seconds.
* @property {SnapshotterOptions} [snapshotterOptions]
* Options to be passed down to the {@link Snapshotter} constructor. This is useful for fine-tuning
@@ -148,7 +148,7 @@ class AutoscaledPool {
scaleDownStepRatio = 0.05,
maybeRunIntervalSecs = 0.5,
loggingIntervalSecs = 60,
- taskTimeoutSecs = 60,
+ taskTimeoutSecs = 0,
autoscaleIntervalSecs = 10,
systemStatusOptions,
snapshotterOptions,
@@ -438,11 +438,15 @@ class AutoscaledPool {
// Execute the current task.
this.log.perf('Running a task.');
- await addTimeoutToPromise(
- this.runTaskFunction(),
- this.taskTimeoutMillis,
- `runTaskFunction timed out after ${this.taskTimeoutMillis / 1000} seconds.`,
- );
+ if (this.taskTimeoutMillis > 0) {
+ await addTimeoutToPromise(
+ this.runTaskFunction(),
+ this.taskTimeoutMillis,
+ `runTaskFunction timed out after ${this.taskTimeoutMillis / 1000} seconds.`,
+ );
+ } else {
+ await this.runTaskFunction();
+ }
this.log.perf('Task finished.');
this._currentConcurrency--;
diff --git a/test/autoscaling/autoscaled_pool.test.js b/test/autoscaling/autoscaled_pool.test.js
index f502ac9c7..6dc8cb53d 100644
--- a/test/autoscaling/autoscaled_pool.test.js
+++ b/test/autoscaling/autoscaled_pool.test.js
@@ -454,7 +454,7 @@ describe('AutoscaledPool', () => {
test('should timeout after taskTimeoutSecs', async () => {
const runTaskFunction = async () => {
- await new Promise((resolve) => setTimeout(resolve, 1e3));
+ await sleep(1e3);
return 1;
};
@@ -471,6 +471,50 @@ describe('AutoscaledPool', () => {
await expect(pool.run()).rejects.toThrow('runTaskFunction timed out after 0.1 seconds.');
expect(Date.now() - now).toBeGreaterThanOrEqual(100);
});
+
+ test('should not timeout if taskTimeoutSecs === 0', async () => {
+ let finished = false;
+
+ const runTaskFunction = async () => {
+ await sleep(1e3);
+ finished = true;
+ return 1;
+ };
+ const pool = new AutoscaledPool({
+ minConcurrency: 1,
+ maxConcurrency: 1,
+ runTaskFunction,
+ taskTimeoutSecs: 0,
+ isFinishedFunction: async () => finished,
+ isTaskReadyFunction: async () => !finished,
+ });
+
+ const now = Date.now();
+ await expect(pool.run()).resolves.toBeUndefined();
+ expect(Date.now() - now).toBeGreaterThanOrEqual(1e3);
+ }, 3e3);
+
+ test('should not timeout if taskTimeoutSecs not explicitly set', async () => {
+ let finished = false;
+
+ const runTaskFunction = async () => {
+ await sleep(1e3);
+ finished = true;
+ return 1;
+ };
+
+ const pool = new AutoscaledPool({
+ minConcurrency: 1,
+ maxConcurrency: 1,
+ runTaskFunction,
+ isFinishedFunction: async () => finished,
+ isTaskReadyFunction: async () => !finished,
+ });
+
+ const now = Date.now();
+ await expect(pool.run()).resolves.toBeUndefined();
+ expect(Date.now() - now).toBeGreaterThanOrEqual(1e3);
+ });
});
/* eslint-enable no-underscore-dangle */