Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: disable the `taskTimeoutSecs` by default (#1083)
Martin Adámek94fa888aedb1
diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.jsindex caf340327..ac4022d77 100644--- a/src/autoscaling/autoscaled_pool.js+++ b/src/autoscaling/autoscaled_pool.js@@ -53,7 +53,7 @@ import { addTimeoutToPromise } from '../utils';* Defines in seconds how often the pool should attempt to adjust the desired concurrency* based on the latest system status. Setting it lower than 1 might have a severe impact on performance.* We suggest using a value from 5 to 20.- * @property {number} [taskTimeoutSecs=60]+ * @property {number} [taskTimeoutSecs=0]* Timeout in which the `runTaskFunction` needs to finish, given in seconds.* @property {SnapshotterOptions} [snapshotterOptions]* Options to be passed down to the {@link Snapshotter} constructor. This is useful for fine-tuning@@ -148,7 +148,7 @@ class AutoscaledPool {scaleDownStepRatio = 0.05,maybeRunIntervalSecs = 0.5,loggingIntervalSecs = 60,- taskTimeoutSecs = 60,+ taskTimeoutSecs = 0,autoscaleIntervalSecs = 10,systemStatusOptions,snapshotterOptions,@@ -438,11 +438,15 @@ class AutoscaledPool {// Execute the current task.this.log.perf('Running a task.');- await addTimeoutToPromise(- this.runTaskFunction(),- this.taskTimeoutMillis,- `runTaskFunction timed out after ${this.taskTimeoutMillis / 1000} seconds.`,- );+ if (this.taskTimeoutMillis > 0) {+ await addTimeoutToPromise(+ this.runTaskFunction(),+ this.taskTimeoutMillis,+ `runTaskFunction timed out after ${this.taskTimeoutMillis / 1000} seconds.`,+ );+ } else {+ await this.runTaskFunction();+ }this.log.perf('Task finished.');this._currentConcurrency--;diff --git a/test/autoscaling/autoscaled_pool.test.js b/test/autoscaling/autoscaled_pool.test.jsindex f502ac9c7..6dc8cb53d 100644--- a/test/autoscaling/autoscaled_pool.test.js+++ b/test/autoscaling/autoscaled_pool.test.js@@ -454,7 +454,7 @@ describe('AutoscaledPool', () => {test('should timeout after taskTimeoutSecs', async () => {const runTaskFunction = async () => {- await new Promise((resolve) => setTimeout(resolve, 1e3));+ await sleep(1e3);return 1;};@@ -471,6 +471,50 @@ describe('AutoscaledPool', () => {await expect(pool.run()).rejects.toThrow('runTaskFunction timed out after 0.1 seconds.');expect(Date.now() - now).toBeGreaterThanOrEqual(100);});++ test('should not timeout if taskTimeoutSecs === 0', async () => {+ let finished = false;++ const runTaskFunction = async () => {+ await sleep(1e3);+ finished = true;+ return 1;+ };+ const pool = new AutoscaledPool({+ minConcurrency: 1,+ maxConcurrency: 1,+ runTaskFunction,+ taskTimeoutSecs: 0,+ isFinishedFunction: async () => finished,+ isTaskReadyFunction: async () => !finished,+ });++ const now = Date.now();+ await expect(pool.run()).resolves.toBeUndefined();+ expect(Date.now() - now).toBeGreaterThanOrEqual(1e3);+ }, 3e3);++ test('should not timeout if taskTimeoutSecs not explicitly set', async () => {+ let finished = false;++ const runTaskFunction = async () => {+ await sleep(1e3);+ finished = true;+ return 1;+ };++ const pool = new AutoscaledPool({+ minConcurrency: 1,+ maxConcurrency: 1,+ runTaskFunction,+ isFinishedFunction: async () => finished,+ isTaskReadyFunction: async () => !finished,+ });++ const now = Date.now();+ await expect(pool.run()).resolves.toBeUndefined();+ expect(Date.now() - now).toBeGreaterThanOrEqual(1e3);+ });});/* eslint-enable no-underscore-dangle */