Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

test(basic-crawler): pin that minConcurrency tasks keep running under overload (#4316)

Vlad Frangu11562c825ba3
diff --git a/docs/guides/scaling-crawlers.mdx b/docs/guides/scaling-crawlers.mdx
index d551ae039..461aba30c 100644
--- a/docs/guides/scaling-crawlers.mdx
+++ b/docs/guides/scaling-crawlers.mdx
@@ -33,7 +33,7 @@ This controls how many total requests can be made per minute. It counts the amou
## `minConcurrency` and `maxConcurrency`
-These control how many parallel requests can be run at any time. By default, crawlers will start with one parallel request at a time and scale up over time to a maximum of 200 requests at a time.
+These control how many parallel requests can be run at any time. By default, crawlers will start with one parallel request at a time and scale up over time to a maximum of 200 requests at a time. `minConcurrency` is a floor: the crawler keeps that many requests running even while the system is overloaded, and only scales further up when there is enough free CPU and memory.
:::caution Don't set `minConcurrency` too high!
diff --git a/packages/basic-crawler/src/internals/autoscaling/autoscaled-pool.ts b/packages/basic-crawler/src/internals/autoscaling/autoscaled-pool.ts
index 534dbef1c..00fe61781 100644
--- a/packages/basic-crawler/src/internals/autoscaling/autoscaled-pool.ts
+++ b/packages/basic-crawler/src/internals/autoscaling/autoscaled-pool.ts
@@ -96,7 +96,9 @@ export interface AutoscaledPoolOptions extends TaskLoopPredicates {
/**
* Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
* The pool only starts new tasks while its {@apilink IConcurrencySystem|concurrency system} reports free capacity —
- * that governor is what monitors CPU, memory and event loop load and autoscales the concurrency budget.
+ * that governor is what monitors CPU, memory and event loop load and autoscales the concurrency budget. The default
+ * {@apilink ConcurrencySystem} keeps `minConcurrency` tasks running even while the system is overloaded, and only
+ * starts additional ones when there is enough free CPU and memory.
*
* Before running the pool, you need to implement the following three functions:
* {@apilink AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`},
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 3c05c6240..59d2e1828 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -660,7 +660,8 @@ export interface BasicCrawlerOptions<
*
* The crawler finishes if there are no more {@apilink Request} objects to crawl.
*
- * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
+ * The crawler keeps `minConcurrency` requests in flight even while the system is overloaded, and dispatches
+ * additional ones only when there is enough free CPU and memory available, as judged by its
* {@apilink ConcurrencySystem}.
* Concurrency is tuned via the {@apilink BasicCrawlerOptions.minConcurrency|`minConcurrency`},
* {@apilink BasicCrawlerOptions.maxConcurrency|`maxConcurrency`} and
diff --git a/test/core/autoscaling/autoscaled-pool.test.ts b/test/core/autoscaling/autoscaled-pool.test.ts
index 557631fd3..deb5187e4 100644
--- a/test/core/autoscaling/autoscaled-pool.test.ts
+++ b/test/core/autoscaling/autoscaled-pool.test.ts
@@ -315,7 +315,8 @@ describe('AutoscaledPool', () => {
expect(pool.currentConcurrency).toBe(0);
await pool.run();
- expect(concurrencyLog.some((n) => n > 1)).toBe(false);
+ // The overload caps the pool at the floor, but must not push it below - exactly `minConcurrency` runs.
+ expect(Math.max(...concurrencyLog)).toBe(1);
limit = 50;
concurrencyLog = [];
@@ -323,7 +324,55 @@ describe('AutoscaledPool', () => {
systemOf(pool).minConcurrency = 5;
await pool.run();
- expect(concurrencyLog.some((n) => n > 5)).toBe(false);
+ expect(Math.max(...concurrencyLog)).toBe(5);
+ });
+
+ test('settles at minConcurrency instead of freezing when an overload starts mid-run', async () => {
+ let running = 0;
+ let peak = 0;
+ let done = 0;
+
+ const pool = await makePool(
+ {
+ runTaskFunction: async () => {
+ running++;
+ peak = Math.max(peak, running);
+ await sleep(10);
+ running--;
+ done++;
+ },
+ isFinishedFunction: async () => false,
+ isTaskReadyFunction: async () => true,
+ },
+ { ...concurrencyOptions, minConcurrency: 2, maxConcurrency: 4, desiredConcurrency: 4 },
+ );
+
+ const waitFor = async (condition: () => boolean) => {
+ const deadline = Date.now() + 5_000;
+ while (!condition()) {
+ if (Date.now() > deadline) throw new Error('Timed out waiting for the pool to settle.');
+ await sleep(5);
+ }
+ };
+
+ const runPromise = pool.run();
+ onTestFinished(async () => {
+ await pool.abort();
+ await runPromise;
+ });
+
+ await waitFor(() => peak === 4);
+
+ // Only the gating status flips - `desiredConcurrency` stays at 4, so what keeps tasks running (or not)
+ // from here on is the overload check alone.
+ load.okNow = false;
+ await waitFor(() => pool.currentConcurrency <= 2);
+
+ peak = 0;
+ const doneBefore = done;
+ await waitFor(() => done >= doneBefore + 4);
+
+ expect(peak).toBe(2);
});
});
diff --git a/test/core/autoscaling/concurrency-system.test.ts b/test/core/autoscaling/concurrency-system.test.ts
index edf949ea8..e6df48c98 100644
--- a/test/core/autoscaling/concurrency-system.test.ts
+++ b/test/core/autoscaling/concurrency-system.test.ts
@@ -73,6 +73,42 @@ describe('ConcurrencySystem', () => {
expect(system.currentConcurrency).toBe(1);
});
+ test('hasCapacityForTask() keeps minConcurrency tasks running while the system is overloaded', async () => {
+ // Every sample reports overload, so the gating status is never idle.
+ const overloaded: LoadSignal = {
+ name: 'overloaded',
+ overloadedRatio: 0.5,
+ async start() {},
+ async stop() {},
+ getSample: () => [{ createdAt: new Date(), isOverloaded: true }],
+ };
+ const system = new ConcurrencySystem({
+ minConcurrency: 2,
+ maxConcurrency: 4,
+ desiredConcurrency: 4,
+ loadSignals: {
+ memory: false,
+ eventLoop: false,
+ cpu: false,
+ storageBackend: false,
+ custom: [overloaded],
+ },
+ });
+ await system.start();
+ onTestFinished(async () => system.stop());
+
+ // Below the minimum the overload is ignored - the floor must keep running, or the pool would freeze.
+ expect(system.tryRegisterTaskStart()).toBe(true);
+ expect(system.tryRegisterTaskStart()).toBe(true);
+ expect(system.currentConcurrency).toBe(2);
+ // At the minimum the overload is what stops further tasks, well before the desired budget of 4.
+ expect(system.hasCapacityForTask()).toBe(false);
+ expect(system.tryRegisterTaskStart()).toBe(false);
+
+ system.registerTaskEnd();
+ expect(system.hasCapacityForTask()).toBe(true);
+ });
+
test('maxTasksPerMinute === Infinity never limits', () => {
const system = new ConcurrencySystem({
minConcurrency: 1000,
@@ -598,6 +634,56 @@ describe('ConcurrencySystem', () => {
expect(combinedPeak).toBeLessThanOrEqual(4);
});
+ test('minConcurrency under overload is a machine-wide floor, not a per-pool one', async () => {
+ const overloaded: LoadSignal = {
+ name: 'overloaded',
+ overloadedRatio: 0.5,
+ async start() {},
+ async stop() {},
+ getSample: () => [{ createdAt: new Date(), isOverloaded: true }],
+ };
+ // One shared floor of 2 under permanent overload; two pools that would each happily run 4 in parallel.
+ const system = new ConcurrencySystem({
+ minConcurrency: 2,
+ maxConcurrency: 4,
+ desiredConcurrency: 4,
+ loadSignals: {
+ memory: false,
+ eventLoop: false,
+ cpu: false,
+ storageBackend: false,
+ custom: [overloaded],
+ },
+ });
+
+ let combinedCurrent = 0;
+ let combinedPeak = 0;
+
+ const makePool = (id: string, taskCount: number) => {
+ let done = 0;
+ return new AutoscaledPool({
+ concurrencySystem: system,
+ consumer: { id },
+ runTaskFunction: async () => {
+ combinedCurrent++;
+ combinedPeak = Math.max(combinedPeak, combinedCurrent);
+ await sleep(5);
+ done++;
+ combinedCurrent--;
+ },
+ isFinishedFunction: async () => done >= taskCount,
+ isTaskReadyFunction: async () => done < taskCount,
+ });
+ };
+
+ await system.start();
+ await Promise.all([makePool('first', 20).run(), makePool('second', 20).run()]);
+ await system.stop();
+
+ // The floor is shared: the two pools together keep exactly `minConcurrency` running, not 2 each.
+ expect(combinedPeak).toBe(2);
+ });
+
test('a pool that ran out of work finishes while another one saturates the shared budget', async () => {
// A budget of exactly one slot, so the hog below holds all of it for as long as we let it.
const system = new ConcurrencySystem({ minConcurrency: 1, maxConcurrency: 1, desiredConcurrency: 1 });