Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add `initialConcurrency` crawler option (#4067)

Jindřich Bär45565e276d55
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md
index 466ff3cac..9b543bd67 100644
--- a/docs/public-api/crawlee-basic.api.md
+++ b/docs/public-api/crawlee-basic.api.md
@@ -124,6 +124,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
httpClient?: BaseHttpClient;
id?: string;
ignoreHttpErrorStatusCodes?: number[];
+ initialConcurrency?: number;
keepAlive?: boolean;
logger?: CrawleeLogger;
maxConcurrency?: number;
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index f9474283c..c399229f0 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -392,8 +392,8 @@ export interface BasicCrawlerOptions<
* single budget. Each crawler still builds and drives its own {@apilink AutoscaledPool}; only the load/scaling
* accounting is shared.
*
- * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts, which configure
- * the default system this one replaces — combining the two throws.
+ * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute`
+ * shortcuts, which configure the default system this one replaces — combining the two throws.
*
* You own a supplied system's lifecycle: `start()` it before `run()` (which throws otherwise) and `stop()` it once
* every crawler borrowing it has finished. The crawler does neither on your behalf.
@@ -416,6 +416,13 @@ export interface BasicCrawlerOptions<
*/
maxConcurrency?: number;
+ /**
+ * Sets the concurrency (parallelism) the crawl starts with, before any scaling happens. Shortcut for the
+ * {@apilink ConcurrencySystemOptions.desiredConcurrency|`desiredConcurrency`} option of the crawler's default
+ * {@apilink ConcurrencySystem}. Defaults to `minConcurrency`.
+ */
+ initialConcurrency?: number;
+
/**
* The maximum number of requests per minute the crawler should run.
* By default, this is set to `Infinity`, but we can pass any positive, non-zero integer.
@@ -937,6 +944,7 @@ export class BasicCrawler<
// AutoscaledPool shorthands
minConcurrency: schemas.anyNumber.optional(),
maxConcurrency: schemas.anyNumber.optional(),
+ initialConcurrency: schemas.anyNumber.optional(),
maxRequestsPerMinute: schemas.anyNumber
.refine((value) => Number.isInteger(value) || value === Infinity, 'Expected an integer or infinite number')
.refine((value) => value >= 1, 'Expected a number greater than or equal to 1')
@@ -987,6 +995,7 @@ export class BasicCrawler<
// AutoscaledPool shorthands
minConcurrency,
maxConcurrency,
+ initialConcurrency,
maxRequestsPerMinute,
blockedStatusCodes: blockedStatusCodesInput,
@@ -1011,12 +1020,15 @@ export class BasicCrawler<
// hammering a site.
if (
concurrencySystem !== undefined &&
- (minConcurrency !== undefined || maxConcurrency !== undefined || maxRequestsPerMinute !== undefined)
+ (minConcurrency !== undefined ||
+ maxConcurrency !== undefined ||
+ initialConcurrency !== undefined ||
+ maxRequestsPerMinute !== undefined)
) {
throw new Error(
- 'The `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts cannot be combined with ' +
- '`concurrencySystem` - they configure the default `ConcurrencySystem` that a supplied one ' +
- 'replaces. Pass them to the `ConcurrencySystem` constructor instead.',
+ 'The `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts ' +
+ 'cannot be combined with `concurrencySystem` - they configure the default `ConcurrencySystem` ' +
+ 'that a supplied one replaces. Pass them to the `ConcurrencySystem` constructor instead.',
);
}
@@ -1348,6 +1360,9 @@ export class BasicCrawler<
minConcurrency,
maxConcurrency,
maxTasksPerMinute: maxRequestsPerMinute,
+ // Spread conditionally - an explicit `undefined` would clobber a subclass default, see
+ // `HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS`.
+ ...(initialConcurrency !== undefined && { desiredConcurrency: initialConcurrency }),
log: this.log,
}),
);
@@ -1358,7 +1373,7 @@ export class BasicCrawler<
/**
* Builds the crawler-owned default {@apilink ConcurrencySystem} from the resolved
- * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
+ * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
* {@apilink BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
*
* Subclasses may override this to tune the default system (e.g. {@apilink HttpCrawler} raises the starting
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 0dac5e1d5..8cb7190b2 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -276,7 +276,7 @@ describe('BasicCrawler', () => {
expect(bookedFor).toEqual(new Set(['crawler-a', 'crawler-b']));
});
- test.each(['minConcurrency', 'maxConcurrency', 'maxRequestsPerMinute'] as const)(
+ test.each(['minConcurrency', 'maxConcurrency', 'initialConcurrency', 'maxRequestsPerMinute'] as const)(
'throws when %s is combined with a supplied concurrencySystem',
(shortcut) => {
expect(
@@ -624,6 +624,7 @@ describe('BasicCrawler', () => {
const collect = (crawler: BasicCrawler) => ({
minConcurrency: (crawler.concurrencySystem! as ConcurrencySystem).minConcurrency,
maxConcurrency: (crawler.concurrencySystem! as ConcurrencySystem).maxConcurrency,
+ desiredConcurrency: (crawler.concurrencySystem! as ConcurrencySystem).desiredConcurrency,
// eslint-disable-next-line dot-notation -- private member on the governor
maxTasksPerMinute: (crawler.concurrencySystem! as ConcurrencySystem)['maxTasksPerMinute'],
});
@@ -634,11 +635,17 @@ describe('BasicCrawler', () => {
requestHandler,
minConcurrency: 123,
maxConcurrency: 456,
+ initialConcurrency: 234,
maxRequestsPerMinute: 789,
});
// An injected system carries its own config (the shortcuts are rejected alongside one, see above).
- const injectedSystem = new ConcurrencySystem({ minConcurrency: 16, maxConcurrency: 32, maxTasksPerMinute: 64 });
+ const injectedSystem = new ConcurrencySystem({
+ minConcurrency: 16,
+ maxConcurrency: 32,
+ desiredConcurrency: 24,
+ maxTasksPerMinute: 64,
+ });
const injected = new BasicCrawler({
requestList,
requestHandler,
@@ -650,8 +657,18 @@ describe('BasicCrawler', () => {
await Promise.all([shortcuts.run(), injected.run()]);
await injectedSystem.stop();
- expect(collect(shortcuts)).toEqual({ minConcurrency: 123, maxConcurrency: 456, maxTasksPerMinute: 789 });
- expect(collect(injected)).toEqual({ minConcurrency: 16, maxConcurrency: 32, maxTasksPerMinute: 64 });
+ expect(collect(shortcuts)).toEqual({
+ minConcurrency: 123,
+ maxConcurrency: 456,
+ desiredConcurrency: 234,
+ maxTasksPerMinute: 789,
+ });
+ expect(collect(injected)).toEqual({
+ minConcurrency: 16,
+ maxConcurrency: 32,
+ desiredConcurrency: 24,
+ maxTasksPerMinute: 64,
+ });
// The injected system is the very instance the pool uses.
expect(injected.concurrencySystem!).toBe(injectedSystem);
});
diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts
index 1ba725d90..d18cae419 100644
--- a/test/core/crawlers/http_crawler.test.ts
+++ b/test/core/crawlers/http_crawler.test.ts
@@ -193,6 +193,18 @@ test('concurrency shortcuts coexist with the HTTP-optimized defaults', async ()
expect(crawler.asConfigured!.desiredConcurrency).toBe(5);
});
+test('initialConcurrency overrides the HTTP-optimized starting concurrency', async () => {
+ const crawler = new ObservableHttpCrawler({
+ initialConcurrency: 3,
+ maxRequestRetries: 0,
+ requestHandler: () => {},
+ });
+
+ await crawler.run([url]);
+
+ expect(crawler.asConfigured!.desiredConcurrency).toBe(3);
+});
+
test('parseWithCheerio works', async () => {
const results: string[] = [];