Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: concurrency option override order (#1649)

Jindřich Bär7bbad0380cd6
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 5606ae4a7..d9f9d332f 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -533,8 +533,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
}
const basicCrawlerAutoscaledPoolConfiguration: Partial<AutoscaledPoolOptions> = {
- minConcurrency,
- maxConcurrency,
+ minConcurrency: minConcurrency ?? autoscaledPoolOptions?.minConcurrency,
+ maxConcurrency: maxConcurrency ?? autoscaledPoolOptions?.maxConcurrency,
maxTasksPerMinute: maxRequestsPerMinute ?? autoscaledPoolOptions?.maxTasksPerMinute,
runTaskFunction: this._runTaskFunction.bind(this),
isTaskReadyFunction: async () => {
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index b44b4e5fa..6e831957f 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -18,12 +18,24 @@ import {
CriticalError,
MissingRouteError,
} from '@crawlee/basic';
+import {
+ AutoscaledPool,
+} from '@crawlee/core';
import express from 'express';
import type { Dictionary } from '@crawlee/utils';
import { sleep } from '@crawlee/utils';
import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';
import { startExpressAppPromise } from '../../shared/_helper';
+jest.mock('@crawlee/core', () => {
+ const originalModule = jest.requireActual('@crawlee/core');
+ const AutoscaledPoolMockConstructor = jest.fn((...args) => new originalModule.AutoscaledPool(...args));
+ return {
+ ...originalModule,
+ AutoscaledPool: AutoscaledPoolMockConstructor,
+ };
+});
+
describe('BasicCrawler', () => {
let logLevel: number;
const localStorageEmulator = new MemoryStorageEmulator();
@@ -91,6 +103,55 @@ describe('BasicCrawler', () => {
expect(await requestList.isEmpty()).toBe(true);
});
+ test('should correctly combine shorthand and full length options', async () => {
+ const shorthandOptions = {
+ options: {
+ minConcurrency: 123,
+ maxConcurrency: 456,
+ maxRequestsPerMinute: 789,
+ },
+ compare: {
+ minConcurrency: 123,
+ maxConcurrency: 456,
+ maxTasksPerMinute: 789,
+ },
+ };
+
+ const autoscaledPoolOptions = {
+ minConcurrency: 16,
+ maxConcurrency: 32,
+ maxTasksPerMinute: 64,
+ };
+
+ const requestList = await RequestList.open(null, []);
+ const requestHandler = async () => {};
+
+ await (new BasicCrawler({
+ requestList,
+ requestHandler,
+ ...shorthandOptions.options,
+ })).run();
+
+ expect((AutoscaledPool as any).mock.calls[0][0]).toMatchObject(shorthandOptions.compare);
+
+ await (new BasicCrawler({
+ requestList,
+ requestHandler,
+ autoscaledPoolOptions,
+ })).run();
+
+ expect((AutoscaledPool as any).mock.calls[1][0]).toMatchObject(autoscaledPoolOptions);
+
+ await (new BasicCrawler({
+ requestList,
+ requestHandler,
+ ...shorthandOptions.options,
+ autoscaledPoolOptions,
+ })).run();
+
+ expect((AutoscaledPool as any).mock.calls[2][0]).toMatchObject(shorthandOptions.compare);
+ });
+
test('auto-saved state object', async () => {
const sources = [...Array(50).keys()].map((index) => ({ url: `https://example.com/${index}` }));
const sourcesCopy = JSON.parse(JSON.stringify(sources));