Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: concurrency option override order (#1649)
Jindřich Bär7bbad0380cd6
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 5606ae4a7..d9f9d332f 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -533,8 +533,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext}const basicCrawlerAutoscaledPoolConfiguration: Partial<AutoscaledPoolOptions> = {- minConcurrency,- maxConcurrency,+ minConcurrency: minConcurrency ?? autoscaledPoolOptions?.minConcurrency,+ maxConcurrency: maxConcurrency ?? autoscaledPoolOptions?.maxConcurrency,maxTasksPerMinute: maxRequestsPerMinute ?? autoscaledPoolOptions?.maxTasksPerMinute,runTaskFunction: this._runTaskFunction.bind(this),isTaskReadyFunction: async () => {diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex b44b4e5fa..6e831957f 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -18,12 +18,24 @@ import {CriticalError,MissingRouteError,} from '@crawlee/basic';+import {+ AutoscaledPool,+} from '@crawlee/core';import express from 'express';import type { Dictionary } from '@crawlee/utils';import { sleep } from '@crawlee/utils';import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';import { startExpressAppPromise } from '../../shared/_helper';+jest.mock('@crawlee/core', () => {+ const originalModule = jest.requireActual('@crawlee/core');+ const AutoscaledPoolMockConstructor = jest.fn((...args) => new originalModule.AutoscaledPool(...args));+ return {+ ...originalModule,+ AutoscaledPool: AutoscaledPoolMockConstructor,+ };+});+describe('BasicCrawler', () => {let logLevel: number;const localStorageEmulator = new MemoryStorageEmulator();@@ -91,6 +103,55 @@ describe('BasicCrawler', () => {expect(await requestList.isEmpty()).toBe(true);});+ test('should correctly combine shorthand and full length options', async () => {+ const shorthandOptions = {+ options: {+ minConcurrency: 123,+ maxConcurrency: 456,+ maxRequestsPerMinute: 789,+ },+ compare: {+ minConcurrency: 123,+ maxConcurrency: 456,+ maxTasksPerMinute: 789,+ },+ };++ const autoscaledPoolOptions = {+ minConcurrency: 16,+ maxConcurrency: 32,+ maxTasksPerMinute: 64,+ };++ const requestList = await RequestList.open(null, []);+ const requestHandler = async () => {};++ await (new BasicCrawler({+ requestList,+ requestHandler,+ ...shorthandOptions.options,+ })).run();++ expect((AutoscaledPool as any).mock.calls[0][0]).toMatchObject(shorthandOptions.compare);++ await (new BasicCrawler({+ requestList,+ requestHandler,+ autoscaledPoolOptions,+ })).run();++ expect((AutoscaledPool as any).mock.calls[1][0]).toMatchObject(autoscaledPoolOptions);++ await (new BasicCrawler({+ requestList,+ requestHandler,+ ...shorthandOptions.options,+ autoscaledPoolOptions,+ })).run();++ expect((AutoscaledPool as any).mock.calls[2][0]).toMatchObject(shorthandOptions.compare);+ });+test('auto-saved state object', async () => {const sources = [...Array(50).keys()].map((index) => ({ url: `https://example.com/${index}` }));const sourcesCopy = JSON.parse(JSON.stringify(sources));