Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor!: Make AutoscaledPool internal (#3947)
Jan Buchar83451f216e0d
diff --git a/docs/examples/cheerio_crawler.ts b/docs/examples/cheerio_crawler.tsindex 7a308a6f0..d32dc858c 100644--- a/docs/examples/cheerio_crawler.ts+++ b/docs/examples/cheerio_crawler.ts@@ -9,7 +9,7 @@ log.setLevel(LogLevel.DEBUG);// that automatically loads the URLs and parses their HTML using the cheerio library.const crawler = new CheerioCrawler({// The crawler downloads and processes the web pages in parallel, with a concurrency- // automatically managed based on the available system memory and CPU (see AutoscaledPool class).+ // automatically managed based on the available system memory and CPU (see ConcurrencySystem class).// Here we define some hard limits for the concurrency.minConcurrency: 10,maxConcurrency: 50,diff --git a/docs/examples/http_crawler.ts b/docs/examples/http_crawler.tsindex add990c59..7f661e116 100644--- a/docs/examples/http_crawler.ts+++ b/docs/examples/http_crawler.ts@@ -9,7 +9,7 @@ log.setLevel(LogLevel.DEBUG);// that automatically loads the URLs and saves their HTML.const crawler = new HttpCrawler({// The crawler downloads and processes the web pages in parallel, with a concurrency- // automatically managed based on the available system memory and CPU (see AutoscaledPool class).+ // automatically managed based on the available system memory and CPU (see ConcurrencySystem class).// Here we define some hard limits for the concurrency.minConcurrency: 10,maxConcurrency: 50,diff --git a/docs/examples/jsdom_crawler.ts b/docs/examples/jsdom_crawler.tsindex 6db02c4ef..ed5b8a8cf 100644--- a/docs/examples/jsdom_crawler.ts+++ b/docs/examples/jsdom_crawler.ts@@ -9,7 +9,7 @@ log.setLevel(LogLevel.DEBUG);// that automatically loads the URLs and parses their HTML using the jsdom library.const crawler = new JSDOMCrawler({// The crawler downloads and processes the web pages in parallel, with a concurrency- // automatically managed based on the available system memory and CPU (see AutoscaledPool class).+ // automatically managed based on the available system memory and CPU (see ConcurrencySystem class).// Here we define some hard limits for the concurrency.minConcurrency: 10,maxConcurrency: 50,diff --git a/docs/guides/configuration.mdx b/docs/guides/configuration.mdxindex 07bd5ac80..c51281dfe 100644--- a/docs/guides/configuration.mdx+++ b/docs/guides/configuration.mdx@@ -126,8 +126,8 @@ Enables verbose logging if set to `true`. If not explicitly set to `true` - for#### `CRAWLEE_MEMORY_MBYTES`-Sets the amount of system memory in megabytes to be used by the <ApiLink to="core/class/AutoscaledPool">`AutoscaledPool`</ApiLink>.-It is used to limit the number of concurrently running tasks. By default, the max amount of memory+Sets the amount of system memory in megabytes to be used by the <ApiLink to="core/class/ConcurrencySystem">`ConcurrencySystem`</ApiLink>.+It is used to limit the number of concurrently running requests. By default, the max amount of memoryto be used is set to one quarter of total system memory, i.e. on a system with 8192 MB of memory,the autoscaling feature will only use up to 2048 MB of memory.diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.mdindex 5957bd99f..4993b0cc4 100644--- a/docs/public-api/crawlee-basic.api.md+++ b/docs/public-api/crawlee-basic.api.md@@ -8,8 +8,6 @@ import type { AddRequestsBatchedOptions } from '@crawlee/core';import type { AddRequestsBatchedResult } from '@crawlee/core';import { AnyPredicate } from 'ow';import { ArrayPredicate } from 'ow';-import { AutoscaledPool } from '@crawlee/core';-import type { AutoscaledPoolPredicateOptions } from '@crawlee/core';import type { Awaitable } from '@crawlee/types';import type { BaseHttpClient } from '@crawlee/types';import { BasePredicate } from 'ow';@@ -59,6 +57,7 @@ import type { StatisticState } from '@crawlee/core';import type { StorageBackend } from '@crawlee/types';import type { StorageIdentifier } from '@crawlee/core';import { StringPredicate } from 'ow';+import type { TaskLoopPredicates } from '@crawlee/core';import { TimeoutError } from '@apify/timeout';import type { TypedRequestsLike } from '@crawlee/core';@@ -68,7 +67,6 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con// (undocumented)protected readonly additionalHttpErrorStatusCodes: Set<number>;addRequests(requests: ReadonlyDeep<TypedRequestsLike<Routes>>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;- autoscaledPool?: AutoscaledPool;get basicContextPipeline(): ContextPipeline<{request: Request_2;}, CrawlingContext>;@@ -77,6 +75,7 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Conprotected buildContextPipeline(): ContextPipeline<CrawlingContext, CrawlingContext>;// (undocumented)protected calculateEnqueuedRequestLimit(explicitLimit?: number): Promise<number | undefined>;+ get concurrencySystem(): IConcurrencySystem | undefined;// (undocumented)get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;// (undocumented)@@ -136,7 +135,7 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ConsameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;- autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;+ taskLoopOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;concurrencySystem: ObjectPredicate<object> & BasePredicate<object | undefined>;sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;@@ -160,11 +159,13 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ConstatisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;id: StringPredicate & BasePredicate<string | undefined>;};+ pause(timeoutSecs?: number): Promise<void>;readonly proxyConfiguration?: IProxyConfiguration;pushData(data: Parameters<Dataset['pushData']>[0], datasetIdentifier?: string | StorageIdentifier): Promise<void>;// (undocumented)protected readonly requestHandler: RequestHandler<ExtendedContext>;protected requestManager?: IRequestManager;+ resume(): void;// (undocumented)protected readonly retryOnBlocked: boolean;readonly router: RouterHandler<Context, Routes>;@@ -186,7 +187,6 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con// @public (undocumented)export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {additionalHttpErrorStatusCodes?: number[];- autoscaledPoolOptions?: AutoscaledPoolPredicateOptions;blockedStatusCodes?: number[];concurrencySystem?: IConcurrencySystem;configuration?: Configuration;@@ -225,6 +225,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingCstatusMessageCallback?: StatusMessageCallback;statusMessageLoggingInterval?: number;storageBackend?: StorageBackend;+ taskLoopOptions?: TaskLoopPredicates;}// @public (undocumented)diff --git a/docs/public-api/crawlee-browser.api.md b/docs/public-api/crawlee-browser.api.mdindex d8e93691a..2e82b9f38 100644--- a/docs/public-api/crawlee-browser.api.md+++ b/docs/public-api/crawlee-browser.api.md@@ -84,7 +84,7 @@ export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, ResposameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;- autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;+ taskLoopOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;concurrencySystem: ObjectPredicate<object> & BasePredicate<object | undefined>;sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.mdindex 6fadfca49..279f55f55 100644--- a/docs/public-api/crawlee-core.api.md+++ b/docs/public-api/crawlee-core.api.md@@ -79,36 +79,6 @@ export class ApifyLogAdapter extends BaseCrawleeLogger {logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;}-// @public-export class AutoscaledPool {- constructor(options: AutoscaledPoolOptions);- abort(): Promise<void>;- get currentConcurrency(): number;- get desiredConcurrency(): number;- notify(): Promise<void>;- pause(timeoutSecs?: number): Promise<void>;- resume(): void;- run(): Promise<void>;- get system(): IConcurrencySystem;-}--// @public (undocumented)-export interface AutoscaledPoolOptions extends AutoscaledPoolPredicateOptions {- concurrencySystem: IConcurrencySystem;- consumer: ConcurrencyConsumer;- // (undocumented)- log?: CrawleeLogger;- maybeRunIntervalSecs?: number;- runTaskFunction?: () => Promise<unknown>;- taskTimeoutSecs?: number;-}--// @public-export interface AutoscaledPoolPredicateOptions {- isFinishedFunction?: () => Promise<boolean>;- isTaskReadyFunction?: () => Promise<boolean>;-}-// @publicexport abstract class BaseCrawleeLogger implements CrawleeLogger {constructor(options?: Partial<CrawleeLoggerOptions>);@@ -1763,6 +1733,12 @@ export interface SystemInfo {memTotalBytes?: number;}+// @public+export interface TaskLoopPredicates {+ isFinishedFunction?: () => Promise<boolean>;+ isTaskReadyFunction?: () => Promise<boolean>;+}+export { tryAbsoluteURL }// @public (undocumented)diff --git a/docs/public-api/crawlee-http.api.md b/docs/public-api/crawlee-http.api.mdindex ec72d8014..204dffa49 100644--- a/docs/public-api/crawlee-http.api.md+++ b/docs/public-api/crawlee-http.api.md@@ -126,7 +126,7 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> =sameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;- autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;+ taskLoopOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;concurrencySystem: ObjectPredicate<object> & BasePredicate<object | undefined>;sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;diff --git a/docs/public-api/crawlee-jsdom.api.md b/docs/public-api/crawlee-jsdom.api.mdindex 671aded46..adf0bb80b 100644--- a/docs/public-api/crawlee-jsdom.api.md+++ b/docs/public-api/crawlee-jsdom.api.md@@ -83,7 +83,7 @@ export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContextsameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;- autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;+ taskLoopOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;concurrencySystem: ObjectPredicate<object> & BasePredicate<object | undefined>;sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.mdindex a93237990..04bb34324 100644--- a/docs/public-api/crawlee-playwright.api.md+++ b/docs/public-api/crawlee-playwright.api.md@@ -306,7 +306,7 @@ export class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedConsameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;- autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;+ taskLoopOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;concurrencySystem: ObjectPredicate<object> & BasePredicate<object | undefined>;sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;diff --git a/docs/public-api/crawlee-puppeteer.api.md b/docs/public-api/crawlee-puppeteer.api.mdindex 216462bb5..c0eeb3ed4 100644--- a/docs/public-api/crawlee-puppeteer.api.md+++ b/docs/public-api/crawlee-puppeteer.api.md@@ -213,7 +213,7 @@ export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContsameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;- autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;+ taskLoopOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;concurrencySystem: ObjectPredicate<object> & BasePredicate<object | undefined>;sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;diff --git a/docs/public-api/crawlee-stagehand.api.md b/docs/public-api/crawlee-stagehand.api.mdindex 4b8379989..2c6436728 100644--- a/docs/public-api/crawlee-stagehand.api.md+++ b/docs/public-api/crawlee-stagehand.api.md@@ -110,7 +110,7 @@ export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedContsameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;- autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;+ taskLoopOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;concurrencySystem: ObjectPredicate<object> & BasePredicate<object | undefined>;sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex 09dbef000..adc0391c0 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -50,9 +50,9 @@ This is intentional: these were never a supported API. If you relied on overridiThe change spans, among others:-- **`BasicCrawler`** — `unexpectedStop`, `requestHandlerTimeoutMillis`, `sameDomainDelayMillis`, `domainAccessedTime`, `handledRequestsCount`, `statusMessageLoggingInterval`, `statusMessageCallback`, `ignoreHttpErrorStatusCodes`, `autoscaledPoolOptions`, `respectRobotsTxtFile`, and the helpers `buildBasicContextPipeline`, `validateRequestUserData`, `pauseOnMigration`, `fetchNextRequest`, `delayRequest`, `handleRequest`, `timeoutAndRetry`, `isTaskReadyFunction`, `defaultIsFinishedFunction`, `requestFunctionErrorHandler`, `handleFailedRequestHandler`, `canRequestBeRetried`+- **`BasicCrawler`** — `unexpectedStop`, `requestHandlerTimeoutMillis`, `sameDomainDelayMillis`, `domainAccessedTime`, `handledRequestsCount`, `statusMessageLoggingInterval`, `statusMessageCallback`, `ignoreHttpErrorStatusCodes`, `taskLoopOptions` (was `autoscaledPoolOptions`), `autoscaledPool`, `respectRobotsTxtFile`, and the helpers `buildBasicContextPipeline`, `validateRequestUserData`, `pauseOnMigration`, `fetchNextRequest`, `delayRequest`, `handleRequest`, `timeoutAndRetry`, `isTaskReadyFunction`, `defaultIsFinishedFunction`, `requestFunctionErrorHandler`, `handleFailedRequestHandler`, `canRequestBeRetried`- **`HttpCrawler`** — `preNavigationHooks`, `postNavigationHooks`, `saveResponseCookies`, `navigationTimeoutMillis`, `ignoreSslErrors`, `suggestResponseEncoding`, `forceResponseEncoding`, `supportedMimeTypes`, and the helpers `requestFunction`, `parseResponse`, `getRequestOptions`, `encodeResponse`, `extendSupportedMimeTypes`, `handleRequestTimeout`-- **`AutoscaledPool`** — `maybeRunTask`, `maybeFinish`, `destroy` (the scaling internals `autoscale`, `scaleUp`, `scaleDown`, `incrementTasksDonePerSecond` and `isOverMaxRequestLimit` are private too, but they now live on the `ConcurrencySystem` the pool delegates to — see [below](#autoscaling-split-into-autoscaledpool--concurrencysystem))+- **`AutoscaledPool`** — the whole class is `@internal` in v4, so its members are not enumerated here; see [`AutoscaledPool` is no longer public API](#autoscaledpool-is-no-longer-public-api)- **`SessionPool`** — all pool internals (`log`, `maxPoolSize`, `createSessionFunction`, `keyValueStore`, `sessions`, `sessionMap`, `sessionOptions`, `persistStateKey`, `persistStateKeyValueStoreId`, `events`, `persistenceOptions`, `sessionReuseStrategy`, and the helpers `ensureInitialized`, `maybeLoadSessionPool`, `registerSession`, `createSession`, `hasSpaceForSession`, `pickSession`, `removeRetiredSessions`, `getRandomIndex`, `defaultCreateSessionFunction`)- **`Session`** — `maybeSelfRetire` (`userData` is now `readonly`)- **`RequestList`** — all `_`-prefixed helpers (`addFetchedRequests`, `addPersistedRequests`, `addRequest`, `addRequestsFromSources`, `ensureInProgress`, `ensureIsInitialized`, `ensureUniqueKeyValid`, `fetchRequestsFromUrl`, `getPersistedState`, `loadStateAndPersistedRequests`, `persistRequests`, `restoreState`)@@ -1225,17 +1225,17 @@ If you rely on Crawlee's default configuration (one browser context per session,## The `log` property is typed as `CrawleeLogger`-The `log` property exposed throughout the public API (on the crawling context, `AutoscaledPool`, `Statistics`, `EventManager`, `SessionOptions`, `Dataset`, etc.) is now typed as the `CrawleeLogger` interface (from `@crawlee/types`) rather than the concrete `Log` class from `@apify/log`. If you consume it structurally — calling `log.info(...)`, `log.debug(...)`, `log.child(...)` — nothing changes. You only need to act if you explicitly annotated a variable or parameter with the `Log` type from `@apify/log` and assigned `context.log` to it; type it as `CrawleeLogger` instead.+The `log` property exposed throughout the public API (on the crawling context, `Statistics`, `EventManager`, `SessionOptions`, `Dataset`, etc.) is now typed as the `CrawleeLogger` interface (from `@crawlee/types`) rather than the concrete `Log` class from `@apify/log`. If you consume it structurally — calling `log.info(...)`, `log.debug(...)`, `log.child(...)` — nothing changes. You only need to act if you explicitly annotated a variable or parameter with the `Log` type from `@apify/log` and assigned `context.log` to it; type it as `CrawleeLogger` instead.-## Autoscaling split into `AutoscaledPool` + `ConcurrencySystem`+## Autoscaling moved to `ConcurrencySystem`-Everything that decides whether there is free compute for one more task — snapshotting, the system-status evaluation, the concurrency budget and the scaling logic — moved out of `AutoscaledPool` into a new `ConcurrencySystem`. The pool keeps only the task loop.+Everything that decides whether there is free compute for one more task — snapshotting, the system-status evaluation, the concurrency budget and the scaling logic — moved out of `AutoscaledPool` into a new `ConcurrencySystem`. What is left of the pool is a bare task loop, which is now **internal**: crawlers build and drive one for themselves, and `ConcurrencySystem` is the only part of this you configure.-The point is sharing: inject one instance into several pools (and therefore several crawlers) to cap their **combined** concurrency against a single budget, instead of letting each scale independently and oversubscribe the host. The option is typed as `IConcurrencySystem`, a minimal read-only contract, so an alternative governor can be substituted; `ConcurrencySystem` is the canonical implementation and the one crawlers build by default.+The point is sharing: inject one instance into several crawlers to cap their **combined** concurrency against a single budget, instead of letting each scale independently and oversubscribe the host. The option is typed as `IConcurrencySystem`, a minimal read-only contract, so an alternative governor can be substituted; `ConcurrencySystem` is the canonical implementation and the one crawlers build by default.:::info Who starts and stops it-Whoever *builds* a `ConcurrencySystem` owns its lifecycle. Crawlers do that for the default system they build per run. One **you** supply is yours to `start()` and `stop()`, because no crawler can know when the last borrower has finished. Forgetting to start it makes `run()` **throw**; stopping it while a pool is still running only **warns**. Read `isRunning` to check whether a system handed to you was already started by its owner.+Whoever *builds* a `ConcurrencySystem` owns its lifecycle. Crawlers do that for the default system they build per run. One **you** supply is yours to `start()` and `stop()`, because no crawler can know when the last borrower has finished. Forgetting to start it makes `run()` **throw**; stopping it while a crawler is still running only **warns**. Read `isRunning` to check whether a system handed to you was already started by its owner.:::@@ -1258,11 +1258,50 @@ try {}```-The `AutoscaledPool` migration below spells it out once more at the pool level; the crawler examples elide it, to keep the actual change in focus.+### `AutoscaledPool` is no longer public API-### `AutoscaledPool` requires a `concurrencySystem` and a `consumer`, and no longer takes scaling options+`AutoscaledPool` is `@internal` in v4, along with `AutoscaledPoolOptions`. It is still exported from `@crawlee/core` (and re-exported by `crawlee`), so nothing breaks at import time — but with all the configuration moved to the `ConcurrencySystem`, what remains is a bare parallel task runner, and a scraping library has no business promising backwards compatibility for one. Reach for it if you must; it can change without a major bump, and if you only wanted bounded parallelism, a `p-limit`-style helper is a better fit than an internal Crawlee class.-All scaling and load-monitoring options were **removed** from `AutoscaledPoolOptions` and now live on `ConcurrencySystemOptions`: `minConcurrency`, `maxConcurrency`, `desiredConcurrency`, `desiredConcurrencyRatio`, `scaleUpStepRatio`, `scaleDownStepRatio`, `loggingIntervalSecs`, `autoscaleIntervalSecs`, and `maxTasksPerMinute`, plus the load-signal configuration described [below](#load-signal-options-restructured). The `snapshotterOptions` and `systemStatusOptions` bags are both gone. In their place, `AutoscaledPoolOptions` gained a **required** `concurrencySystem`, plus a **required** `consumer` — the pool's identity (`{ id }`), which it presents to the governor on every capacity query and booking so that a shared one can tell several pools' tasks apart. Crawlers pass their own identity, so this only concerns pools you build yourself.+The crawler's `autoscaledPool` property is **private** as a result. Everything it was reached for has a crawler-level counterpart:++| Before | After |+|---|---|+| `await crawler.autoscaledPool.pause(secs)` | `await crawler.pause(secs)` |+| `crawler.autoscaledPool.resume()` | `crawler.resume()` |+| `await crawler.autoscaledPool.abort()` | `await crawler.teardown()` — or `crawler.stop()`, to let in-flight requests finish |+| `crawler.autoscaledPool.system` | `crawler.concurrencySystem` |+| `crawler.autoscaledPool.desiredConcurrency` | `crawler.concurrencySystem?.desiredConcurrency` |+| `crawler.autoscaledPool.currentConcurrency` | `crawler.concurrencySystem?.currentConcurrency` |++```typescript+const crawler = new CheerioCrawler({+ async requestHandler({ crawler, log }) {+ log.info(`Currently running ${crawler.concurrencySystem?.currentConcurrency} requests in parallel.`);+ },+});+```++`crawler.concurrencySystem` is `undefined` until `run()` has resolved it, and a crawler-owned default is rebuilt for every run — so read it during a run rather than caching it across runs. A system *you* injected is of course simply the instance you passed in.++The getter is typed as the read-only `IConcurrencySystem`, and the pool never had setters to begin with in v4, so retuning concurrency mid-crawl means owning the instance:++```typescript+const concurrencySystem = new ConcurrencySystem({ maxConcurrency: 50 });++const crawler = new CheerioCrawler({+ concurrencySystem,+ requestHandler,+ errorHandler: async ({ response }) => {+ if (response?.status === 429) concurrencySystem.maxConcurrency = 10;+ },+});+```++`crawler.pause()` resolves once the requests already in flight have settled, and leaves `run()` pending until you `resume()` — unlike `crawler.stop()`, which ends the run gracefully. One behavioral consequence of the split: pausing no longer suspends autoscaling, because the autoscaling interval belongs to the `ConcurrencySystem`, which knows nothing about its borrowers' pause state — deliberately, since other crawlers sharing it may still need scaling. A paused crawler's system keeps evaluating (and possibly scaling down) the desired concurrency and keeps emitting its periodic state log. Scaling *up* stays effectively blocked, as the current concurrency drains below the ratio required for a scale-up. To silence the system during a long pause, `stop()` it (if you own it) and `start()` it again before resuming; a restart discards the snapshots taken before it, so the pause is not mistaken for load.++#### If you were driving an `AutoscaledPool` directly++All scaling and load-monitoring options were **removed** from `AutoscaledPoolOptions` and now live on `ConcurrencySystemOptions`: `minConcurrency`, `maxConcurrency`, `desiredConcurrency`, `desiredConcurrencyRatio`, `scaleUpStepRatio`, `scaleDownStepRatio`, `loggingIntervalSecs`, `autoscaleIntervalSecs`, and `maxTasksPerMinute`, plus the load-signal configuration described [below](#load-signal-options-restructured). The `snapshotterOptions` and `systemStatusOptions` bags are both gone, as are the `minConcurrency`/`maxConcurrency` accessors and every setter (`desiredConcurrency`, `currentConcurrency` and `system` remain as read-only getters). In their place, `AutoscaledPoolOptions` gained a **required** `concurrencySystem`, plus a **required** `consumer` — the pool's identity (`{ id }`), which it presents to the governor on every capacity query and booking so that a shared one can tell several pools' tasks apart.**Before:**```typescript@@ -1303,27 +1342,11 @@ try {}```-The pool's concurrency accessors are now read-only telemetry: `desiredConcurrency` and `currentConcurrency` remain as getters, while the `minConcurrency`/`maxConcurrency` accessors and every setter were **removed**. Runtime tuning happens on the `ConcurrencySystem` instead — which means building it yourself, since the system a crawler builds for itself is not reachable for reconfiguration:--```typescript-const concurrencySystem = new ConcurrencySystem({ maxConcurrency: 50 });--const crawler = new CheerioCrawler({- concurrencySystem,- requestHandler,- errorHandler: async ({ response }) => {- if (response?.status === 429) concurrencySystem.maxConcurrency = 10;- },-});-```--One behavioral consequence: `pool.pause()` no longer suspends autoscaling, because the autoscaling interval belongs to the `ConcurrencySystem`, which knows nothing about its borrowers' pause state — deliberately, since other pools sharing it may still need scaling. A paused pool's system keeps evaluating (and possibly scaling down) the desired concurrency and keeps emitting its periodic state log. Scaling *up* stays effectively blocked, as the current concurrency drains below the ratio required for a scale-up. To silence the system during a long pause, `stop()` it (if you own it) and `start()` it again before resuming; a restart discards the snapshots taken before it, so the pause is not mistaken for load.--### `BasicCrawlerOptions.autoscaledPoolOptions` no longer carries concurrency config+### `autoscaledPoolOptions` is now `taskLoopOptions`, and no longer carries concurrency config-`autoscaledPoolOptions` now accepts **only** the task-loop predicates `isFinishedFunction` and `isTaskReadyFunction` (its type changed from `AutoscaledPoolOptions` to `AutoscaledPoolPredicateOptions`). Concurrency configuration goes through either the `minConcurrency` / `maxConcurrency` / `maxRequestsPerMinute` shortcuts (which configure the crawler's default `ConcurrencySystem`), or — for anything finer — a supplied `concurrencySystem`.+The crawler option was renamed — it was named after a class that is now internal — and narrowed to **only** the task-loop predicates `isFinishedFunction` and `isTaskReadyFunction`. Its type changed from `AutoscaledPoolOptions` to `TaskLoopPredicates` (itself a rename of the interim `AutoscaledPoolPredicateOptions`). Concurrency configuration goes through either the `minConcurrency` / `maxConcurrency` / `maxRequestsPerMinute` shortcuts (which configure the crawler's default `ConcurrencySystem`), or — for anything finer — a supplied `concurrencySystem`.-Note that this narrowing also drops three options that did *not* move to the `ConcurrencySystem`: `maybeRunIntervalSecs`, `taskTimeoutSecs` and `log` are still `AutoscaledPoolOptions`, but a crawler no longer forwards them, so they can only be set by driving an `AutoscaledPool` yourself. If you were tuning the crawler's task-loop cadence through `autoscaledPoolOptions`, there is no replacement.+Note that this narrowing also drops three options that did *not* move to the `ConcurrencySystem`: `maybeRunIntervalSecs`, `taskTimeoutSecs` and `log` still exist on `AutoscaledPoolOptions`, but a crawler never forwarded them and the type is internal now, so they are effectively unreachable. If you were tuning the crawler's task-loop cadence through `autoscaledPoolOptions`, there is no replacement.**Before:**```typescriptdiff --git a/packages/basic-crawler/README.md b/packages/basic-crawler/README.mdindex cf8a445a2..31e24e544 100644--- a/packages/basic-crawler/README.md+++ b/packages/basic-crawler/README.md@@ -11,7 +11,7 @@ If both [`requestList`](https://crawlee.dev/js/api/basic-crawler/interface/BasicThe crawler finishes if there are no more [Request](https://crawlee.dev/js/api/core/class/Request) objects to crawl.-New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the [AutoscaledPool](https://crawlee.dev/js/api/core/class/AutoscaledPool) class. Concurrency is tuned via the [`minConcurrency`](https://crawlee.dev/js/api/basic-crawler/interface/BasicCrawlerOptions#minConcurrency), [`maxConcurrency`](https://crawlee.dev/js/api/basic-crawler/interface/BasicCrawlerOptions#maxConcurrency) and [`maxRequestsPerMinute`](https://crawlee.dev/js/api/basic-crawler/interface/BasicCrawlerOptions#maxRequestsPerMinute) options of the `BasicCrawler` constructor, or, for finer control, by injecting a pre-configured [`concurrencySystem`](https://crawlee.dev/js/api/basic-crawler/interface/BasicCrawlerOptions#concurrencySystem) — see `ConcurrencySystem`.+New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's `ConcurrencySystem`. Concurrency is tuned via the [`minConcurrency`](https://crawlee.dev/js/api/basic-crawler/interface/BasicCrawlerOptions#minConcurrency), [`maxConcurrency`](https://crawlee.dev/js/api/basic-crawler/interface/BasicCrawlerOptions#maxConcurrency) and [`maxRequestsPerMinute`](https://crawlee.dev/js/api/basic-crawler/interface/BasicCrawlerOptions#maxRequestsPerMinute) options of the `BasicCrawler` constructor, or, for finer control, by injecting a pre-configured [`concurrencySystem`](https://crawlee.dev/js/api/basic-crawler/interface/BasicCrawlerOptions#concurrencySystem).## Example usagediff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex a5c9186db..d0027ff9e 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -5,7 +5,6 @@ import type {AddRequestsBatchedOptions,AddRequestsBatchedResult,AutoscaledPoolOptions,- AutoscaledPoolPredicateOptions,ConcurrencySystemOptions,Configuration,CrawleeLogger,@@ -30,6 +29,7 @@ import type {StatisticsOptions,StatisticState,StorageIdentifier,+ TaskLoopPredicates,TypedRequestsLike,} from '@crawlee/core';import {@@ -311,13 +311,14 @@ export interface BasicCrawlerOptions<maxCrawlDepth?: number;/**- * Lets you override the task-loop predicates (`isFinishedFunction`, `isTaskReadyFunction`) of the crawler's- * underlying {@apilink AutoscaledPool}. The `runTaskFunction` is owned by the crawler and cannot be overridden.+ * Lets you override the predicates that steer the crawler's task loop: `isTaskReadyFunction` (may another request+ * start?) and `isFinishedFunction` (is the crawl over?). The task itself — fetching a request and running it+ * through the pipeline — is owned by the crawler and cannot be overridden.** Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`* shortcuts, or a {@apilink BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.*/- autoscaledPoolOptions?: AutoscaledPoolPredicateOptions;+ taskLoopOptions?: TaskLoopPredicates;/*** A pre-configured concurrency governor — the component that decides whether there is free compute for one more@@ -537,8 +538,8 @@ export interface BasicCrawlerOptions<** The crawler finishes if there are no more {@apilink Request} objects to crawl.*- * New requests are only dispatched when there is enough free CPU and memory available,- * using the functionality provided by the {@apilink AutoscaledPool} class.+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's+ * {@apilink ConcurrencySystem}.* Concurrency is tuned via the {@apilink BasicCrawlerOptions.minConcurrency|`minConcurrency`},* {@apilink BasicCrawlerOptions.maxConcurrency|`maxConcurrency`} and* {@apilink BasicCrawlerOptions.maxRequestsPerMinute|`maxRequestsPerMinute`} shortcuts, or, for finer control, by@@ -661,16 +662,29 @@ export class BasicCrawler<private concurrencySystemDep?: OwnedOrInjected<IConcurrencySystem, ConcurrencySystem>;/**- * A reference to the underlying {@apilink AutoscaledPool} class that runs the crawler's task loop.- * > *NOTE:* This property is only initialized after calling the {@apilink BasicCrawler.run|`crawler.run()`} function.- * We can use it to pause the crawler by calling {@apilink AutoscaledPool.pause|`autoscaledPool.pause()`}- * or to abort it by calling {@apilink AutoscaledPool.abort|`autoscaledPool.abort()`}.+ * The concurrency governor this run is booking its requests against — either the+ * {@apilink BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the+ * crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.*- * The pool only exposes read-only concurrency telemetry. To tune concurrency at runtime, keep a reference to a- * {@apilink ConcurrencySystem} and inject it via- * {@apilink BasicCrawlerOptions.concurrencySystem|`concurrencySystem`}.+ * > *NOTE:* `undefined` until {@apilink BasicCrawler.run|`crawler.run()`} has resolved it. A crawler-owned default+ * is also rebuilt for every run, so the instance is not stable across runs.+ *+ * {@apilink IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means+ * owning the instance: build a {@apilink ConcurrencySystem} yourself and inject it, then set+ * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.+ */+ get concurrencySystem(): IConcurrencySystem | undefined {+ return this.concurrencySystemDep?.maybeValue;+ }++ /**+ * The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with+ * no configuration left of its own (see {@apilink ConcurrencySystem}), and everything a caller legitimately did+ * with it now has a crawler-level counterpart: {@apilink BasicCrawler.pause|`pause()`},+ * {@apilink BasicCrawler.resume|`resume()`}, {@apilink BasicCrawler.teardown|`teardown()`} and+ * {@apilink BasicCrawler.concurrencySystem|`concurrencySystem`}.*/- autoscaledPool?: AutoscaledPool;+ private autoscaledPool?: AutoscaledPool;/*** A reference to the underlying {@apilink IProxyConfiguration} instance that manages the crawler's proxies.@@ -746,12 +760,12 @@ export class BasicCrawler<protected readonly additionalHttpErrorStatusCodes: Set<number>;private ignoreHttpErrorStatusCodes: Set<number>;/**- * The resolved task-loop options for the crawler's own {@apilink AutoscaledPool} — the crawler-owned- * `runTaskFunction`, the (possibly user-overridden) ready/finished predicates and cadence/logging. Concurrency- * configuration lives on the {@apilink ConcurrencySystem} instead, and the pool's `consumer` identity is the- * crawler's own, so neither is settable here.+ * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly+ * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the+ * {@apilink ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is+ * settable here.*/- private autoscaledPoolOptions: Omit<AutoscaledPoolOptions, 'concurrencySystem' | 'consumer'>;+ private taskLoopOptions: Omit<AutoscaledPoolOptions, 'concurrencySystem' | 'consumer'>;protected readonly httpClient: BaseHttpClient;protected readonly retryOnBlocked: boolean;private respectRobotsTxtFile: boolean | { userAgent?: string };@@ -782,7 +796,7 @@ export class BasicCrawler<sameDomainDelaySecs: ow.optional.number,maxRequestsPerCrawl: ow.optional.number,maxCrawlDepth: ow.optional.number,- autoscaledPoolOptions: ow.optional.object,+ taskLoopOptions: ow.optional.object,concurrencySystem: ow.optional.object,sessionPool: ow.optional.object.validate(validators.sessionPool),proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),@@ -834,7 +848,7 @@ export class BasicCrawler<sameDomainDelaySecs = 0,maxRequestsPerCrawl,maxCrawlDepth,- autoscaledPoolOptions = {},+ taskLoopOptions = {},concurrencySystem,keepAlive,sessionPool,@@ -1019,14 +1033,14 @@ export class BasicCrawler<this.maxRequestsPerCrawl && this.maxRequestsPerCrawl <= this.handledRequestsCount;// eslint-disable-next-line prefer-const- let { isFinishedFunction, isTaskReadyFunction } = autoscaledPoolOptions;+ let { isFinishedFunction, isTaskReadyFunction } = taskLoopOptions;// override even if `isFinishedFunction` provided by user - `keepAlive` has higher priorityif (keepAlive) {isFinishedFunction = async () => false;}- const basicCrawlerAutoscaledPoolConfiguration: Partial<typeof this.autoscaledPoolOptions> = {+ const crawlerOwnedTaskLoopConfiguration: Partial<typeof this.taskLoopOptions> = {runTaskFunction: async () => {const source = this.requestManager;if (!source) throw new Error('Request provider is not initialized!');@@ -1151,7 +1165,7 @@ export class BasicCrawler<log: this.log,};- this.autoscaledPoolOptions = { ...autoscaledPoolOptions, ...basicCrawlerAutoscaledPoolConfiguration };+ this.taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };this.resolveConcurrencySystem = () =>OwnedOrInjected.resolve<IConcurrencySystem, ConcurrencySystem>(concurrencySystem, () =>@@ -1412,7 +1426,7 @@ export class BasicCrawler<const total = await this.requestManager?.getTotalCount();message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.stats.state.requestsFailed- } failed requests, desired concurrency ${this.autoscaledPool?.desiredConcurrency ?? 0}.`;+ } failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;}if (this.statusMessageCallback) {@@ -1433,8 +1447,9 @@ export class BasicCrawler<}/**- * Runs the crawler. Returns a promise that resolves once all the requests are processed- * and `autoscaledPool.isFinished` returns `true`.+ * Runs the crawler. Returns a promise that resolves once every request has been processed and the crawler's+ * finished-check ({@apilink BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`}, or the+ * default "the request manager is empty") reports that the crawl is over.** We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for* running {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} before {@apilink BasicCrawler.run|`crawler.run()`}.@@ -1593,6 +1608,36 @@ export class BasicCrawler<this.unexpectedStop = true;}+ /**+ * Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects+ * after `timeoutSecs` if they take too long. Unlike {@apilink BasicCrawler.stop|`stop()`}, this does not end the+ * run — {@apilink BasicCrawler.run|`run()`} stays pending until {@apilink BasicCrawler.resume|`resume()`}.+ *+ * > *NOTE:* The {@apilink BasicCrawler.concurrencySystem|concurrency system} keeps monitoring and autoscaling+ * throughout, since a shared one may still be serving other crawlers.+ */+ async pause(timeoutSecs?: number): Promise<void> {+ if (!this.autoscaledPool) {+ this.log.warning('Cannot pause a crawler that is not running.');+ return;+ }++ await this.autoscaledPool.pause(timeoutSecs);+ }++ /**+ * Resumes a run suspended with {@apilink BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests+ * again. A no-op on a crawler that is not paused.+ */+ resume(): void {+ if (!this.autoscaledPool) {+ this.log.warning('Cannot resume a crawler that is not running.');+ return;+ }++ this.autoscaledPool.resume();+ }+/*** Returns the crawler's {@apilink IRequestManager|request manager}, opening the default {@apilink RequestQueue}* if none has been configured or opened yet.@@ -1939,7 +1984,7 @@ export class BasicCrawler<await this.concurrencySystemDep.ifOwned((system) => system.start());this.autoscaledPool = new AutoscaledPool({- ...this.autoscaledPoolOptions,+ ...this.taskLoopOptions,concurrencySystem: this.concurrencySystemDep.value,consumer: this.identity,});diff --git a/packages/browser-crawler/README.md b/packages/browser-crawler/README.mdindex 03b632235..e98d2d89b 100644--- a/packages/browser-crawler/README.md+++ b/packages/browser-crawler/README.md@@ -12,7 +12,7 @@ The crawler finishes when there are no more [Request](https://crawlee.dev/js/api`BrowserCrawler` opens a new browser page (i.e. tab or window) for each [Request](https://crawlee.dev/js/api/core/class/Request) object to crawl and then calls the function provided by user as the [`requestHandler`](https://crawlee.dev/js/api/browser-crawler/interface/BrowserCrawlerOptions#requestHandler) option.-New pages are only opened when there is enough free CPU and memory available, using the functionality provided by the [AutoscaledPool](https://crawlee.dev/js/api/core/class/AutoscaledPool) class.-Concurrency is tuned via the [`minConcurrency`](https://crawlee.dev/js/api/browser-crawler/interface/BrowserCrawlerOptions#minConcurrency), [`maxConcurrency`](https://crawlee.dev/js/api/browser-crawler/interface/BrowserCrawlerOptions#maxConcurrency) and [`maxRequestsPerMinute`](https://crawlee.dev/js/api/browser-crawler/interface/BrowserCrawlerOptions#maxRequestsPerMinute) options of the `BrowserCrawler` constructor, or, for finer control, by injecting a pre-configured [`concurrencySystem`](https://crawlee.dev/js/api/browser-crawler/interface/BrowserCrawlerOptions#concurrencySystem) — see `ConcurrencySystem`.+New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's `ConcurrencySystem`.+Concurrency is tuned via the [`minConcurrency`](https://crawlee.dev/js/api/browser-crawler/interface/BrowserCrawlerOptions#minConcurrency), [`maxConcurrency`](https://crawlee.dev/js/api/browser-crawler/interface/BrowserCrawlerOptions#maxConcurrency) and [`maxRequestsPerMinute`](https://crawlee.dev/js/api/browser-crawler/interface/BrowserCrawlerOptions#maxRequestsPerMinute) options of the `BrowserCrawler` constructor, or, for finer control, by injecting a pre-configured [`concurrencySystem`](https://crawlee.dev/js/api/browser-crawler/interface/BrowserCrawlerOptions#concurrencySystem).> *NOTE:* the pool of browser instances is internally managed by the [BrowserPool](https://crawlee.dev/js/api/browser-pool/class/BrowserPool) class.diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 10831c08b..8d7dcc633 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -324,8 +324,8 @@ export interface BrowserCrawlerOptions<* `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@apilink Request} object to crawl* and then calls the function provided by user as the {@apilink BrowserCrawlerOptions.requestHandler|`requestHandler`} option.*- * New pages are only opened when there is enough free CPU and memory available,- * using the functionality provided by the {@apilink AutoscaledPool} class.+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's+ * {@apilink ConcurrencySystem}.* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the* `BrowserCrawler` constructor, or, for finer control, by injecting a pre-configured* {@apilink ConcurrencySystem|`concurrencySystem`}.diff --git a/packages/cheerio-crawler/README.md b/packages/cheerio-crawler/README.mdindex ea62f7cbb..11a2ec3c8 100644--- a/packages/cheerio-crawler/README.md+++ b/packages/cheerio-crawler/README.md@@ -24,7 +24,7 @@ preNavigationHooks: [By default, `CheerioCrawler` only processes web pages with the `text/html` and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header), and skips pages with other content types. If you want the crawler to process other content types, use the [CheerioCrawlerOptions.additionalMimeTypes](https://crawlee.dev/js/api/cheerio-crawler/interface/CheerioCrawlerOptions#additionalMimeTypes) constructor option. Beware that the parsing behavior differs for HTML, XML, JSON and other types of content. For more details, see [CheerioCrawlerOptions.requestHandler](https://crawlee.dev/js/api/cheerio-crawler/interface/CheerioCrawlerOptions#requestHandler).-New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the [AutoscaledPool](https://crawlee.dev/js/api/core/class/AutoscaledPool) class. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `CheerioCrawler` constructor, or, for finer control, by injecting a pre-configured `concurrencySystem` — see `ConcurrencySystem`.+New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's `ConcurrencySystem`. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `CheerioCrawler` constructor, or, for finer control, by injecting a pre-configured `concurrencySystem`.## Example usagediff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.tsindex e75fab875..665676c2a 100644--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts@@ -148,8 +148,8 @@ export type CheerioRequestHandler<* Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.* For more details, see {@apilink CheerioCrawlerOptions.requestHandler}.*- * New requests are only dispatched when there is enough free CPU and memory available,- * using the functionality provided by the {@apilink AutoscaledPool} class.+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's+ * {@apilink ConcurrencySystem}.* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the* `CheerioCrawler` constructor, or, for finer control, by injecting a pre-configured* {@apilink ConcurrencySystem|`concurrencySystem`}.diff --git a/packages/core/src/autoscaling/autoscaled_pool.ts b/packages/core/src/autoscaling/autoscaled_pool.tsindex 3bd842c6c..d86f61bce 100644--- a/packages/core/src/autoscaling/autoscaled_pool.ts+++ b/packages/core/src/autoscaling/autoscaled_pool.ts@@ -10,11 +10,10 @@ import type { CrawleeLogger } from '../log.js';import { serviceLocator } from '../service_locator.js';/**- * The task-readiness predicates a consumer may supply to steer an {@apilink AutoscaledPool}'s run loop — the parts of- * the loop a higher-level driver (e.g. a crawler) legitimately overrides, as opposed to the crawler-owned- * `runTaskFunction`.+ * The two predicates that steer a task loop: *is there work ready?* and *are we done?* These are the parts of the loop+ * a caller legitimately overrides, as opposed to the task itself (`runTaskFunction`), which the loop's driver owns.*/-export interface AutoscaledPoolPredicateOptions {+export interface TaskLoopPredicates {/*** A function that indicates whether `runTaskFunction` should be called.* This function is called every time there is free capacity for a new task and it should@@ -25,15 +24,15 @@ export interface AutoscaledPoolPredicateOptions {/*** A function that is called only when there are no tasks to be processed.- * If it resolves to `true` then the pool's run finishes. Being called only+ * If it resolves to `true` then the run finishes. Being called only* when there are no tasks being processed means that as long as `isTaskReadyFunction()`* keeps resolving to `true`, `isFinishedFunction()` will never be called.- * To abort a run, use the {@apilink AutoscaledPool.abort} method.*/isFinishedFunction?: () => Promise<boolean>;}-export interface AutoscaledPoolOptions extends AutoscaledPoolPredicateOptions {+/** @internal */+export interface AutoscaledPoolOptions extends TaskLoopPredicates {/*** The governor that decides whether there is free compute for one more task. Typically a* {@apilink ConcurrencySystem}, but any {@apilink IConcurrencySystem} works. Share a single instance across@@ -79,8 +78,8 @@ export interface AutoscaledPoolOptions extends AutoscaledPoolPredicateOptions {** Before running the pool, you need to implement the following three functions:* {@apilink AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`},- * {@apilink AutoscaledPoolPredicateOptions.isTaskReadyFunction|`isTaskReadyFunction`} and- * {@apilink AutoscaledPoolPredicateOptions.isFinishedFunction|`isFinishedFunction`}.+ * {@apilink TaskLoopPredicates.isTaskReadyFunction|`isTaskReadyFunction`} and+ * {@apilink TaskLoopPredicates.isFinishedFunction|`isFinishedFunction`}.** The auto-scaled pool is started by calling the {@apilink AutoscaledPool.run} function.* The pool periodically queries `isTaskReadyFunction` for more tasks, managing optimal concurrency, until the function@@ -121,7 +120,8 @@ export interface AutoscaledPoolOptions extends AutoscaledPoolPredicateOptions {* await concurrencySystem.stop();* }* ```- * @category Scaling+ *+ * @internal*/export class AutoscaledPool {private readonly log: CrawleeLogger;diff --git a/packages/http-crawler/README.md b/packages/http-crawler/README.mdindex 89b15b86f..1f54c810c 100644--- a/packages/http-crawler/README.md+++ b/packages/http-crawler/README.md@@ -22,7 +22,7 @@ preNavigationHooks: [By default, `HttpCrawler` only processes web pages with the `text/html` and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header), and skips pages with other content types. If you want the crawler to process other content types, use the [HttpCrawlerOptions.additionalMimeTypes](https://crawlee.dev/js/api/http-crawler/interface/HttpCrawlerOptions#additionalMimeTypes) constructor option. Beware that the parsing behavior differs for HTML, XML, JSON and other types of content. For more details, see [HttpCrawlerOptions.requestHandler](https://crawlee.dev/js/api/http-crawler/interface/HttpCrawlerOptions#requestHandler).-New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the [AutoscaledPool](https://crawlee.dev/js/api/core/class/AutoscaledPool) class. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `HttpCrawler` constructor, or, for finer control, by injecting a pre-configured `concurrencySystem` — see `ConcurrencySystem`.+New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's `ConcurrencySystem`. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `HttpCrawler` constructor, or, for finer control, by injecting a pre-configured `concurrencySystem`.## Example usagediff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.tsindex d047212c9..e68fce93b 100644--- a/packages/http-crawler/src/internals/file-download.ts+++ b/packages/http-crawler/src/internals/file-download.ts@@ -140,7 +140,7 @@ export function ByteCounterStream({* ]* ```*- * New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the {@apilink AutoscaledPool} class. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@apilink ConcurrencySystem|`concurrencySystem`}.+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@apilink ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@apilink ConcurrencySystem|`concurrencySystem`}.** ## Example usage*diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 987648874..a56a27d33 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -302,8 +302,8 @@ export type HttpRequestHandler<* Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.* For details, see {@apilink HttpCrawlerOptions.requestHandler}.*- * New requests are only dispatched when there is enough free CPU and memory available,- * using the functionality provided by the {@apilink AutoscaledPool} class.+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's+ * {@apilink ConcurrencySystem}.* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the* constructor, or, for finer control, by injecting a pre-configured* {@apilink ConcurrencySystem|`concurrencySystem`}.diff --git a/packages/jsdom-crawler/README.md b/packages/jsdom-crawler/README.mdindex c2cd250a1..818d84fa4 100644--- a/packages/jsdom-crawler/README.md+++ b/packages/jsdom-crawler/README.md@@ -24,7 +24,7 @@ preNavigationHooks: [By default, `JSDOMCrawler` only processes web pages with the `text/html` and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header), and skips pages with other content types. If you want the crawler to process other content types, use the [JSDOMCrawlerOptions.additionalMimeTypes](https://crawlee.dev/js/api/jsdom-crawler/interface/JSDOMCrawlerOptions#additionalMimeTypes) constructor option. Beware that the parsing behavior differs for HTML, XML, JSON and other types of content. For more details, see [JSDOMCrawlerOptions.requestHandler](https://crawlee.dev/js/api/jsdom-crawler/interface/JSDOMCrawlerOptions#requestHandler).-New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the [AutoscaledPool](https://crawlee.dev/js/api/core/class/AutoscaledPool) class. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `JSDOMCrawler` constructor, or, for finer control, by injecting a pre-configured `concurrencySystem` — see `ConcurrencySystem`.+New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's `ConcurrencySystem`. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `JSDOMCrawler` constructor, or, for finer control, by injecting a pre-configured `concurrencySystem`.## Example usagediff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsindex 16d5e3c36..39e5cdb06 100644--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts@@ -154,8 +154,8 @@ export type JSDOMRequestHandler<* Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.* For more details, see {@apilink JSDOMCrawlerOptions.requestHandler}.*- * New requests are only dispatched when there is enough free CPU and memory available,- * using the functionality provided by the {@apilink AutoscaledPool} class.+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's+ * {@apilink ConcurrencySystem}.* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the* `JSDOMCrawler` constructor, or, for finer control, by injecting a pre-configured* {@apilink ConcurrencySystem|`concurrencySystem`}.diff --git a/packages/linkedom-crawler/README.md b/packages/linkedom-crawler/README.mdindex 47e4d8c26..a75a6cd0c 100644--- a/packages/linkedom-crawler/README.md+++ b/packages/linkedom-crawler/README.md@@ -26,7 +26,7 @@ preNavigationHooks: [By default, `LinkeDOMCrawler` only processes web pages with the `text/html` and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header), and skips pages with other content types. If you want the crawler to process other content types, use the [LinkeDOMCrawlerOptions.additionalMimeTypes](https://crawlee.dev/js/api/linkedom-crawler/interface/LinkeDOMCrawlerOptions#additionalMimeTypes) constructor option. Beware that the parsing behavior differs for HTML, XML, JSON and other types of content. For more details, see [LinkeDOMCrawlerOptions.requestHandler](https://crawlee.dev/js/api/linkedom-crawler/interface/LinkeDOMCrawlerOptions#requestHandler).-New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the [AutoscaledPool](https://crawlee.dev/js/api/core/class/AutoscaledPool) class. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `LinkeDOMCrawler` constructor, or, for finer control, by injecting a pre-configured `concurrencySystem` — see `ConcurrencySystem`.+New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's `ConcurrencySystem`. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `LinkeDOMCrawler` constructor, or, for finer control, by injecting a pre-configured `concurrencySystem`.## Example usagediff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.tsindex 12e531eb8..dae37dfc0 100644--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts@@ -144,8 +144,8 @@ export type LinkeDOMRequestHandler<* Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.* For more details, see {@apilink LinkeDOMCrawlerOptions.requestHandler}.*- * New requests are only dispatched when there is enough free CPU and memory available,- * using the functionality provided by the {@apilink AutoscaledPool} class.+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's+ * {@apilink ConcurrencySystem}.* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the* `LinkeDOMCrawler` constructor, or, for finer control, by injecting a pre-configured* {@apilink ConcurrencySystem|`concurrencySystem`}.diff --git a/packages/playwright-crawler/README.md b/packages/playwright-crawler/README.mdindex adae570b6..35423de94 100644--- a/packages/playwright-crawler/README.md+++ b/packages/playwright-crawler/README.md@@ -12,7 +12,7 @@ The crawler finishes when there are no more [Request](https://crawlee.dev/js/api`PlaywrightCrawler` opens a new Chrome page (i.e. tab) for each [Request](https://crawlee.dev/js/api/core/class/Request) object to crawl and then calls the function provided by user as the [PlaywrightCrawlerOptions.requestHandler](https://crawlee.dev/js/api/playwright-crawler/interface/PlaywrightCrawlerOptions#requestHandler) option.-New pages are only opened when there is enough free CPU and memory available, using the functionality provided by the [AutoscaledPool](https://crawlee.dev/js/api/core/class/AutoscaledPool) class. Concurrency is tuned via the [`minConcurrency`](https://crawlee.dev/js/api/playwright-crawler/interface/PlaywrightCrawlerOptions#minConcurrency), [`maxConcurrency`](https://crawlee.dev/js/api/playwright-crawler/interface/PlaywrightCrawlerOptions#maxConcurrency) and [`maxRequestsPerMinute`](https://crawlee.dev/js/api/playwright-crawler/interface/PlaywrightCrawlerOptions#maxRequestsPerMinute) options of the `PlaywrightCrawler` constructor, or, for finer control, by injecting a pre-configured [`concurrencySystem`](https://crawlee.dev/js/api/playwright-crawler/interface/PlaywrightCrawlerOptions#concurrencySystem) — see `ConcurrencySystem`.+New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's `ConcurrencySystem`. Concurrency is tuned via the [`minConcurrency`](https://crawlee.dev/js/api/playwright-crawler/interface/PlaywrightCrawlerOptions#minConcurrency), [`maxConcurrency`](https://crawlee.dev/js/api/playwright-crawler/interface/PlaywrightCrawlerOptions#maxConcurrency) and [`maxRequestsPerMinute`](https://crawlee.dev/js/api/playwright-crawler/interface/PlaywrightCrawlerOptions#maxRequestsPerMinute) options of the `PlaywrightCrawler` constructor, or, for finer control, by injecting a pre-configured [`concurrencySystem`](https://crawlee.dev/js/api/playwright-crawler/interface/PlaywrightCrawlerOptions#concurrencySystem).Note that the pool of Playwright instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex 90b8f297f..996a06d4e 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -143,8 +143,8 @@ export interface PlaywrightCrawlerOptions<* `PlaywrightCrawler` opens a new Chrome page (i.e. tab) for each {@apilink Request} object to crawl* and then calls the function provided by user as the {@apilink PlaywrightCrawlerOptions.requestHandler} option.*- * New pages are only opened when there is enough free CPU and memory available,- * using the functionality provided by the {@apilink AutoscaledPool} class.+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's+ * {@apilink ConcurrencySystem}.* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the* `PlaywrightCrawler` constructor, or, for finer control, by injecting a pre-configured* {@apilink ConcurrencySystem|`concurrencySystem`}.diff --git a/packages/puppeteer-crawler/README.md b/packages/puppeteer-crawler/README.mdindex c52535543..3b1fdada4 100644--- a/packages/puppeteer-crawler/README.md+++ b/packages/puppeteer-crawler/README.md@@ -16,8 +16,8 @@ The crawler finishes when there are no more [Request](https://crawlee.dev/js/apiand then calls the function provided by user as the [PuppeteerCrawlerOptions.requestHandler](https://crawlee.dev/js/api/puppeteer-crawler/interface/PuppeteerCrawlerOptions#requestHandler) option.New pages are only opened when there is enough free CPU and memory available,-using the functionality provided by the [AutoscaledPool](https://crawlee.dev/js/api/core/class/AutoscaledPool) class.-Concurrency is tuned via the [`minConcurrency`](https://crawlee.dev/js/api/puppeteer-crawler/interface/PuppeteerCrawlerOptions#minConcurrency), [`maxConcurrency`](https://crawlee.dev/js/api/puppeteer-crawler/interface/PuppeteerCrawlerOptions#maxConcurrency) and [`maxRequestsPerMinute`](https://crawlee.dev/js/api/puppeteer-crawler/interface/PuppeteerCrawlerOptions#maxRequestsPerMinute) options of the `PuppeteerCrawler` constructor, or, for finer control, by injecting a pre-configured [`concurrencySystem`](https://crawlee.dev/js/api/puppeteer-crawler/interface/PuppeteerCrawlerOptions#concurrencySystem) — see `ConcurrencySystem`.+as judged by the crawler's `ConcurrencySystem`.+Concurrency is tuned via the [`minConcurrency`](https://crawlee.dev/js/api/puppeteer-crawler/interface/PuppeteerCrawlerOptions#minConcurrency), [`maxConcurrency`](https://crawlee.dev/js/api/puppeteer-crawler/interface/PuppeteerCrawlerOptions#maxConcurrency) and [`maxRequestsPerMinute`](https://crawlee.dev/js/api/puppeteer-crawler/interface/PuppeteerCrawlerOptions#maxRequestsPerMinute) options of the `PuppeteerCrawler` constructor, or, for finer control, by injecting a pre-configured [`concurrencySystem`](https://crawlee.dev/js/api/puppeteer-crawler/interface/PuppeteerCrawlerOptions#concurrencySystem).Note that the pool of Puppeteer instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex cbbdb1f1c..62b0bc51c 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -120,8 +120,8 @@ export interface PuppeteerCrawlerOptions<* `PuppeteerCrawler` opens a new Chrome page (i.e. tab) for each {@apilink Request} object to crawl* and then calls the function provided by user as the {@apilink PuppeteerCrawlerOptions.requestHandler} option.*- * New pages are only opened when there is enough free CPU and memory available,- * using the functionality provided by the {@apilink AutoscaledPool} class.+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's+ * {@apilink ConcurrencySystem}.* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the* `PuppeteerCrawler` constructor, or, for finer control, by injecting a pre-configured* {@apilink ConcurrencySystem|`concurrencySystem`}.diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 11391ca17..1c30154d4 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -22,6 +22,7 @@ import {serviceLocator,SessionPool,} from '@crawlee/basic';+import type { IConcurrencySystem } from '@crawlee/core';import { ConcurrencySystem, Dataset, MemoryStorageBackend, RequestState, SessionPool, Statistics } from '@crawlee/core';import type { Dictionary, ISession, ProxyInfo } from '@crawlee/types';import { RobotsTxtFile, sleep } from '@crawlee/utils';@@ -166,7 +167,7 @@ describe('BasicCrawler', () => {await basicCrawler.run();- expect((basicCrawler.autoscaledPool!.system as ConcurrencySystem).minConcurrency).toBe(25);+ expect((basicCrawler.concurrencySystem! as ConcurrencySystem).minConcurrency).toBe(25);expect(processed).toEqual(sourcesCopy);expect(await requestList.isFinished()).toBe(true);expect(await requestList.isEmpty()).toBe(true);@@ -198,8 +199,8 @@ describe('BasicCrawler', () => {await system.stop();// The crawler built its own pool but wired the shared governor into it.- expect(basicCrawler.autoscaledPool!.system).toBe(system);- expect((basicCrawler.autoscaledPool!.system as ConcurrencySystem).minConcurrency).toBe(7);+ expect(basicCrawler.concurrencySystem!).toBe(system);+ expect((basicCrawler.concurrencySystem! as ConcurrencySystem).minConcurrency).toBe(7);// Work actually ran (the crawler kept its own task loop).expect(processed).toHaveLength(20);// The crawler never touched the borrowed system's lifecycle — only our two explicit calls did.@@ -304,12 +305,12 @@ describe('BasicCrawler', () => {});await crawler.run(['https://example.com/1']);- const firstSystem = crawler.autoscaledPool!.system as ConcurrencySystem;+ const firstSystem = crawler.concurrencySystem! as ConcurrencySystem;// Simulate scaling state left behind by the first run.firstSystem.desiredConcurrency = 42;await crawler.run(['https://example.com/2']);- const secondSystem = crawler.autoscaledPool!.system;+ const secondSystem = crawler.concurrencySystem!;// The crawler-owned governor is rebuilt per run, so no previous-run state (resource snapshots,// autoscaled desired concurrency, per-minute task counts) can distort the next run's scaling.@@ -335,7 +336,7 @@ describe('BasicCrawler', () => {await expect(crawler.run()).rejects.toThrow(failure);// The intervals would otherwise keep the event loop alive for the rest of the process's life.- expect((crawler.autoscaledPool!.system as ConcurrencySystem).isRunning).toBe(false);+ expect((crawler.concurrencySystem! as ConcurrencySystem).isRunning).toBe(false);// A failed startup is not a run, so the crawler must not stay wedged as `running`.getRequestManager.mockRestore();@@ -577,10 +578,10 @@ describe('BasicCrawler', () => {const requestHandler = async () => {};const collect = (crawler: BasicCrawler) => ({- minConcurrency: (crawler.autoscaledPool!.system as ConcurrencySystem).minConcurrency,- maxConcurrency: (crawler.autoscaledPool!.system as ConcurrencySystem).maxConcurrency,+ minConcurrency: (crawler.concurrencySystem! as ConcurrencySystem).minConcurrency,+ maxConcurrency: (crawler.concurrencySystem! as ConcurrencySystem).maxConcurrency,// eslint-disable-next-line dot-notation -- private member on the governor- maxTasksPerMinute: (crawler.autoscaledPool!.system as ConcurrencySystem)['maxTasksPerMinute'],+ maxTasksPerMinute: (crawler.concurrencySystem! as ConcurrencySystem)['maxTasksPerMinute'],});// Shortcuts feed the default ConcurrencySystem the crawler builds.@@ -608,7 +609,7 @@ describe('BasicCrawler', () => {expect(collect(shortcuts)).toEqual({ minConcurrency: 123, maxConcurrency: 456, maxTasksPerMinute: 789 });expect(collect(injected)).toEqual({ minConcurrency: 16, maxConcurrency: 32, maxTasksPerMinute: 64 });// The injected system is the very instance the pool uses.- expect(injected.autoscaledPool!.system).toBe(injectedSystem);+ expect(injected.concurrencySystem!).toBe(injectedSystem);});test('auto-saved state object', async () => {@@ -783,7 +784,7 @@ describe('BasicCrawler', () => {expect(setValueSpy).toBeCalled();// clean up- // @ts-expect-error Accessing private method+ // @ts-expect-error Accessing a private property and its private methodawait basicCrawler.autoscaledPool!.destroy();},);@@ -1240,7 +1241,7 @@ describe('BasicCrawler', () => {expect(await crawler.isTaskReadyFunction()).toBe(false);});- test('should be possible to override isFinishedFunction and isTaskReadyFunction of underlying AutoscaledPool', async () => {+ test('should be possible to override isFinishedFunction and isTaskReadyFunction via taskLoopOptions', async () => {const requestQueue = await RequestQueue.open({ id: 'xxx' });const processed: Request[] = [];const queue: Request[] = [];@@ -1252,7 +1253,7 @@ describe('BasicCrawler', () => {requestQueue,minConcurrency: 1,maxConcurrency: 1,- autoscaledPoolOptions: {+ taskLoopOptions: {isFinishedFunction: async () => {isFinishedFunctionCalled = true;return Promise.resolve(isFinished);@@ -1270,7 +1271,7 @@ describe('BasicCrawler', () => {// Speed up the test// @ts-expect-error Accessing private prop- basicCrawler.autoscaledPoolOptions.maybeRunIntervalSecs = 0.05;+ basicCrawler.taskLoopOptions.maybeRunIntervalSecs = 0.05;const request0 = new Request({ url: 'http://example.com/0' });const request1 = new Request({ url: 'http://example.com/1' });@@ -1330,7 +1331,7 @@ describe('BasicCrawler', () => {// Speed up the test// @ts-expect-error Accessing private prop- basicCrawler.autoscaledPoolOptions.maybeRunIntervalSecs = 0.05;+ basicCrawler.taskLoopOptions.maybeRunIntervalSecs = 0.05;const request0 = new Request({ url: 'http://example.com/0' });const request1 = new Request({ url: 'http://example.com/1' });@@ -1364,6 +1365,83 @@ describe('BasicCrawler', () => {vitest.restoreAllMocks();});+ test('pause() suspends dispatch without ending the run, resume() picks it back up', async () => {+ const sources = [...Array(10).keys()].map((index) => ({ url: `https://example.com/${index}` }));+ const requestList = await RequestList.open(null, sources);+ const processed: string[] = [];++ let resolvePaused: () => void;+ const paused = new Promise<void>((resolve) => {+ resolvePaused = resolve;+ });++ const crawler = new BasicCrawler({+ requestList,+ // One request at a time, so the pause is triggered by exactly one handler.+ minConcurrency: 1,+ maxConcurrency: 1,+ requestHandler: async ({ request }) => {+ processed.push(request.url);++ if (processed.length === 3) {+ // Deliberately not awaited: `pause()` only resolves once the in-flight requests have settled,+ // and this handler is one of them. Dispatch stops synchronously all the same.+ void crawler.pause().then(() => resolvePaused());+ }++ await sleep(10);+ },+ });++ const runPromise = crawler.run();+ await paused;++ expect(processed).toHaveLength(3);++ // Unlike `stop()`, pausing leaves `run()` pending and dispatches nothing in the meantime.+ await sleep(300);+ expect(processed).toHaveLength(3);++ crawler.resume();+ await runPromise;++ expect(processed).toHaveLength(10);+ });++ test('pause() and resume() warn instead of throwing on a crawler that is not running', async () => {+ const crawler = new BasicCrawler({ requestHandler: async () => {} });+ const warning = vitest.spyOn(crawler.log, 'warning').mockImplementation(() => {});++ // The task loop and the owned governor are both built by `run()`.+ expect(crawler.concurrencySystem).toBeUndefined();++ await crawler.pause();+ crawler.resume();++ expect(warning).toHaveBeenCalledTimes(2);++ vitest.restoreAllMocks();+ });++ test('exposes the concurrency system it is running against', async () => {+ const requestList = await RequestList.open(null, [{ url: 'https://example.com/1' }]);+ let observed: IConcurrencySystem | undefined;++ const crawler = new BasicCrawler({+ requestList,+ maxConcurrency: 17,+ requestHandler: async () => {+ observed = crawler.concurrencySystem;+ },+ });++ await crawler.run();++ // The same (owned, default) governor the handler saw, still readable after the run.+ expect(crawler.concurrencySystem).toBe(observed);+ expect((observed as ConcurrencySystem).maxConcurrency).toBe(17);+ });+test('should support maxRequestsPerCrawl parameter', async () => {const sources = [{ url: 'http://example.com/1' },diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.tsindex a958600c8..4532ae84f 100644--- a/test/core/crawlers/browser_crawler.test.ts+++ b/test/core/crawlers/browser_crawler.test.ts@@ -108,7 +108,7 @@ describe('BrowserCrawler', () => {await browserCrawler.run();- expect((browserCrawler.autoscaledPool!.system as ConcurrencySystem).minConcurrency).toBe(1);+ expect((browserCrawler.concurrencySystem! as ConcurrencySystem).minConcurrency).toBe(1);expect(processed).toHaveLength(6);expect(failed).toHaveLength(0);diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex 60fadd34e..deda02330 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -106,7 +106,7 @@ describe('CheerioCrawler', () => {await cheerioCrawler.run();- expect((cheerioCrawler.autoscaledPool!.system as ConcurrencySystem).minConcurrency).toBe(2);+ expect((cheerioCrawler.concurrencySystem! as ConcurrencySystem).minConcurrency).toBe(2);expect(processed).toHaveLength(4);expect(failed).toHaveLength(0);@@ -139,7 +139,7 @@ describe('CheerioCrawler', () => {await cheerioCrawler.run();- expect((cheerioCrawler.autoscaledPool!.system as ConcurrencySystem).minConcurrency).toBe(2);+ expect((cheerioCrawler.concurrencySystem! as ConcurrencySystem).minConcurrency).toBe(2);expect(processed).toHaveLength(4);expect(failed).toHaveLength(0);@@ -175,7 +175,7 @@ describe('CheerioCrawler', () => {await cheerioCrawler.run();- expect((cheerioCrawler.autoscaledPool!.system as ConcurrencySystem).minConcurrency).toBe(2);+ expect((cheerioCrawler.concurrencySystem! as ConcurrencySystem).minConcurrency).toBe(2);expect(processed).toHaveLength(4);expect(failed).toHaveLength(0);@@ -557,7 +557,7 @@ describe('CheerioCrawler', () => {await cheerioCrawler.run();- expect((cheerioCrawler.autoscaledPool!.system as ConcurrencySystem).minConcurrency).toBe(2);+ expect((cheerioCrawler.concurrencySystem! as ConcurrencySystem).minConcurrency).toBe(2);expect(failed).toHaveLength(0);});@@ -578,7 +578,7 @@ describe('CheerioCrawler', () => {await cheerioCrawler.run();- expect((cheerioCrawler.autoscaledPool!.system as ConcurrencySystem).minConcurrency).toBe(2);+ expect((cheerioCrawler.concurrencySystem! as ConcurrencySystem).minConcurrency).toBe(2);expect(failed).toHaveLength(4);});diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.tsindex debb2e124..1e485afde 100644--- a/test/core/crawlers/http_crawler.test.ts+++ b/test/core/crawlers/http_crawler.test.ts@@ -180,7 +180,7 @@ test('concurrency shortcuts coexist with the HTTP-optimized defaults', async ()// The shortcut applies, and the governor really is the one wired into the pool (autoscaling never moves the// ceiling, so this one is safe to read after the run)...- expect((crawler.autoscaledPool!.system as ConcurrencySystem).maxConcurrency).toBe(5);+ expect((crawler.concurrencySystem! as ConcurrencySystem).maxConcurrency).toBe(5);// ...without discarding the HTTP-optimized starting concurrency, which the max then caps.expect(crawler.asConfigured!.desiredConcurrency).toBe(5);});@@ -427,7 +427,7 @@ test('should ignore http error status codes set by user', async () => {await crawler.run([`${url}/500Error`]);- expect((crawler.autoscaledPool!.system as ConcurrencySystem).minConcurrency).toBe(2);+ expect((crawler.concurrencySystem! as ConcurrencySystem).minConcurrency).toBe(2);expect(failed).toHaveLength(0);});@@ -446,7 +446,7 @@ test('should throw an error on http error status codes set by user', async () =>await crawler.run([`${url}/hello.html`]);- expect((crawler.autoscaledPool!.system as ConcurrencySystem).minConcurrency).toBe(2);+ expect((crawler.concurrencySystem! as ConcurrencySystem).minConcurrency).toBe(2);expect(failed).toHaveLength(1);});diff --git a/test/core/crawlers/playwright_crawler.test.ts b/test/core/crawlers/playwright_crawler.test.tsindex 29c83696b..67813b3e5 100644--- a/test/core/crawlers/playwright_crawler.test.ts+++ b/test/core/crawlers/playwright_crawler.test.ts@@ -113,7 +113,7 @@ describe('PlaywrightCrawler', () => {await playwrightCrawler.run();- expect((playwrightCrawler.autoscaledPool!.system as ConcurrencySystem).minConcurrency).toBe(1);+ expect((playwrightCrawler.concurrencySystem! as ConcurrencySystem).minConcurrency).toBe(1);expect(processed).toHaveLength(6);expect(failed).toHaveLength(0);diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.tsindex b0b3fafa7..054c5f84c 100644--- a/test/core/crawlers/puppeteer_crawler.test.ts+++ b/test/core/crawlers/puppeteer_crawler.test.ts@@ -126,7 +126,7 @@ describe('PuppeteerCrawler', () => {await puppeteerCrawler.run();- expect((puppeteerCrawler.autoscaledPool!.system as ConcurrencySystem).minConcurrency).toBe(1);+ expect((puppeteerCrawler.concurrencySystem! as ConcurrencySystem).minConcurrency).toBe(1);expect(processed).toHaveLength(6);expect(failed).toHaveLength(0);