Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

refactor!: Accept a pre-configured Statistics instance in crawlers instead of an option bag (#3966)

Jan Buchard915e537dca2
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md
index 697e1cc7e..7e5ba3119 100644
--- a/docs/public-api/crawlee-basic.api.md
+++ b/docs/public-api/crawlee-basic.api.md
@@ -34,6 +34,7 @@ import { IRequestLoader } from '@crawlee/core';
import { IRequestManager } from '@crawlee/core';
import type { ISession } from '@crawlee/types';
import type { ISessionPool } from '@crawlee/types';
+import { IStatistics } from '@crawlee/core';
import { NumberPredicate } from 'ow';
import { ObjectPredicate } from 'ow';
import { Predicate } from 'ow';
@@ -46,8 +47,6 @@ import type { RouterHandler } from '@crawlee/core';
import type { RouterRoutes } from '@crawlee/core';
import type { SetStatusMessageOptions } from '@crawlee/types';
import type { SkippedRequestCallback } from '@crawlee/core';
-import { Statistics } from '@crawlee/core';
-import type { StatisticsOptions } from '@crawlee/core';
import type { StatisticState } from '@crawlee/core';
import type { StorageBackend } from '@crawlee/types';
import type { StorageIdentifier } from '@crawlee/core';
@@ -152,7 +151,7 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con
maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
- statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ statistics: ObjectPredicate<object> & BasePredicate<object | undefined>;
id: StringPredicate & BasePredicate<string | undefined>;
};
pause(timeoutSecs?: number): Promise<void>;
@@ -172,7 +171,7 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con
protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
get sessionPool(): ISessionPool;
setStatusMessage(message: string, options?: SetStatusMessageOptions): void;
- readonly stats: Statistics;
+ get stats(): IStatistics;
stop(reason?: string): void;
teardown(): Promise<void>;
protected _throwOnBlockedRequest(statusCode: number): void;
@@ -217,7 +216,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
retryOnBlocked?: boolean;
sameDomainDelaySecs?: number;
sessionPool?: ISessionPool;
- statisticsOptions?: StatisticsOptions;
+ statistics?: IStatistics;
statusMessageCallback?: StatusMessageCallback;
statusMessageLoggingInterval?: number;
storageBackend?: StorageBackend;
diff --git a/docs/public-api/crawlee-browser.api.md b/docs/public-api/crawlee-browser.api.md
index 3ebd1b34b..7b53887da 100644
--- a/docs/public-api/crawlee-browser.api.md
+++ b/docs/public-api/crawlee-browser.api.md
@@ -106,7 +106,7 @@ export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Respo
maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
- statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ statistics: ObjectPredicate<object> & BasePredicate<object | undefined>;
id: StringPredicate & BasePredicate<string | undefined>;
};
protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md
index 8daf0892b..432de615f 100644
--- a/docs/public-api/crawlee-core.api.md
+++ b/docs/public-api/crawlee-core.api.md
@@ -131,6 +131,17 @@ interface BrowserPage {
content: () => Promise<string>;
}
+// @public
+export interface CalculatedStatistics {
+ crawlerRuntimeMillis: number;
+ requestAvgFailedDurationMillis: number;
+ requestAvgFinishedDurationMillis: number;
+ requestsFailedPerMinute: number;
+ requestsFinishedPerMinute: number;
+ requestsTotal: number;
+ requestTotalDurationMillis: number;
+}
+
// @public
export const checkStorageAccess: () => void | undefined;
@@ -817,6 +828,23 @@ export interface IRequestManager extends IRequestLoader {
setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
}
+// @public
+export interface IStatistics {
+ calculate(): CalculatedStatistics;
+ discardJob(id: number | string): void;
+ readonly errorTracker: ErrorTracker;
+ readonly errorTrackerRetry: ErrorTracker;
+ failJob(id: number | string, retryCount: number): void;
+ finishJob(id: number | string, retryCount: number): void;
+ persistState?(options?: PersistenceOptions): Promise<void>;
+ registerStatusCode(code: number): void;
+ readonly requestRetryHistogram: number[];
+ startCapturing(): Promise<void>;
+ startJob(id: number | string): void;
+ readonly state: StatisticState;
+ stopCapturing(): Promise<void>;
+}
+
// @public
export interface IStorage {
// (undocumented)
@@ -1880,16 +1908,9 @@ export interface StatisticPersistedState extends Omit<StatisticState, 'statsPers
}
// @public
-export class Statistics {
- calculate(): {
- requestAvgFailedDurationMillis: number;
- requestAvgFinishedDurationMillis: number;
- requestsFinishedPerMinute: number;
- requestsFailedPerMinute: number;
- requestTotalDurationMillis: number;
- requestsTotal: number;
- crawlerRuntimeMillis: number;
- };
+export class Statistics implements IStatistics {
+ constructor(options?: StatisticsOptions);
+ calculate(): CalculatedStatistics;
readonly errorTracker: ErrorTracker;
readonly errorTrackerRetry: ErrorTracker;
readonly id: string;
diff --git a/docs/public-api/crawlee-http.api.md b/docs/public-api/crawlee-http.api.md
index 78d28252d..07a7d75dc 100644
--- a/docs/public-api/crawlee-http.api.md
+++ b/docs/public-api/crawlee-http.api.md
@@ -156,7 +156,7 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> =
maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
- statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ statistics: ObjectPredicate<object> & BasePredicate<object | undefined>;
id: StringPredicate & BasePredicate<string | undefined>;
};
}
diff --git a/docs/public-api/crawlee-jsdom.api.md b/docs/public-api/crawlee-jsdom.api.md
index 39d82600f..741b4fc56 100644
--- a/docs/public-api/crawlee-jsdom.api.md
+++ b/docs/public-api/crawlee-jsdom.api.md
@@ -100,7 +100,7 @@ export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext
maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
- statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ statistics: ObjectPredicate<object> & BasePredicate<object | undefined>;
id: StringPredicate & BasePredicate<string | undefined>;
};
}
diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.md
index d6dd5f707..f54730fa8 100644
--- a/docs/public-api/crawlee-playwright.api.md
+++ b/docs/public-api/crawlee-playwright.api.md
@@ -94,7 +94,7 @@ export class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
// (undocumented)
protected runRequestHandler(crawlingContext: CrawlingContext_2): Promise<void>;
// (undocumented)
- readonly stats: AdaptivePlaywrightCrawlerStatistics;
+ get stats(): AdaptivePlaywrightCrawlerStatistics;
// (undocumented)
teardown(): Promise<void>;
}
@@ -390,7 +390,7 @@ export class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedCon
maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
- statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ statistics: ObjectPredicate<object> & BasePredicate<object | undefined>;
id: StringPredicate & BasePredicate<string | undefined>;
};
}
diff --git a/docs/public-api/crawlee-puppeteer.api.md b/docs/public-api/crawlee-puppeteer.api.md
index 5fb1fee84..6d244e8c4 100644
--- a/docs/public-api/crawlee-puppeteer.api.md
+++ b/docs/public-api/crawlee-puppeteer.api.md
@@ -242,7 +242,7 @@ export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedCont
maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
- statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ statistics: ObjectPredicate<object> & BasePredicate<object | undefined>;
id: StringPredicate & BasePredicate<string | undefined>;
};
}
diff --git a/docs/public-api/crawlee-stagehand.api.md b/docs/public-api/crawlee-stagehand.api.md
index a90c95838..8581e2086 100644
--- a/docs/public-api/crawlee-stagehand.api.md
+++ b/docs/public-api/crawlee-stagehand.api.md
@@ -130,7 +130,7 @@ export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedCont
maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
- statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ statistics: ObjectPredicate<object> & BasePredicate<object | undefined>;
id: StringPredicate & BasePredicate<string | undefined>;
};
}
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md
index 1b7a7879c..c77732a5d 100644
--- a/docs/upgrading/upgrading_v4.md
+++ b/docs/upgrading/upgrading_v4.md
@@ -17,7 +17,7 @@ This page summarizes the breaking changes in Crawlee v4. There are many, so the
- **Timeouts that mean what they say.** Navigation and the request handler are [timed separately](#navigation-and-the-request-handler-are-timed-separately) — no more mysteriously summed limits — and a single route can get [its own timeout](#per-route-and-per-request-handler-timeouts) or extend it mid-flight.
- **Composable crawling context.** The new `extendContext` option and `ContextPipeline` composition replace subclassing tricks for [adding members to the crawling context](#crawling-context-no-longer-includes-a-reference-to-the-crawler-itself).
-- **Bring your own implementation.** Crawlers now accept any [`ISessionPool`](#custom-sessionpool-implementations-via-the-isessionpool-interface), [`IBrowserPool`](#custom-browserpool-implementations-via-the-ibrowserpool-interface), [`IRenderingTypePredictor`](#custom-rendering-type-predictors-via-the-irenderingtypepredictor-interface) or [`IRequestManager`](#request-loaders-and-managers) — and never tear down an instance they did not create.
+- **Bring your own implementation.** Crawlers now accept any [`ISessionPool`](#custom-sessionpool-implementations-via-the-isessionpool-interface), [`IBrowserPool`](#custom-browserpool-implementations-via-the-ibrowserpool-interface), [`IRenderingTypePredictor`](#custom-rendering-type-predictors-via-the-irenderingtypepredictor-interface), [`IRequestManager`](#request-loaders-and-managers) or [`IStatistics`](#statisticsoptions-is-replaced-by-a-statistics-instance) — and never tear down an instance they did not create.
- **One concurrency budget for several crawlers.** The new [`ConcurrencySystem`](#autoscaling-moved-to-concurrencysystem) can be shared between crawlers, capping their combined concurrency instead of letting each one oversubscribe the host.
- **Native `fetch` types.** HTTP clients and `context.response` now use the [standard `Response`](#crawlingcontextresponse-is-now-of-type-response), and `got-scraping` is an [opt-in dependency](#http-client-packages-and-basehttpclient-reshaped) instead of a mandatory one.
- **The session is the rotation unit.** A session carries its proxy, cookies and error score, and is rotated as a whole when blocked — replacing [proxy tiers](#tieredproxyurls-is-removed-from-proxyconfiguration) and [session rotation counters](#maxsessionrotations-and-requestsessionrotationcount-are-removed).
@@ -933,6 +933,28 @@ If you don't pass a predictor, nothing changes: the crawler builds one from `ren
This experimental option relied on an outdated manifest version for browser extensions, it is not possible to achieve this with the currently supported versions.
+## Only if you customize crawler statistics
+
+Applies when you passed `statisticsOptions` to a crawler, or subclassed `Statistics`.
+
+### `statisticsOptions` is replaced by a `statistics` instance
+
+The `statisticsOptions` option has been removed from the crawler constructor. Instead of passing options for the crawler to build its `Statistics` from, construct a `Statistics` instance yourself and pass it via the new `statistics` option — the same inject-or-default idiom as `sessionPool` and `browserPool`.
+
+```typescript
+import { Statistics } from '@crawlee/core';
+
+const crawler = new BasicCrawler({
+ // The old parameter won't work anymore
+ // statisticsOptions: { saveErrorSnapshots: true },
+ statistics: new Statistics({ saveErrorSnapshots: true }),
+});
+```
+
+Omit the option and the crawler builds its own default, exactly as before. A supplied instance is treated as borrowed: the crawler records into it and drives its capture lifecycle for the run, but never `reset()`s it between `run()` calls — so a preconfigured instance keeps whatever state it was handed. This is also the supported way to plug in a custom `Statistics` subclass that tracks extra fields (superseding the previous subclass-and-reassign pattern).
+
+The option accepts the built-in `Statistics` or any object implementing the new `IStatistics` interface, so a fully custom statistics backend can be plugged in without subclassing. The crawler exposes it as `crawler.stats` typed as `IStatistics`.
+
## Only if you wrote a custom HTTP client or used `got-scraping` directly
Applies when you implemented `BaseHttpClient` yourself, or imported `gotScraping` from `@crawlee/utils`.
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index d1145ca75..81e17c4f0 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -19,6 +19,7 @@ import type {
IProxyConfiguration,
IRequestLoader,
IRequestManager,
+ IStatistics,
Request,
RequestsLike,
RequestTransform,
@@ -26,7 +27,6 @@ import type {
RouterRoutes,
SkippedRequestCallback,
Source,
- StatisticsOptions,
StatisticState,
StorageIdentifier,
TaskLoopPredicates,
@@ -452,10 +452,11 @@ export interface BasicCrawlerOptions<
onSkippedRequest?: SkippedRequestCallback;
/**
- * Customize the way statistics collecting works, such as logging interval or
- * whether to output them to the Key-Value store.
+ * A preconfigured statistics instance. When provided, the crawler records into it instead of building its own and
+ * will not `reset()` it between `run()` calls. Accepts the built-in {@apilink Statistics} (subclass it to track
+ * extra fields) or any object implementing {@apilink IStatistics}.
*/
- statisticsOptions?: StatisticsOptions;
+ statistics?: IStatistics;
/**
* HTTP client implementation for the `sendRequest` context helper and for plain HTTP crawling.
@@ -626,10 +627,16 @@ export class BasicCrawler<
*/
private static useStateAnonymousIndices = new Set<number>();
+ /** Backs the {@apilink BasicCrawler.stats|`stats`} getter. */
+ private statsDep: OwnedOrInjected<IStatistics, Statistics>;
+
/**
- * A reference to the underlying {@apilink Statistics} class that collects and logs run statistics for requests.
+ * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
+ * crawler-built default. Typed as {@apilink IStatistics} so custom implementations can be plugged in.
*/
- readonly stats: Statistics;
+ get stats(): IStatistics {
+ return this.statsDep.value;
+ }
/**
* The main request-handling component of the crawler. It manages the requests that the crawler processes,
@@ -836,7 +843,7 @@ export class BasicCrawler<
maxRequestsPerMinute: ow.optional.number.integerOrInfinite.positive.greaterThanOrEqual(1),
keepAlive: ow.optional.boolean,
- statisticsOptions: ow.optional.object,
+ statistics: ow.optional.object,
id: ow.optional.string,
};
@@ -890,7 +897,7 @@ export class BasicCrawler<
failedRequestHandler,
statusMessageLoggingInterval = 10,
statusMessageCallback,
- statisticsOptions,
+ statistics,
httpClient,
id,
@@ -996,12 +1003,15 @@ export class BasicCrawler<
this.maxRequestRetries = maxRequestRetries;
this.maxCrawlDepth = maxCrawlDepth;
this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
- this.stats = new Statistics({
- logMessage: `${this.constructor.name} request statistics:`,
- log: this.log,
- id: this.identity.id,
- ...statisticsOptions,
- });
+ this.statsDep = OwnedOrInjected.resolve<IStatistics, Statistics>(
+ statistics,
+ () =>
+ new Statistics({
+ logMessage: `${this.constructor.name} request statistics:`,
+ log: this.log,
+ id: this.identity.id,
+ }),
+ );
if (sessionPool && proxyConfiguration) {
this.log.warning(
@@ -1516,8 +1526,11 @@ export class BasicCrawler<
await managerToPurge.purge();
}
- this.stats.reset();
- await this.stats.resetStore();
+ // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
+ await this.statsDep.ifOwned(async (stats) => {
+ stats.reset();
+ await stats.resetStore();
+ });
await this.sessionPoolDep.ifOwned((pool) => pool.resetStore());
}
@@ -2189,7 +2202,7 @@ export class BasicCrawler<
}
})();
- await Promise.all([requestManagerPersistPromise, this.stats.persistState()]);
+ await Promise.all([requestManagerPersistPromise, this.stats.persistState?.()]);
}
/**
diff --git a/packages/core/src/crawlers/error_snapshotter.ts b/packages/core/src/crawlers/error_snapshotter.ts
index 441597cc8..09832c7ec 100644
--- a/packages/core/src/crawlers/error_snapshotter.ts
+++ b/packages/core/src/crawlers/error_snapshotter.ts
@@ -30,9 +30,7 @@ interface ErrorSnapshot {
* ```ts
* const crawler = new BasicCrawler({
* // ...
- * statisticsOptions: {
- * saveErrorSnapshots: true,
- * },
+ * statistics: new Statistics({ saveErrorSnapshots: true }),
* });
* ```
*/
diff --git a/packages/core/src/crawlers/statistics.ts b/packages/core/src/crawlers/statistics.ts
index 1943f6693..f404ceb16 100644
--- a/packages/core/src/crawlers/statistics.ts
+++ b/packages/core/src/crawlers/statistics.ts
@@ -44,6 +44,85 @@ export interface PersistenceOptions {
enable?: boolean;
}
+/**
+ * The statistics surface a crawler depends on: recording per-request outcomes, tracking errors, and driving the
+ * capture lifecycle for a run. Injected via the crawler's `statistics` option, so a custom implementation (or a
+ * {@apilink Statistics} subclass tracking extra fields) can be plugged in without subclassing the crawler.
+ *
+ * The owned-only mutators the crawler uses to *own* a default it built - `reset()`/`resetStore()` - are deliberately
+ * absent: an injected instance is borrowed, and the crawler never wipes it. Those live on the concrete
+ * {@apilink Statistics} only.
+ *
+ * @category Crawlers
+ */
+export interface IStatistics {
+ /** Tracker for errors on the final retry of a request. */
+ readonly errorTracker: ErrorTracker;
+
+ /** Tracker for errors on retries prior to the final one. */
+ readonly errorTrackerRetry: ErrorTracker;
+
+ /** The live statistics state the crawler reads for status messages and the final summary. */
+ readonly state: StatisticState;
+
+ /** Retries histogram - index `i` holds the number of requests that finished after `i` retries. */
+ readonly requestRetryHistogram: number[];
+
+ /** Marks a request as started, so its duration can be measured on finish/fail. */
+ startJob(id: number | string): void;
+
+ /** Marks a started request as finished, updating the finished counters and durations. */
+ finishJob(id: number | string, retryCount: number): void;
+
+ /** Marks a started request as failed, updating the failed counters and durations. */
+ failJob(id: number | string, retryCount: number): void;
+
+ /** Drops a started request without counting it as finished or failed (e.g. skipped by robots.txt). */
+ discardJob(id: number | string): void;
+
+ /** Increments the counter for the given HTTP status code. */
+ registerStatusCode(code: number): void;
+
+ /** Computes the derived aggregates (averages, per-minute rates, totals) from the current state. */
+ calculate(): CalculatedStatistics;
+
+ /** Begins a capture window: loads any persisted state, subscribes to persistence events, starts periodic logging. */
+ startCapturing(): Promise<void>;
+
+ /** Ends the capture window: stops logging, unsubscribes, and persists the final state. */
+ stopCapturing(): Promise<void>;
+
+ /**
+ * Persists the current state to the key-value store. Optional - the crawler calls it on migration, but a backend
+ * with no persistence of its own can omit it.
+ */
+ persistState?(options?: PersistenceOptions): Promise<void>;
+}
+
+/** The derived aggregates computed by {@apilink IStatistics.calculate} from the current {@apilink StatisticState}. */
+export interface CalculatedStatistics {
+ /** Mean duration of a failed request, in milliseconds; `Infinity` when nothing has failed. */
+ requestAvgFailedDurationMillis: number;
+
+ /** Mean duration of a finished request, in milliseconds; `Infinity` when nothing has finished. */
+ requestAvgFinishedDurationMillis: number;
+
+ /** Requests finished per minute over the run so far. */
+ requestsFinishedPerMinute: number;
+
+ /** Requests failed per minute over the run so far. */
+ requestsFailedPerMinute: number;
+
+ /** Combined duration of all finished and failed requests, in milliseconds. */
+ requestTotalDurationMillis: number;
+
+ /** Total number of settled requests (finished plus failed). */
+ requestsTotal: number;
+
+ /** Wall-clock runtime since capturing started, in milliseconds. */
+ crawlerRuntimeMillis: number;
+}
+
/**
* The statistics class provides an interface to collecting and logging run
* statistics for requests.
@@ -54,7 +133,7 @@ export interface PersistenceOptions {
*
* @category Crawlers
*/
-export class Statistics {
+export class Statistics implements IStatistics {
private static id = 0;
/**
@@ -102,7 +181,8 @@ export class Statistics {
}
/**
- * @internal
+ * Construct a statistics instance to pass to a crawler via its `statistics` option, e.g. to preconfigure
+ * persistence or error snapshots, share it across sequential runs, or subclass it to track extra fields.
*/
constructor(options: StatisticsOptions = {}) {
ow(
@@ -260,7 +340,7 @@ export class Statistics {
/**
* Calculate the current statistics
*/
- calculate() {
+ calculate(): CalculatedStatistics {
const {
requestsFailed,
requestsFinished,
@@ -287,6 +367,12 @@ export class Statistics {
* displaying the current state in predefined intervals
*/
async startCapturing() {
+ // A single instance drives one logging interval and one PERSIST_STATE listener, so a second concurrent
+ // capture (e.g. sharing one instance across crawlers running at once) would orphan the first. Fail loudly.
+ if (this.logInterval) {
+ throw new Error('Statistics.startCapturing() was already called - this instance is already capturing.');
+ }
+
this.keyValueStore ??= await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
if (this.state.crawlerStartedAt === null) {
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index 36ae9e14d..8f177f195 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -330,7 +330,11 @@ export class AdaptivePlaywrightCrawler<
private staticContextPipeline: ContextPipeline<CrawlingContext, ExtendedContext>;
private browserContextPipeline: ContextPipeline<CrawlingContext, ExtendedContext>;
private individualRequestHandlerTimeoutMillis: number;
- declare readonly stats: AdaptivePlaywrightCrawlerStatistics;
+
+ // The constructor always injects an `AdaptivePlaywrightCrawlerStatistics`, so narrowing the cast is sound.
+ override get stats(): AdaptivePlaywrightCrawlerStatistics {
+ return super.stats as AdaptivePlaywrightCrawlerStatistics;
+ }
private resultObjects = new WeakMap<CrawlingContext, RequestHandlerResult>();
private inFlightRenderingTypeDetections = 0;
@@ -344,7 +348,7 @@ export class AdaptivePlaywrightCrawler<
resultChecker,
shouldPropagateError,
resultComparator,
- statisticsOptions,
+ statistics,
preventDirectStorageAccess = true,
requestHandlerTimeoutSecs = 60,
errorHandler,
@@ -356,12 +360,25 @@ export class AdaptivePlaywrightCrawler<
...rest
} = options;
+ if (statistics !== undefined && !(statistics instanceof AdaptivePlaywrightCrawlerStatistics)) {
+ throw new Error(
+ 'AdaptivePlaywrightCrawler tracks extra fields on its own Statistics subclass and cannot use a ' +
+ 'plain `statistics` instance. Omit the option to let the crawler build its own.',
+ );
+ }
+
super({
...rest,
errorHandler,
failedRequestHandler,
requestHandler,
requestHandlerTimeoutSecs,
+ // Inject our subclass so the base tracks the extra adaptive fields instead of building a plain `Statistics`.
+ statistics:
+ statistics ??
+ new AdaptivePlaywrightCrawlerStatistics({
+ logMessage: `${AdaptivePlaywrightCrawler.name} request statistics:`,
+ }),
contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
});
this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
@@ -400,9 +417,7 @@ export class AdaptivePlaywrightCrawler<
// each hook's overrides at runtime regardless of the static type.
const staticCrawler = new CheerioCrawler({
...rest,
- statisticsOptions: {
- persistenceOptions: { enable: false },
- },
+ statistics: new Statistics({ persistenceOptions: { enable: false } }),
preNavigationHooks,
postNavigationHooks,
extendContext,
@@ -410,9 +425,7 @@ export class AdaptivePlaywrightCrawler<
const browserCrawler = new PlaywrightCrawler({
...rest,
- statisticsOptions: {
- persistenceOptions: { enable: false },
- },
+ statistics: new Statistics({ persistenceOptions: { enable: false } }),
preNavigationHooks: preNavigationHooks as unknown as PlaywrightHook[],
postNavigationHooks: postNavigationHooks as unknown as PlaywrightHook[],
extendContext,
@@ -428,11 +441,6 @@ export class AdaptivePlaywrightCrawler<
action: this.adaptPlaywrightContext.bind(this),
}) as unknown as ContextPipeline<CrawlingContext, ExtendedContext>;
- this.stats = new AdaptivePlaywrightCrawlerStatistics({
- logMessage: `${this.log.getOptions().prefix} request statistics:`,
- ...statisticsOptions,
- });
-
this.preventDirectStorageAccess = preventDirectStorageAccess;
}
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 3cc1103d3..2dbe040e6 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -10,6 +10,7 @@ import {
Configuration,
CriticalError,
defaultRoute,
+ ErrorTracker,
EventType,
KeyValueStore,
MissingRouteError,
@@ -22,8 +23,9 @@ import {
Router,
serviceLocator,
SessionPool,
+ Statistics,
} from '@crawlee/basic';
-import type { IConcurrencySystem } from '@crawlee/core';
+import type { CalculatedStatistics, IConcurrencySystem, IStatistics } from '@crawlee/core';
import { ConcurrencySystem, Dataset, MemoryStorageBackend, RequestState, SessionPool, Statistics } from '@crawlee/core';
import type { Dictionary, ISession, ProxyInfo } from '@crawlee/types';
import { RobotsTxtFile, sleep } from '@crawlee/utils';
@@ -2083,6 +2085,93 @@ describe('BasicCrawler', () => {
});
});
+ describe('statistics injection', () => {
+ it('uses the crawler-built default when no instance is supplied', async () => {
+ const crawler = new BasicCrawler({
+ requestList: await RequestList.open({ sources: [{ url: 'https://example.com' }] }),
+ requestHandler: async () => {},
+ });
+
+ expect(crawler.stats).toBeInstanceOf(Statistics);
+ });
+
+ it('records into a supplied Statistics instance', async () => {
+ const stats = new Statistics({ persistenceOptions: { enable: false } });
+ const crawler = new BasicCrawler({
+ requestList: await RequestList.open({ sources: [{ url: 'https://example.com' }] }),
+ statistics: stats,
+ requestHandler: async () => {},
+ });
+
+ expect(crawler.stats).toBe(stats);
+
+ await crawler.run();
+
+ expect(stats.state.requestsFinished).toBe(1);
+ });
+
+ it('drives a foreign IStatistics implementation through the interface alone', async () => {
+ // A fake that is not a `Statistics` (nor a subclass), recording the lifecycle the crawler drives.
+ const calls: string[] = [];
+ const customStats: IStatistics = {
+ errorTracker: new ErrorTracker(),
+ errorTrackerRetry: new ErrorTracker(),
+ state: { requestsFinished: 0 } as IStatistics['state'],
+ requestRetryHistogram: [],
+ startJob: () => calls.push('startJob'),
+ finishJob: () => {
+ customStats.state.requestsFinished += 1;
+ calls.push('finishJob');
+ },
+ failJob: () => {},
+ discardJob: () => {},
+ registerStatusCode: () => {},
+ calculate: () => ({}) as CalculatedStatistics,
+ startCapturing: async () => void calls.push('startCapturing'),
+ stopCapturing: async () => void calls.push('stopCapturing'),
+ // `persistState` is optional on `IStatistics`; this backend omits it.
+ };
+
+ const crawler = new BasicCrawler({ statistics: customStats, requestHandler: async () => {} });
+ expect(crawler.stats).toBe(customStats);
+
+ await crawler.run([{ url: 'https://example.com' }]);
+
+ expect(calls).toEqual(['startCapturing', 'startJob', 'finishJob', 'stopCapturing']);
+ expect(customStats.state.requestsFinished).toBe(1);
+ });
+
+ it('does not reset a supplied instance between runs, but does reset a default', async () => {
+ const stats = new Statistics({ persistenceOptions: { enable: false } });
+ const injectingCrawler = new BasicCrawler({
+ statistics: stats,
+ requestHandler: async () => {},
+ });
+ const resetSpy = vitest.spyOn(stats, 'reset');
+
+ await injectingCrawler.run([{ url: 'https://example.com', uniqueKey: 'run-1' }]);
+ await injectingCrawler.run([{ url: 'https://example.com', uniqueKey: 'run-2' }]);
+
+ // Two runs, one request each - the injected instance keeps accumulating instead of being wiped.
+ expect(resetSpy).not.toHaveBeenCalled();
+ expect(stats.state.requestsFinished).toBe(2);
+
+ const owningCrawler = new BasicCrawler({
+ requestHandler: async () => {},
+ });
+ // `crawler.stats` is typed as `IStatistics`, which omits the owned-only `reset()` - the default is a
+ // concrete `Statistics`, so cast to spy on it.
+ const ownedResetSpy = vitest.spyOn(owningCrawler.stats as Statistics, 'reset');
+
+ await owningCrawler.run([{ url: 'https://example.com', uniqueKey: 'owned-1' }]);
+ await owningCrawler.run([{ url: 'https://example.com', uniqueKey: 'owned-2' }]);
+
+ // A crawler-owned default is wiped at the start of each run.
+ expect(ownedResetSpy).toHaveBeenCalled();
+ expect(owningCrawler.stats.state.requestsFinished).toBe(1);
+ });
+ });
+
describe('proxyConfiguration', () => {
it('assigns a proxyInfo from the proxyConfiguration to each Session and exposes it on the context', async () => {
const proxyUrls = [0, 1, 2].map((n) => `http://proxy.example.com:${1000 + n}`);
diff --git a/test/core/crawlers/statistics.test.ts b/test/core/crawlers/statistics.test.ts
index 924b5a092..b0cfc1204 100644
--- a/test/core/crawlers/statistics.test.ts
+++ b/test/core/crawlers/statistics.test.ts
@@ -349,6 +349,17 @@ describe('Statistics', () => {
expect(stats.requestRetryHistogram).toEqual([]);
});
+ test('should throw when startCapturing is called while already capturing', async () => {
+ await stats.startCapturing();
+
+ await expect(stats.startCapturing()).rejects.toThrow('already capturing');
+
+ await stats.stopCapturing();
+ // stopCapturing tears the interval down, so capturing can be resumed on the same instance.
+ await expect(stats.startCapturing()).resolves.toBeUndefined();
+ await stats.stopCapturing();
+ });
+
describe('explicit id option', () => {
test('statistics with same explicit id should share persisted state', async () => {
const stats1 = new Statistics({ id: 'shared-stats' });
diff --git a/test/e2e/cheerio-error-snapshot/actor/main.js b/test/e2e/cheerio-error-snapshot/actor/main.js
index 988cf66f4..0ea80f4d7 100644
--- a/test/e2e/cheerio-error-snapshot/actor/main.js
+++ b/test/e2e/cheerio-error-snapshot/actor/main.js
@@ -1,4 +1,5 @@
import { CheerioCrawler } from '@crawlee/cheerio';
+import { Statistics } from '@crawlee/core';
import { sleep } from '@crawlee/utils';
import { Actor } from 'apify';
@@ -22,9 +23,7 @@ await Actor.main(async () => {
const crawler = new CheerioCrawler({
requestHandlerTimeoutSecs: 2,
maxRequestRetries: 0,
- statisticsOptions: {
- saveErrorSnapshots: true,
- },
+ statistics: new Statistics({ saveErrorSnapshots: true }),
async requestHandler({ $, request, log }) {
const {
userData: { label },
diff --git a/test/e2e/puppeteer-error-snapshot/actor/main.js b/test/e2e/puppeteer-error-snapshot/actor/main.js
index 6ee43c8ef..1f7d5f0cb 100644
--- a/test/e2e/puppeteer-error-snapshot/actor/main.js
+++ b/test/e2e/puppeteer-error-snapshot/actor/main.js
@@ -1,3 +1,4 @@
+import { Statistics } from '@crawlee/core';
import { PuppeteerCrawler } from '@crawlee/puppeteer';
import { sleep } from '@crawlee/utils';
import { Actor } from 'apify';
@@ -22,9 +23,7 @@ await Actor.main(async () => {
const crawler = new PuppeteerCrawler({
requestHandlerTimeoutSecs: 15,
maxRequestRetries: 0,
- statisticsOptions: {
- saveErrorSnapshots: true,
- },
+ statistics: new Statistics({ saveErrorSnapshots: true }),
async requestHandler({ request, log, page }) {
const {
userData: { label },