Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor: Use RecoverableState to implement Statistics persistence (#4014)
Jan Bucharf08e8d6a8c60
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.mdindex d1c7eab32..d32341b16 100644--- a/docs/public-api/crawlee-core.api.md+++ b/docs/public-api/crawlee-core.api.md@@ -847,7 +847,7 @@ export interface IStatistics {readonly errorTrackerRetry: ErrorTracker;failJob(id: number | string, retryCount: number): void;finishJob(id: number | string, retryCount: number): void;- persistState?(options?: PersistenceOptions): Promise<void>;+ persistState?(): Promise<void>;registerStatusCode(code: number): void;readonly requestRetryHistogram: number[];startCapturing(): Promise<void>;@@ -1930,15 +1930,22 @@ export class StateValidationError extends Error {}// @public-export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {+export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt' | 'crawlerStartedAt' | 'crawlerFinishedAt' | 'requestMinDurationMillis' | 'requestsFailedPerMinute' | 'requestsFinishedPerMinute' | 'requestRetryHistogram' | 'instanceStart'> {// (undocumented)+ crawlerFinishedAt: string | null;crawlerLastStartTimestamp: number;+ crawlerStartedAt: string | null;// (undocumented)- requestAvgFailedDurationMillis: number;+ requestAvgFailedDurationMillis: number | null;// (undocumented)- requestAvgFinishedDurationMillis: number;+ requestAvgFinishedDurationMillis: number | null;// (undocumented)- requestRetryHistogram: number[];+ requestMinDurationMillis: number | null;+ requestRetryHistogram: (number | null)[];+ // (undocumented)+ requestsFailedPerMinute: number | null;+ // (undocumented)+ requestsFinishedPerMinute: number | null;// (undocumented)requestsTotal: number;// (undocumented)@@ -1953,22 +1960,21 @@ export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersexport class Statistics implements IStatistics {constructor(options?: StatisticsOptions);calculate(): CalculatedStatistics;+ protected defaultState(): StatisticState;+ protected deserializeState(persistedState: StatisticPersistedState): StatisticState;readonly errorTracker: ErrorTracker;readonly errorTrackerRetry: ErrorTracker;readonly id: string;- // (undocumented)- protected keyValueStore?: KeyValueStore;- protected maybeLoadStatistics(): Promise<void>;- persistState(options?: PersistenceOptions): Promise<void>;+ persistState(): Promise<void>;// (undocumented)protected readonly persistStateKey: string;registerStatusCode(code: number): void;- readonly requestRetryHistogram: number[];+ get requestRetryHistogram(): number[];reset(): void;- // (undocumented)- resetStore(options?: PersistenceOptions): Promise<void>;+ resetStore(): Promise<void>;+ protected serializeState(state: StatisticState): StatisticPersistedState;startCapturing(): Promise<void>;- state: StatisticState;+ get state(): StatisticState;stopCapturing(): Promise<void>;toJSON(): StatisticPersistedState;}@@ -1994,10 +2000,12 @@ export interface StatisticState {crawlerStartedAt: Date | string | null;// (undocumented)errors: Record<string, unknown>;+ instanceStart: number;// (undocumented)requestMaxDurationMillis: number;// (undocumented)requestMinDurationMillis: number;+ requestRetryHistogram: number[];// (undocumented)requestsFailed: number;// (undocumented)diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.mdindex 686e7a656..c69fff43c 100644--- a/docs/public-api/crawlee-playwright.api.md+++ b/docs/public-api/crawlee-playwright.api.md@@ -53,8 +53,8 @@ import type { RouterRoutes as RouterRoutes_2 } from '@crawlee/core';import type { RouteSchemas } from '@crawlee/browser';import type { RoutesFromSchemas } from '@crawlee/browser';import type { SkippedRequestCallback } from '@crawlee/browser';+import type { StatisticPersistedState } from '@crawlee/core';import { Statistics } from '@crawlee/core';-import type { StatisticsOptions } from '@crawlee/core';import type { StatisticState } from '@crawlee/core';import type { StorageTransactionView } from '@crawlee/core';import { StorageWritePolicy } from '@crawlee/browser';@@ -124,16 +124,26 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<shouldPropagateError?: (error: Error, context: PlaywrightCrawlingContext) => Awaitable<boolean>;}+// Not exported by the entry point; reachable only as a referenced type.+// @public (undocumented)+interface AdaptivePlaywrightCrawlerPersistedStatisticState extends StatisticPersistedState {+ // (undocumented)+ browserRequestHandlerRuns?: number;+ // (undocumented)+ httpOnlyRequestHandlerRuns?: number;+ // (undocumented)+ renderingTypeMispredictions?: number;+}+// Not exported by the entry point; reachable only as a referenced type.// @public (undocumented)class AdaptivePlaywrightCrawlerStatistics extends Statistics {- constructor(options?: StatisticsOptions);// (undocumented)- protected maybeLoadStatistics(): Promise<void>;+ protected defaultState(): AdaptivePlaywrightCrawlerStatisticState;// (undocumented)- reset(): void;+ protected deserializeState(persistedState: AdaptivePlaywrightCrawlerPersistedStatisticState): AdaptivePlaywrightCrawlerStatisticState;// (undocumented)- state: AdaptivePlaywrightCrawlerStatisticState;+ get state(): AdaptivePlaywrightCrawlerStatisticState;// (undocumented)trackBrowserRequestHandlerRun(): void;// (undocumented)diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex 1919f1429..41053bb7a 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -75,6 +75,10 @@ Support for older TypeScript versions was dropped. Crawlee ships compiled JavaScPreviously, we kept the dependency on cheerio locked to the latest RC version, since there were many breaking changes introduced in v1.0. This release bumps cheerio to the stable v1. Also, we now use the default `parse5` internally.+### Zod 4.1+ required++`@crawlee/core` used to accept zod 3 as well; it now needs 4.1 or newer, for the codecs it validates persisted state with. Zod is an ordinary dependency rather than a peer, so a project pinned to zod 3 keeps working — it just ends up with both versions installed.+### Deprecated crawler options are removedThe crawler following options are removed:@@ -700,7 +704,7 @@ The change spans, among others:- **`RequestList`** — all `_`-prefixed helpers (`addFetchedRequests`, `addPersistedRequests`, `addRequest`, `addRequestsFromSources`, `ensureInProgress`, `ensureIsInitialized`, `ensureUniqueKeyValid`, `fetchRequestsFromUrl`, `getPersistedState`, `loadStateAndPersistedRequests`, `persistRequests`, `restoreState`)- **`RequestQueue`** — `proxyConfiguration`, `requestCache`, `requestSeenCache`, `queuePausedForMigration`, `inProgressRequestBatchCount`, `expectedRequestProcessingSecs`, `httpClient`, `events`, and the helpers `cacheRequest`, `fetchRequestsFromUrl`, `addFetchedRequests` (`id`, `name`, `backend`, `log` are now `readonly`)- **`ProxyConfiguration`** — `nextCustomUrlIndex`, `proxyUrls`, `newUrlFunction`, and the helpers `handleProxyUrlsList`, `callNewUrlFunction`, `throwCannotCombineCustomMethods`, `throwNoOptionsProvided` (the internal `log` field and `usedProxyUrls` map are removed; `isManInTheMiddle` is now `readonly`)-- **`Statistics`** — `saveRetryCountForJob`, `teardown` (`errorTracker`, `errorTrackerRetry` are now `readonly`)+- **`Statistics`** — `saveRetryCountForJob`, `teardown`, `keyValueStore` (`errorTracker`, `errorTrackerRetry` are now `readonly`, and `state` / `requestRetryHistogram` are getters)- **`SystemStatus`** — `isSystemIdle`- **`Router`** — the constructor is now `private`; use the static `Router.create()` factory- **`BaseHttpClient`** — `log` (subclasses receive it via the constructor `logger` option instead of reading `this.log`)@@ -709,7 +713,6 @@ The change spans, among others:- **`RenderingTypePredictor`** — `calculateFeatureVector`, `retrain`- **`BrowserCrawler`** — `navigationTimeoutMillis`, `preNavigationHooks`, `postNavigationHooks`, `saveResponseCookies` (now `private readonly`; configure them through the constructor options as before), and the helpers `isRequestBlocked`, `applyCookies` (was `_applyCookies`), `handleNavigationTimeout` (was `_handleNavigationTimeout`), `throwIfProxyError` (was `_throwIfProxyError`)- **`BrowserLauncher`** — the helpers `getChromeExecutablePath`, `getTypicalChromeExecutablePath`, `validateProxyUrlProtocol` (were `_`-prefixed). `getDefaultHeadlessOption` (was `_getDefaultHeadlessOption`) stays `protected` — it is an override point (`PuppeteerLauncher` overrides it) — but lost its underscore prefix-- **`Statistics.maybeLoadStatistics`** (was `_maybeLoadStatistics`) — stays `protected` (it is overridden by `AdaptivePlaywrightCrawler`'s statistics), but lost its underscore prefix; rename any `super._maybeLoadStatistics()` overrides accordingly- **`RobotsTxtFile.load` and `Sitemap.parse`** — internal static factory helpers, now `private` (use the public `RobotsTxtFile.from` / `Sitemap.load` / `Sitemap.fromXmlString` entry points)- Various internal fields on `BrowserController` (`id`, `browserPlugin`, `log`) and `BrowserPlugin` (`name`, `library`, `launchOptions`, `proxyUrl`, `userDataDir`, `browserPerProxy`, `ignoreProxyCertificate`, `log`) are now `readonly`@@ -1024,6 +1027,12 @@ Omit the option and the crawler builds its own default, exactly as before. A supThe option accepts the built-in `Statistics` or any object implementing the new `IStatistics` interface, so a fully custom statistics backend can be plugged in without subclassing. The crawler exposes it as `crawler.stats` typed as `IStatistics`.+### The `Statistics` persistence lifecycle is stricter++`persistState()` and `resetStore()` no longer take `PersistenceOptions` — persistence is enabled or disabled once, in the constructor. `resetStore()` throws while the instance is capturing, where the next `PERSIST_STATE` event would write the record straight back; call it before `startCapturing()` or after `stopCapturing()`. And `reset()` only resets the counters — it no longer stops an ongoing capture, which `stopCapturing()` does.++A persisted record is also validated on load now. One that does not match the expected shape is discarded whole, with a warning, and the statistics start from scratch — where v3 would copy the malformed values into the live state and let them corrupt every later increment.+## Only if you wrote a custom HTTP client or used `got-scraping` directlyApplies when you implemented `BaseHttpClient` yourself, or imported `gotScraping` from `@crawlee/utils`.diff --git a/packages/core/package.json b/packages/core/package.jsonindex 981456452..f5ea38df3 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -69,6 +69,6 @@"tough-cookie": "^6.0.0","tslib": "^2.8.1","type-fest": "^4.41.0",- "zod": "^3.24.0 || ^4.0.0"+ "zod": "^4.1.0"}}diff --git a/packages/core/src/crawlers/statistics.ts b/packages/core/src/crawlers/statistics.tsindex 24c9f2a20..05c8789e2 100644--- a/packages/core/src/crawlers/statistics.ts+++ b/packages/core/src/crawlers/statistics.ts@@ -1,10 +1,10 @@import ow from 'ow';+import { z } from 'zod';-import type { EventManager } from '../events/event_manager.js';-import { EventType } from '../events/event_manager.js';import type { CrawleeLogger } from '../log.js';+import { RecoverableState } from '../recoverable_state.js';import { serviceLocator } from '../service_locator.js';-import { KeyValueStore } from '../storages/key_value_store.js';+import type { KeyValueStore } from '../storages/key_value_store.js';import { ErrorTracker } from './error_tracker.js';/**@@ -33,6 +33,131 @@ const errorTrackerConfig = {showFullMessage: false,};+/**+ * The persisted record, in the order it is written - the schema rebuilds the object on the way out, so the field+ * order here *is* the record's field order (guarded by a test).+ *+ * JSON has no infinity, so the three fields that are `Infinity` until the first request settles are written as+ * `null`. Both {@apilink Statistics.serializeState} and {@apilink Statistics.deserializeState} run through this,+ * which is what keeps them describing the same record.+ *+ * Nothing is optional on purpose: `serializeState` has always written every field, so a record missing one is not+ * one of ours and is discarded whole rather than partially trusted - a counter restored as a string would poison+ * every later increment.+ */+const persistedStatisticState = z+ .object({+ requestsFinished: z.number(),+ requestsFailed: z.number(),+ requestsRetries: z.number(),+ requestsFailedPerMinute: z.number().nullable(),+ requestsFinishedPerMinute: z.number().nullable(),+ requestMinDurationMillis: z.number().nullable(),+ requestMaxDurationMillis: z.number(),+ requestTotalFailedDurationMillis: z.number(),+ requestTotalFinishedDurationMillis: z.number(),+ crawlerStartedAt: z.string().nullable(),+ crawlerFinishedAt: z.string().nullable(),+ statsPersistedAt: z.string(),+ crawlerRuntimeMillis: z.number(),+ crawlerLastStartTimestamp: z.number(),+ // A retry count that never occurred leaves a hole in the live histogram, written out as a `null`. We+ // once saw a record whose histogram was not an array at all and crashed the crawler on load.+ requestRetryHistogram: z.array(z.number().nullable()),+ statsId: z.string(),+ requestAvgFailedDurationMillis: z.number().nullable(),+ requestAvgFinishedDurationMillis: z.number().nullable(),+ requestTotalDurationMillis: z.number(),+ requestsTotal: z.number(),+ requestsWithStatusCode: z.record(z.string(), z.number()),+ errors: z.record(z.string(), z.unknown()),+ retryErrors: z.record(z.string(), z.unknown()),+ })+ // A subclass tracking extra fields spreads them into the record; they are none of this schema's business,+ // but they must not be dropped on the way through it.+ .catchall(z.unknown());++/** `Infinity` is what the statistics use for "nothing to average yet"; JSON has only `null` for it. */+function finiteOrNull(value: number): number | null {+ return Number.isFinite(value) ? value : null;+}++/**+ * The conversion between the live state and the record above, in both directions.+ *+ * Built per instance rather than kept as a constant because a record carries three things the state does not: the+ * instance `id`, the derived aggregates of the overridable {@apilink Statistics.calculate}, and - on the way back -+ * the fields that are rebuilt from {@apilink Statistics.defaultState} rather than restored, the error trackers+ * among them.+ *+ * The model side is deliberately opaque: zod rebuilds what it validates, and `state.errors` has to stay the very+ * object the error trackers write into, not a copy of it.+ */+function buildStatisticStateCodec(statistics: {+ statsId: string;+ defaultState: () => StatisticState;+ calculate: () => CalculatedStatistics;+}) {+ return z.codec(persistedStatisticState, z.custom<StatisticState>(), {+ decode: (record) => ({+ ...statistics.defaultState(),+ requestsFinished: record.requestsFinished,+ requestsFailed: record.requestsFailed,+ requestsRetries: record.requestsRetries,+ requestTotalFailedDurationMillis: record.requestTotalFailedDurationMillis,+ requestTotalFinishedDurationMillis: record.requestTotalFinishedDurationMillis,+ // Restoring the `null` as-is would make every later `duration < min` comparison fail, leaving the+ // minimum `null` for the rest of the run.+ requestMinDurationMillis: record.requestMinDurationMillis ?? Infinity,+ requestMaxDurationMillis: record.requestMaxDurationMillis,+ crawlerRuntimeMillis: record.crawlerRuntimeMillis,+ // A `null` stands for the zero requests that reached that retry count - restore it as such.+ requestRetryHistogram: record.requestRetryHistogram.map((count) => count ?? 0),+ // The record keeps ISO strings, the live state keeps `Date`s.+ crawlerStartedAt: record.crawlerStartedAt === null ? null : new Date(record.crawlerStartedAt),+ crawlerFinishedAt: record.crawlerFinishedAt === null ? null : new Date(record.crawlerFinishedAt),+ statsPersistedAt: new Date(record.statsPersistedAt),+ // Rebased so that the runtime reported by `calculate()` spans the migration instead of restarting.+ instanceStart:+ Date.now() - (new Date(record.statsPersistedAt).getTime() - record.crawlerLastStartTimestamp),+ }),+ encode: (state) => {+ const { requestsWithStatusCode, errors, retryErrors, requestRetryHistogram, instanceStart, ...counters } =+ state;+ // Every rate and average `calculate()` derives is `Infinity` until the run is long enough, or until+ // something has finished or failed, to divide by.+ const {+ requestAvgFailedDurationMillis,+ requestAvgFinishedDurationMillis,+ requestsFailedPerMinute,+ requestsFinishedPerMinute,+ ...aggregates+ } = statistics.calculate();++ return {+ ...counters,+ requestMinDurationMillis: finiteOrNull(state.requestMinDurationMillis),+ crawlerStartedAt: state.crawlerStartedAt ? new Date(state.crawlerStartedAt).toISOString() : null,+ crawlerFinishedAt: state.crawlerFinishedAt ? new Date(state.crawlerFinishedAt).toISOString() : null,+ statsPersistedAt: new Date().toISOString(),+ crawlerLastStartTimestamp: instanceStart,+ // `Array.from`, not `map` - a hole left by a retry count no request reached is skipped by `map`+ // and would stay a hole, which is not something the record can carry.+ requestRetryHistogram: Array.from(requestRetryHistogram, (count) => count ?? null),+ statsId: statistics.statsId,+ ...aggregates,+ requestAvgFailedDurationMillis: finiteOrNull(requestAvgFailedDurationMillis),+ requestAvgFinishedDurationMillis: finiteOrNull(requestAvgFinishedDurationMillis),+ requestsFailedPerMinute: finiteOrNull(requestsFailedPerMinute),+ requestsFinishedPerMinute: finiteOrNull(requestsFinishedPerMinute),+ requestsWithStatusCode,+ errors,+ retryErrors,+ };+ },+ });+}+/*** Persistence-related options to control how and when crawler's data gets persisted.*/@@ -96,7 +221,7 @@ export interface IStatistics {* Persists the current state to the key-value store. Optional - the crawler calls it on migration, but a backend* with no persistence of its own can omit it.*/- persistState?(options?: PersistenceOptions): Promise<void>;+ persistState?(): Promise<void>;}/** The derived aggregates computed by {@apilink IStatistics.calculate} from the current {@apilink StatisticState}. */@@ -152,33 +277,27 @@ export class Statistics implements IStatistics {*/readonly id: string;- /**- * Current statistic state used for doing calculations on {@apilink Statistics.calculate} calls- */- state!: StatisticState;-- /**- * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on- */- readonly requestRetryHistogram: number[] = [];-- protected keyValueStore?: KeyValueStore = undefined;protected readonly persistStateKey: string;+ readonly #stateCodec: ReturnType<typeof buildStatisticStateCodec>;+ readonly #recoverableState: RecoverableState<StatisticState, StatisticPersistedState>;#logIntervalMillis: number;#logMessage: string;- #listener: () => Promise<void>;#requestsInProgress = new Map<number | string, Job>();private readonly log: CrawleeLogger;- #instanceStart!: number;#logInterval: unknown;- #events?: EventManager;- #persistenceOptions: PersistenceOptions;- private get events(): EventManager {- if (!this.#events) {- this.#events = serviceLocator.getEventManager();- }- return this.#events;+ /**+ * Current statistic state used for doing calculations on {@apilink Statistics.calculate} calls+ */+ get state(): StatisticState {+ return this.#recoverableState.currentValue;+ }++ /**+ * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on+ */+ get requestRetryHistogram(): number[] {+ return this.state.requestRetryHistogram;}/**@@ -218,22 +337,47 @@ export class Statistics implements IStatistics {this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });this.#logIntervalMillis = logIntervalSecs * 1000;this.#logMessage = logMessage;- this.keyValueStore = keyValueStore;- this.#listener = this.persistState.bind(this);- this.#persistenceOptions = persistenceOptions;++ // Late-bound on purpose - both hooks are override points, and a subclass's must be the ones that run.+ this.#stateCodec = buildStatisticStateCodec({+ statsId: this.id,+ defaultState: () => this.defaultState(),+ calculate: () => this.calculate(),+ });++ this.#recoverableState = new RecoverableState({+ persistStateKey: this.persistStateKey,+ persistenceEnabled: persistenceOptions.enable,+ keyValueStore,+ logger: this.log,+ defaultState: () => this.defaultState(),+ serialize: (state) => this.serializeState(state),+ deserialize: (persistedState) => this.deserializeState(persistedState),+ });// initialize by "resetting"this.reset();}/**- * Set the current statistic instance to pristine values+ * Set the current statistic instance to pristine values.+ *+ * The persisted record is left alone - use {@apilink Statistics.resetStore} to clear that as well.*/reset() {this.errorTracker.reset();this.errorTrackerRetry.reset();+ this.#recoverableState.reset();+ this.#requestsInProgress.clear();+ }- this.state = {+ /**+ * The pristine state a new instance starts with and {@apilink Statistics.reset} restores.+ *+ * A subclass tracking extra fields declares their initial values here.+ */+ protected defaultState(): StatisticState {+ return {requestsFinished: 0,requestsFailed: 0,requestsRetries: 0,@@ -248,30 +392,21 @@ export class Statistics implements IStatistics {statsPersistedAt: null,crawlerRuntimeMillis: 0,requestsWithStatusCode: {},+ // Aliases, not copies - the trackers keep writing into these objects.errors: this.errorTracker.result,retryErrors: this.errorTrackerRetry.result,+ requestRetryHistogram: [],+ instanceStart: Date.now(),};-- this.requestRetryHistogram.length = 0;- this.#requestsInProgress.clear();- this.#instanceStart = Date.now();-- this.teardown();}/**- * @param options - Override the persistence options provided in the constructor+ * Clear the persisted statistics record, leaving the in-memory state alone.+ *+ * Throws while capturing - the next PERSIST_STATE event would write the record straight back.*/- async resetStore(options?: PersistenceOptions) {- if (!this.#persistenceOptions.enable && !options?.enable) {- return;- }-- if (!this.keyValueStore) {- return;- }-- await this.keyValueStore.setValue(this.persistStateKey, null);+ async resetStore() {+ await this.#recoverableState.resetStore();}/**@@ -348,7 +483,7 @@ export class Statistics implements IStatistics {requestTotalFailedDurationMillis,requestTotalFinishedDurationMillis,} = this.state;- const totalMillis = Date.now() - this.#instanceStart;+ const totalMillis = Date.now() - this.state.instanceStart;const totalMinutes = totalMillis / 1000 / 60;return {@@ -374,17 +509,13 @@ export class Statistics implements IStatistics {throw new Error('Statistics.startCapturing() was already called - this instance is already capturing.');}- this.keyValueStore ??= await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });+ await this.#recoverableState.initialize();+ // After the load, so that a restored record keeps the timestamp of the run it belongs to.if (this.state.crawlerStartedAt === null) {this.state.crawlerStartedAt = new Date();}- if (this.#persistenceOptions.enable) {- await this.maybeLoadStatistics();- this.events.on(EventType.PERSIST_STATE, this.#listener);- }-this.#logInterval = setInterval(() => {this.log.info(this.#logMessage, {...this.calculate(),@@ -397,11 +528,11 @@ export class Statistics implements IStatistics {* Stops logging and remove event listeners, then persist*/async stopCapturing() {- this.teardown();+ this.#stopLogging();this.state.crawlerFinishedAt = new Date();- await this.persistState();+ await this.#recoverableState.teardown();}private saveRetryCountForJob(retryCount: number) {@@ -411,115 +542,66 @@ export class Statistics implements IStatistics {}/**- * Persist internal state to the key value store- * @param options - Override the persistence options provided in the constructor+ * Persist internal state to the key value store.+ *+ * Statistics are bookkeeping - a store that refuses the write is worth a warning, not a failed crawl. The+ * crawler calls this from its migration handler, where a rejection would go unhandled.*/- async persistState(options?: PersistenceOptions) {- if (!this.#persistenceOptions.enable && !options?.enable) {- return;- }-- // this might be called before startCapturing was called without using await, should not crash- if (!this.keyValueStore) {- return;- }-- this.log.debug('Persisting state', { persistStateKey: this.persistStateKey });-- await this.keyValueStore- .setValue(this.persistStateKey, this.toJSON())+ async persistState() {+ await this.#recoverableState+ .persistState().catch((error) =>this.log.warning(`Failed to persist the statistics to ${this.persistStateKey}`, { error }),);}/**- * Loads the current statistic from the key value store if any+ * Rebuilds the state from a persisted record.+ *+ * A subclass tracking extra fields restores them here, on top of the result of `super.deserializeState()`.*/- protected async maybeLoadStatistics() {- // this might be called before startCapturing was called without using await, should not crash- if (!this.keyValueStore) {- return;- }-- const savedState = await this.keyValueStore.getValue<StatisticPersistedState>(this.persistStateKey);-- if (!savedState) return;-- // We saw a run where the requestRetryHistogram was not iterable and crashed- // the crawler. Adding some logging to monitor this problem in the future.- if (!Array.isArray(savedState.requestRetryHistogram)) {- this.log.warning('Received invalid state from Key-value store.', {+ protected deserializeState(persistedState: StatisticPersistedState): StatisticState {+ // The cast is the index signature the `catchall` puts on the schema and an interface cannot have - the+ // record is an unvalidated blob off the key-value store either way, which is what the decode is for.+ const restored = z.safeDecode(this.#stateCodec, persistedState as z.input<typeof persistedStatisticState>);++ if (!restored.success) {+ // Statistics are bookkeeping - a record that cannot be made sense of is worth a warning and a fresh+ // start, not a failed crawl.+ this.log.warning('Received invalid state from Key-value store, starting the statistics from scratch.', {persistStateKey: this.persistStateKey,- state: savedState,+ issues: restored.error.issues,});++ return this.defaultState();}this.log.debug('Recreating state from KeyValueStore', { persistStateKey: this.persistStateKey });- // the `requestRetryHistogram` array might be very large, we could end up with- // `RangeError: Maximum call stack size exceeded` if we use `a.push(...b)`- savedState.requestRetryHistogram.forEach((idx) => this.requestRetryHistogram.push(idx));- this.state.requestsFinished = savedState.requestsFinished;- this.state.requestsFailed = savedState.requestsFailed;- this.state.requestsRetries = savedState.requestsRetries;-- this.state.requestTotalFailedDurationMillis = savedState.requestTotalFailedDurationMillis;- this.state.requestTotalFinishedDurationMillis = savedState.requestTotalFinishedDurationMillis;- this.state.requestMinDurationMillis = savedState.requestMinDurationMillis;- this.state.requestMaxDurationMillis = savedState.requestMaxDurationMillis;- // persisted state uses ISO date strings- this.state.crawlerFinishedAt = savedState.crawlerFinishedAt ? new Date(savedState.crawlerFinishedAt) : null;- this.state.crawlerStartedAt = savedState.crawlerStartedAt ? new Date(savedState.crawlerStartedAt) : null;- this.state.statsPersistedAt = savedState.statsPersistedAt ? new Date(savedState.statsPersistedAt) : null;- this.state.crawlerRuntimeMillis = savedState.crawlerRuntimeMillis;- this.#instanceStart = Date.now() - (+this.state.statsPersistedAt! - savedState.crawlerLastStartTimestamp);-- this.log.debug('Loaded from KeyValueStore');+ return { ...this.defaultState(), ...restored.data };}- private teardown(): void {- // this can be called before a call to startCapturing happens (or in a 'finally' block)- // Only unsubscribe if event manager was already resolved — avoid eagerly resolving it- // (e.g. during the constructor's reset() call, which would capture the wrong context)- this.#events?.off(EventType.PERSIST_STATE, this.#listener);-+ #stopLogging(): void {if (this.#logInterval) {clearInterval(this.#logInterval as number);this.#logInterval = null;}}+ /**+ * Builds the record written to the key value store, merging in the derived aggregates so that a consumer+ * reading the record does not have to reconstruct them.+ */+ protected serializeState(state: StatisticState): StatisticPersistedState {+ return z.encode(this.#stateCodec, state);+ }+/*** Make this class serializable when called with `JSON.stringify(statsInstance)` directly* or through `keyValueStore.setValue('KEY', statsInstance)`*/toJSON(): StatisticPersistedState {- // merge all the current state information that can be used from the outside- // without the need to reconstruct for the sake of stats.calculate()- // omit duplicated information- const result = {- ...this.state,- crawlerLastStartTimestamp: this.#instanceStart,- crawlerFinishedAt: this.state.crawlerFinishedAt- ? new Date(this.state.crawlerFinishedAt).toISOString()- : null,- crawlerStartedAt: this.state.crawlerStartedAt ? new Date(this.state.crawlerStartedAt).toISOString() : null,- requestRetryHistogram: this.requestRetryHistogram,- statsId: this.id,- statsPersistedAt: new Date().toISOString(),- ...this.calculate(),- };-- Reflect.deleteProperty(result, 'requestsWithStatusCode');- Reflect.deleteProperty(result, 'errors');- Reflect.deleteProperty(result, 'retryErrors');-- result.requestsWithStatusCode = this.state.requestsWithStatusCode;- result.errors = this.state.errors;- result.retryErrors = this.state.retryErrors;-- return result;+ return this.serializeState(this.state);}}@@ -574,17 +656,38 @@ export interface StatisticsOptions {}/**- * Format of the persisted stats+ * Format of the persisted stats.+ *+ * The `null`s are `Infinity` on the way out - JSON has no infinity, so a record written before anything+ * finished or failed carries a `null` in its place.*/-export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {- requestRetryHistogram: number[];+export interface StatisticPersistedState extends Omit<+ StatisticState,+ | 'statsPersistedAt'+ | 'crawlerStartedAt'+ | 'crawlerFinishedAt'+ | 'requestMinDurationMillis'+ | 'requestsFailedPerMinute'+ | 'requestsFinishedPerMinute'+ | 'requestRetryHistogram'+ | 'instanceStart'+> {statsId: string;- requestAvgFailedDurationMillis: number;- requestAvgFinishedDurationMillis: number;+ requestsFailedPerMinute: number | null;+ requestsFinishedPerMinute: number | null;+ /** ISO strings - the live state keeps these as `Date`s. */+ crawlerStartedAt: string | null;+ crawlerFinishedAt: string | null;+ statsPersistedAt: string;+ requestMinDurationMillis: number | null;+ /** A retry count that no request ever reached leaves a `null` here. */+ requestRetryHistogram: (number | null)[];+ requestAvgFailedDurationMillis: number | null;+ requestAvgFinishedDurationMillis: number | null;requestTotalDurationMillis: number;requestsTotal: number;+ /** {@apilink StatisticState.instanceStart} of the run that wrote the record. */crawlerLastStartTimestamp: number;- statsPersistedAt: string;}/**@@ -607,4 +710,13 @@ export interface StatisticState {errors: Record<string, unknown>;retryErrors: Record<string, unknown>;requestsWithStatusCode: Record<string, number>;++ /** Retries histogram - index `i` holds the number of requests that finished after `i` retries. */+ requestRetryHistogram: number[];++ /**+ * When the current capture window started, as a `Date.now()` timestamp. Rebased on load so that the runtime+ * reported by {@apilink Statistics.calculate} spans a migration rather than restarting from zero.+ */+ instanceStart: number;}diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex 71d3e6af1..9f3e177b8 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -22,7 +22,6 @@ import type {RestrictedCrawlingContext,RouterRoutes,StatisticPersistedState,- StatisticsOptions,StatisticState,StorageTransaction,StorageTransactionView,@@ -71,33 +70,28 @@ interface AdaptivePlaywrightCrawlerPersistedStatisticState extends StatisticPers}class AdaptivePlaywrightCrawlerStatistics extends Statistics {- override state: AdaptivePlaywrightCrawlerStatisticState = null as any; // this needs to be assigned for a valid override, but the initialization is done by a reset() call from the parent constructor-- constructor(options: StatisticsOptions = {}) {- super(options);- this.reset();+ override get state(): AdaptivePlaywrightCrawlerStatisticState {+ return super.state;}- override reset(): void {- super.reset();- this.state.httpOnlyRequestHandlerRuns = 0;- this.state.browserRequestHandlerRuns = 0;- this.state.renderingTypeMispredictions = 0;+ protected override defaultState(): AdaptivePlaywrightCrawlerStatisticState {+ return {+ ...super.defaultState(),+ httpOnlyRequestHandlerRuns: 0,+ browserRequestHandlerRuns: 0,+ renderingTypeMispredictions: 0,+ };}- protected override async maybeLoadStatistics(): Promise<void> {- await super.maybeLoadStatistics();- const savedState = await this.keyValueStore?.getValue<AdaptivePlaywrightCrawlerPersistedStatisticState>(- this.persistStateKey,- );-- if (!savedState) {- return;- }-- this.state.httpOnlyRequestHandlerRuns = savedState.httpOnlyRequestHandlerRuns;- this.state.browserRequestHandlerRuns = savedState.browserRequestHandlerRuns;- this.state.renderingTypeMispredictions = savedState.renderingTypeMispredictions;+ protected override deserializeState(+ persistedState: AdaptivePlaywrightCrawlerPersistedStatisticState,+ ): AdaptivePlaywrightCrawlerStatisticState {+ return {+ ...super.deserializeState(persistedState),+ httpOnlyRequestHandlerRuns: persistedState.httpOnlyRequestHandlerRuns,+ browserRequestHandlerRuns: persistedState.browserRequestHandlerRuns,+ renderingTypeMispredictions: persistedState.renderingTypeMispredictions,+ };}trackHttpOnlyRequestHandlerRun(): void {diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 9ea7e68ab..e206172d3 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -526,7 +526,7 @@ importers:specifier: ^4.41.0version: 4.41.0zod:- specifier: ^3.24.0 || ^4.0.0+ specifier: ^4.1.0version: 4.3.6packages/crawlee:diff --git a/test/core/crawlers/statistics.test.ts b/test/core/crawlers/statistics.test.tsindex b0cfc1204..ec10960ed 100644--- a/test/core/crawlers/statistics.test.ts+++ b/test/core/crawlers/statistics.test.ts@@ -1,4 +1,5 @@-import { Configuration, EventType, MemoryStorageBackend, serviceLocator, Statistics } from '@crawlee/core';+import type { StatisticPersistedState, StatisticState } from '@crawlee/core';+import { EventType, KeyValueStore, MemoryStorageBackend, serviceLocator, Statistics } from '@crawlee/core';import type { Dictionary } from '@crawlee/types';describe('Statistics', () => {@@ -6,14 +7,21 @@ describe('Statistics', () => {return Math.round(jobCount / (totalTickMillis / 1000 / 60));};+ const persistStateKey = (statistics: Statistics) => `CRAWLEE_CRAWLER_STATISTICS_${statistics.id}`;+let stats: Statistics;+ let store: KeyValueStore;+beforeAll(async () => {vitest.useFakeTimers();});beforeEach(async () => {+ // Restarted for every test so that the timestamps of a persisted record are predictable.+ vitest.setSystemTime(0);serviceLocator.setStorageBackend(new MemoryStorageBackend());stats = new Statistics();+ store = await KeyValueStore.open();});afterEach(async () => {@@ -42,113 +50,195 @@ describe('Statistics', () => {});test('should persist the state to KV and load again', async () => {- const startedAt = 1000;- vitest.advanceTimersByTime(startedAt);+ vitest.advanceTimersByTime(1000);stats.startJob(0);vitest.advanceTimersByTime(100);- stats.finishJob(0, 0);+ stats.finishJob(0, 1);+ stats.registerStatusCode(200);await stats.startCapturing();- await stats.persistState();+ await stats.stopCapturing();- // console.dir(stats);- // @ts-expect-error Accessing private prop- const state = await stats.keyValueStore.getValue(stats.persistStateKey);+ const restored = new Statistics({ id: stats.id });+ vitest.advanceTimersByTime(5000);+ await restored.startCapturing();- /*- expect(state).toEqual({- crawlerFinishedAt: null,- crawlerLastStartTimestamp: 0,- crawlerRuntimeMillis: 1100,- crawlerStartedAt: toISOString(startedAt + 100),- requestAvgFailedDurationMillis: null,- requestAvgFinishedDurationMillis: 100,- requestMaxDurationMillis: 100,- requestMinDurationMillis: 100,- requestTotalDurationMillis: 100,- requestRetryHistogram: [1],- requestTotalFailedDurationMillis: 0,- requestTotalFinishedDurationMillis: 100,- requestsFailed: 0,- requestsFailedPerMinute: 0,+ expect(restored.state).toMatchObject({requestsFinished: 1,- requestsFinishedPerMinute: 55,- requestsRetries: 0,- requestsTotal: 1,- statsId: stats.id,- statsPersistedAt: toISOString(startedAt + 100),+ requestsRetries: 1,+ requestTotalFinishedDurationMillis: 100,+ requestMinDurationMillis: 100,+ requestMaxDurationMillis: 100,+ crawlerStartedAt: stats.state.crawlerStartedAt,});+ // JSON writes the hole left by a retry count that never occurred as a `null` - it is restored as the+ // zero requests it stands for.+ expect(restored.requestRetryHistogram).toEqual([0, 1]);+ // The gap between the two runs is not counted, but the first run's runtime is.+ expect(restored.calculate().crawlerRuntimeMillis).toEqual(1100);+ await restored.stopCapturing();+ });++ test('should restore an Infinity minimum duration written out as null', async () => {+ await stats.startCapturing();await stats.stopCapturing();- stats.reset();- expect(stats.toJSON()).toEqual({- crawlerFinishedAt: null,- crawlerRuntimeMillis: 0,- crawlerLastStartTimestamp: 1100,- crawlerStartedAt: null,- requestAvgFailedDurationMillis: Infinity,- requestAvgFinishedDurationMillis: Infinity,- requestMaxDurationMillis: 0,- requestMinDurationMillis: Infinity,- requestRetryHistogram: [],- requestTotalDurationMillis: 0,- requestTotalFailedDurationMillis: 0,- requestTotalFinishedDurationMillis: 0,- requestsFailed: 0,- requestsFailedPerMinute: 0,- requestsFinished: 0,- requestsFinishedPerMinute: 0,- requestsRetries: 0,- requestsTotal: 0,- statsId: stats.id,- statsPersistedAt: toISOString(startedAt + 100),- });+ const record = await store.getValue<StatisticPersistedState>(persistStateKey(stats));+ expect(record!.requestMinDurationMillis).toBeNull();+ const restored = new Statistics({ id: stats.id });+ await restored.startCapturing();++ expect(restored.state.requestMinDurationMillis).toEqual(Infinity);++ // The whole point of the minimum - a `duration < null` comparison would never win.+ restored.startJob(0);+ vitest.advanceTimersByTime(100);+ restored.finishJob(0, 0);+ expect(restored.state.requestMinDurationMillis).toEqual(100);++ await restored.stopCapturing();+ });++ test('should start from scratch on a record it cannot make sense of', async () => {await stats.startCapturing();+ stats.startJob(0);+ vitest.advanceTimersByTime(100);+ stats.finishJob(0, 0);+ await stats.stopCapturing();++ const record = (await store.getValue<StatisticPersistedState>(persistStateKey(stats)))!;+ await store.setValue(persistStateKey(stats), { ...record, requestsFinished: 'plenty' });++ const restored = new Statistics({ id: stats.id });+ // @ts-expect-error Accessing private prop+ const warningSpy = vitest.spyOn(restored.log, 'warning').mockImplementation(() => {});++ // A corrupt counter must not take the crawl down with it, and must not be trusted either - an+ // increment on a string would poison every later one.+ await restored.startCapturing();+ expect(warningSpy).toHaveBeenCalledWith(+ expect.stringContaining('starting the statistics from scratch'),+ expect.objectContaining({ persistStateKey: persistStateKey(stats) }),+ );+ expect(restored.state.requestsFinished).toEqual(0);+ expect(restored.state.requestTotalFinishedDurationMillis).toEqual(0);++ await restored.stopCapturing();+ });++ test('should write the record when the run is too short to have a rate', async () => {+ // Everything inside one millisecond: `calculate()` divides by a zero-length run and reports the+ // per-minute rates as `Infinity`, which the record can only carry as `null`.+ stats.startJob(0);+ stats.finishJob(0, 0);stats.startJob(1);+ stats.failJob(1, 0);++ await stats.startCapturing();+ await stats.persistState();++ expect(await store.getValue<StatisticPersistedState>(persistStateKey(stats))).toMatchObject({+ requestsFinished: 1,+ requestsFailed: 1,+ requestsFinishedPerMinute: null,+ requestsFailedPerMinute: null,+ });++ await stats.stopCapturing();+ });++ test('should persist the extra fields of a subclass', async () => {+ type ExtendedState = StatisticState & { productsFound: number };++ class ExtendedStatistics extends Statistics {+ override get state(): ExtendedState {+ return super.state as ExtendedState;+ }++ protected override defaultState(): ExtendedState {+ return { ...super.defaultState(), productsFound: 0 };+ }+ }++ const extended = new ExtendedStatistics({ id: 'extended' });+ await extended.startCapturing();+ extended.state.productsFound = 7;+ await extended.persistState();++ // The record schema knows nothing about them, and must not drop them on the way through.+ expect(await store.getValue(persistStateKey(extended))).toMatchObject({ productsFound: 7 });++ await extended.stopCapturing();+ });++ test('should keep the shape of the persisted record', async () => {+ stats.startJob(0);vitest.advanceTimersByTime(100);- stats.finishJob(1);+ stats.finishJob(0, 0);+ stats.registerStatusCode(200);- vitest.advanceTimersByTime(1000);+ await stats.startCapturing();+ await stats.persistState();- expect(stats.toJSON()).toEqual({- crawlerRuntimeMillis: 2200,- crawlerLastStartTimestamp: 0,- crawlerFinishedAt: toISOString(startedAt + 100),- crawlerStartedAt: toISOString(startedAt + 100),- requestAvgFailedDurationMillis: Infinity,- requestAvgFinishedDurationMillis: 100,- requestMaxDurationMillis: 100,- requestMinDurationMillis: 100,- requestRetryHistogram: [2],- requestTotalDurationMillis: 200,- requestTotalFailedDurationMillis: 0,- requestTotalFinishedDurationMillis: 200,+ const record = (await store.getValue<StatisticPersistedState>(persistStateKey(stats)))!;++ expect(record).toEqual({+ requestsFinished: 1,requestsFailed: 0,- requestsFailedPerMinute: 0,- requestsFinished: 2,- requestsFinishedPerMinute: 55,requestsRetries: 0,- requestsTotal: 2,+ requestsFailedPerMinute: 0,+ requestsFinishedPerMinute: 600,+ requestMinDurationMillis: 100,+ requestMaxDurationMillis: 100,+ requestTotalFailedDurationMillis: 0,+ requestTotalFinishedDurationMillis: 100,+ crawlerStartedAt: '1970-01-01T00:00:00.100Z',+ crawlerFinishedAt: null,+ statsPersistedAt: '1970-01-01T00:00:00.100Z',+ crawlerRuntimeMillis: 100,+ crawlerLastStartTimestamp: 0,+ requestRetryHistogram: [1],statsId: stats.id,- statsPersistedAt: toISOString(startedAt + 1200),- });-- vitest.advanceTimersByTime(10000);-- expect(stats.calculate()).toEqual({- crawlerRuntimeMillis: 12200,- requestAvgFailedDurationMillis: Infinity,+ requestAvgFailedDurationMillis: null,requestAvgFinishedDurationMillis: 100,- requestTotalDurationMillis: 200,- requestsFailedPerMinute: 0,- requestsFinishedPerMinute: getPerMinute(2, 12200),- requestsTotal: 2,+ requestTotalDurationMillis: 100,+ requestsTotal: 1,+ requestsWithStatusCode: { 200: 1 },+ errors: {},+ retryErrors: {},});- */+ // The record is read by tooling outside Crawlee - the key order is part of its shape.+ expect(Object.keys(record)).toEqual([+ 'requestsFinished',+ 'requestsFailed',+ 'requestsRetries',+ 'requestsFailedPerMinute',+ 'requestsFinishedPerMinute',+ 'requestMinDurationMillis',+ 'requestMaxDurationMillis',+ 'requestTotalFailedDurationMillis',+ 'requestTotalFinishedDurationMillis',+ 'crawlerStartedAt',+ 'crawlerFinishedAt',+ 'statsPersistedAt',+ 'crawlerRuntimeMillis',+ 'crawlerLastStartTimestamp',+ 'requestRetryHistogram',+ 'statsId',+ 'requestAvgFailedDurationMillis',+ 'requestAvgFinishedDurationMillis',+ 'requestTotalDurationMillis',+ 'requestsTotal',+ 'requestsWithStatusCode',+ 'errors',+ 'retryErrors',+ ]);++ await stats.stopCapturing();});test('should remove persist state event listener', async () => {@@ -160,7 +250,7 @@ describe('Statistics', () => {expect(events.listenerCount(EventType.PERSIST_STATE)).toEqual(0);await stats.startCapturing();expect(events.listenerCount(EventType.PERSIST_STATE)).toEqual(1);- stats.reset();+ await stats.stopCapturing();expect(events.listenerCount(EventType.PERSIST_STATE)).toEqual(0);});@@ -170,23 +260,17 @@ describe('Statistics', () => {vitest.advanceTimersByTime(100);stats.finishJob(0, 0);- await stats.startCapturing(); // keyValueStore is initialized here+ await stats.startCapturing();const state = stats.toJSON();- // @ts-expect-error Accessing private prop- const setValueSpy = vitest.spyOn(stats.keyValueStore, 'setValue');serviceLocator.getEventManager().emit(EventType.PERSIST_STATE);+ await serviceLocator.getEventManager().waitForAllListenersToComplete();- // TODO: these properties don't exist on the calculate return type- // @ts-expect-error Incorrect types?- const { retryHistogram, finished, failed, ...rest } = stats.calculate();+ // `Infinity` does not survive JSON, so compare against what a record can hold.+ expect(await store.getValue(persistStateKey(stats))).toEqual(JSON.parse(JSON.stringify(state)));- expect(setValueSpy).toBeCalledWith(- // @ts-expect-error Accessing private prop- stats.persistStateKey,- { ...state, ...rest },- );+ await stats.stopCapturing();}, 2000);test('an explicit id restores state across instances under a stable key', async () => {@@ -347,6 +431,10 @@ describe('Statistics', () => {stats.reset();expect(stats.state.requestsFinished).toEqual(0);expect(stats.requestRetryHistogram).toEqual([]);++ // Resetting the counters does not end the capture - stopCapturing() is what does.+ expect(serviceLocator.getEventManager().listenerCount(EventType.PERSIST_STATE)).toEqual(1);+ await stats.stopCapturing();});test('should throw when startCapturing is called while already capturing', async () => {