Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: Extend RecoverableState API to accommodate future adopters (#4006)
Jan Buchar0d880420c7b1
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.mdindex f03fbb76e..d1c7eab32 100644--- a/docs/public-api/crawlee-core.api.md+++ b/docs/public-api/crawlee-core.api.md@@ -1217,32 +1217,31 @@ export interface RecordOptions {}// @public-export class RecoverableState<TStateModel = Record<string, unknown>> {- constructor(options: RecoverableStateOptions<TStateModel>);+export class RecoverableState<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> {+ constructor(options: RecoverableStateOptions<TStateModel, TPersistedState>);get currentValue(): TStateModel;initialize(): Promise<TStateModel>;- persistState(eventData?: {- isMigrating: boolean;- }): Promise<void>;- reset(): Promise<void>;+ persistState(eventData?: Record<string, unknown>): Promise<void>;+ reset(): void;+ resetStore(): Promise<void>;teardown(): Promise<void>;}// @public-export interface RecoverableStateOptions<TStateModel = Record<string, unknown>> extends RecoverableStatePersistenceOptions {+export interface RecoverableStateOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> extends RecoverableStatePersistenceOptions {configuration?: Configuration;- defaultState: TStateModel;- deserialize?: (serializedState: string) => TStateModel;+ defaultState: TStateModel | (() => TStateModel);+ deserialize?: StateConversion<TPersistedState, TStateModel>;logger?: CrawleeLogger;- serialize?: (state: TStateModel) => string;+ serialize?: StateConversion<TStateModel, TPersistedState>;}// @public (undocumented)export interface RecoverableStatePersistenceOptions {+ keyValueStore?: KeyValueStore | PromiseLike<KeyValueStore>;persistenceEnabled?: boolean;+ persistenceTimeoutMillis?: number;persistStateKey: string;- persistStateKvsId?: string;- persistStateKvsName?: string;}// @public (undocumented)@@ -1554,19 +1553,9 @@ export interface RequestTransform {// @publicexport class RequestValidationError extends NonRetryableError {- constructor(label: string | symbol, issues: readonly {- readonly message: string;- readonly path?: readonly (PropertyKey | {- key: PropertyKey;- })[];- }[]);- // (undocumented)- readonly issues: readonly {- readonly message: string;- readonly path?: readonly (PropertyKey | {- key: PropertyKey;- })[];- }[];+ constructor(label: string | symbol, issues: readonly SchemaIssue[]);+ // (undocumented)+ readonly issues: readonly SchemaIssue[];// (undocumented)readonly label: string | symbol;}@@ -1667,6 +1656,16 @@ export type RoutesFromSchemas<Schemas extends RouteSchemas> = {[defaultRoute]: SchemaUserData<Schemas[typeof defaultRoute]>;} : {});+// @public+export interface SchemaIssue {+ // (undocumented)+ readonly message: string;+ // (undocumented)+ readonly path?: readonly (PropertyKey | {+ key: PropertyKey;+ })[];+}+// Not exported by the entry point; reachable only as a referenced type.// @publictype SchemaUserData<Schema extends StandardSchemaV1> = StandardSchemaV1.InferOutput<Schema> extends Dictionary ? StandardSchemaV1.InferOutput<Schema> : Dictionary;@@ -1918,6 +1917,18 @@ export type Source = (Partial<RequestOptions> & {regex?: RegExp;}) | Request_2;+// @public+export type StateConversion<TFrom, TTo> = ((value: TFrom) => Awaitable<TTo>) | StandardSchemaV1<TFrom, TTo>;++// @public+export class StateValidationError extends Error {+ constructor(persistStateKey: string, issues: readonly SchemaIssue[]);+ // (undocumented)+ readonly issues: readonly SchemaIssue[];+ // (undocumented)+ readonly persistStateKey: string;+}+// @publicexport interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {// (undocumented)diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex 7e720872e..1919f1429 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -727,6 +727,10 @@ The exported handler types were reshaped accordingly. `ErrorHandler` and `Reques`PlaywrightHook`, `PuppeteerHook` and `StagehandHook` are now type aliases (previously interfaces) generic over the request's `userData` type. A hook that types its context — via the generic (e.g. `PlaywrightHook<MyUserData>`) or an explicit context annotation — is now assignable to the `preNavigationHooks` / `postNavigationHooks` options of an untyped crawler. If you extended one of these interfaces, use an intersection type instead.+### `RecoverableState` reshaped++`serialize` and `deserialize` now take and return values rather than strings (so v3 records will not load) and each also accept a [Standard Schema](https://standardschema.dev) — a zod codec works as `deserialize` directly — `reset()` is synchronous, no longer clears the persisted record (the new `resetStore()` does) and doubles as a way to establish the state without awaiting `initialize()`, `persistStateKvsName` and `persistStateKvsId` collapsed into a single `keyValueStore` option taking a store (or a pending `KeyValueStore.open()`), `defaultState` also accepts a factory (which you need for a state `structuredClone` cannot rebuild, as the deep copy no longer goes through `serialize`/`deserialize`), and there is a new `persistenceTimeoutMillis` option. `teardown()` is no longer terminal — `initialize()` can be called again to open another persistence window — and a write that fails during a periodic `PERSIST_STATE` or during `teardown()` is warned about rather than thrown. A direct `persistState()` still throws.+### The `log` property is typed as `CrawleeLogger`The `log` property exposed throughout the public API (on the crawling context, `Statistics`, `EventManager`, `SessionOptions`, `Dataset`, etc.) is now typed as the `CrawleeLogger` interface (from `@crawlee/types`) rather than the concrete `Log` class from `@apify/log`. If you consume it structurally — calling `log.info(...)`, `log.debug(...)`, `log.child(...)` — nothing changes. You only need to act if you explicitly annotated a variable or parameter with the `Log` type from `@apify/log` and assigned `context.log` to it; type it as `CrawleeLogger` instead.diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.tsindex a5ed96dfc..f558e86b5 100644--- a/packages/core/src/errors.ts+++ b/packages/core/src/errors.ts@@ -16,6 +16,26 @@ export class CriticalError extends NonRetryableError {}*/export class MissingRouteError extends CriticalError {}+/**+ * A schema validation issue, structurally compatible with `StandardSchemaV1.Issue`. Declared here so that+ * error types do not have to depend on `@standard-schema/spec`.+ */+export interface SchemaIssue {+ readonly message: string;+ readonly path?: readonly (PropertyKey | { key: PropertyKey })[];+}++function formatIssues(issues: readonly SchemaIssue[]): string {+ return issues+ .map((issue) => {+ const path = (issue.path ?? [])+ .map((segment) => (typeof segment === 'object' ? segment.key : segment))+ .join('.');+ return `- ${path ? `${path}: ` : ''}${issue.message}`;+ })+ .join('\n');+}+/*** Thrown when a request's `userData` does not match the {@apilink RouteSchemas|Standard Schema} registered for its label.*@@ -24,21 +44,24 @@ export class MissingRouteError extends CriticalError {}export class RequestValidationError extends NonRetryableError {constructor(readonly label: string | symbol,- readonly issues: readonly {- readonly message: string;- readonly path?: readonly (PropertyKey | { key: PropertyKey })[];- }[],+ readonly issues: readonly SchemaIssue[],+ ) {+ super(`Request userData for label '${String(label)}' failed schema validation:\n${formatIssues(issues)}`);+ }+}++/**+ * Thrown by {@apilink RecoverableState} when a persisted state record does not match its `stateSchema`.+ *+ * Whether a corrupt record should abort the run or be discarded in favour of the defaults depends on what the+ * state is for, so {@apilink RecoverableState.initialize} always throws and leaves the choice to the caller.+ */+export class StateValidationError extends Error {+ constructor(+ readonly persistStateKey: string,+ readonly issues: readonly SchemaIssue[],) {- const details = issues- .map((issue) => {- const path = (issue.path ?? [])- .map((segment) => (typeof segment === 'object' ? segment.key : segment))- .join('.');- return `- ${path ? `${path}: ` : ''}${issue.message}`;- })- .join('\n');-- super(`Request userData for label '${String(label)}' failed schema validation:\n${details}`);+ super(`State persisted under key '${persistStateKey}' failed schema validation:\n${formatIssues(issues)}`);}}diff --git a/packages/core/src/recoverable_state.ts b/packages/core/src/recoverable_state.tsindex dbb0ff37a..b6a3cffaf 100644--- a/packages/core/src/recoverable_state.ts+++ b/packages/core/src/recoverable_state.ts@@ -1,5 +1,20 @@+import { addTimeoutToPromise, storage as timeoutStorage } from '@apify/timeout';import type { Configuration, CrawleeLogger } from '@crawlee/core';-import { EventType, KeyValueStore, serviceLocator } from '@crawlee/core';+import { EventType, KeyValueStore, serviceLocator, StateValidationError } from '@crawlee/core';+import type { Awaitable } from '@crawlee/types';+import type { StandardSchemaV1 } from '@standard-schema/spec';++const DEFAULT_PERSISTENCE_TIMEOUT_MILLIS = 60_000;++/**+ * One direction of the conversion between the state model and its persisted form - either a plain function, or a+ * [Standard Schema](https://standardschema.dev) whose validated output is the result.+ *+ * A schema that fails to validate makes {@apilink RecoverableState} throw a {@apilink StateValidationError}. Zod+ * codecs work directly, as their validation *is* the decode direction; use `(state) => codec.encode(state)` for the+ * other one.+ */+export type StateConversion<TFrom, TTo> = ((value: TFrom) => Awaitable<TTo>) | StandardSchemaV1<TFrom, TTo>;export interface RecoverableStatePersistenceOptions {/**@@ -13,16 +28,16 @@ export interface RecoverableStatePersistenceOptions {persistenceEnabled?: boolean;/**- * The name of the KeyValueStore to use for persistence.- * If neither a name nor an id are supplied, the default store will be used.+ * The KeyValueStore to persist into, defaulting to the default store. Accepts a pending+ * {@apilink KeyValueStore.open} so that callers do not have to be async to point at a specific store.*/- persistStateKvsName?: string;+ keyValueStore?: KeyValueStore | PromiseLike<KeyValueStore>;/**- * The identifier of the KeyValueStore to use for persistence.- * If neither a name nor an id are supplied, the default store will be used.+ * Time limit for a single load or save of the state, in milliseconds.+ * @default 60_000*/- persistStateKvsId?: string;+ persistenceTimeoutMillis?: number;}/**@@ -30,12 +45,15 @@ export interface RecoverableStatePersistenceOptions {*/export interface RecoverableStateOptions<TStateModel = Record<string, unknown>,+ TPersistedState = TStateModel,> extends RecoverableStatePersistenceOptions {/**- * The default state used if no persisted state is found.- * A deep copy is made each time the state is used.+ * The state used when no persisted state is found, and the state {@apilink RecoverableState.reset} restores.+ *+ * A plain value is deep-copied with `structuredClone` each time it is used, so pass a factory for a state+ * that `structuredClone` cannot rebuild - one holding class instances, say, or one derived from a schema.*/- defaultState: TStateModel;+ defaultState: TStateModel | (() => TStateModel);/*** A logger instance for logging operations related to state persistence@@ -43,22 +61,21 @@ export interface RecoverableStateOptions<logger?: CrawleeLogger;/**- * Configuration instance to use+ * Configuration instance to use when opening the KeyValueStore*/configuration?: Configuration;/**- * Optional function to transform the state to a JSON string before persistence.- * If not provided, JSON.stringify will be used.+ * Optional conversion of the state to a plain JSON-serializable value before it is persisted.+ * If not provided, the state is persisted as is.*/- serialize?: (state: TStateModel) => string;+ serialize?: StateConversion<TStateModel, TPersistedState>;/**- * Optional function to transform a JSON-serialized object back to the state model.- * If not provided, JSON.parse is used.- * It is advisable to perform validation in this function and to throw an exception if it fails.+ * Optional conversion of a persisted value back to the state model, and the place to validate a record before+ * trusting it. If not provided, the persisted value is used as is.*/- deserialize?: (serializedState: string) => TStateModel;+ deserialize?: StateConversion<TPersistedState, TStateModel>;}/**@@ -71,71 +88,103 @@ export interface RecoverableStateOptions<* The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.* The class automatically hooks into the event system to persist state when needed.*/-export class RecoverableState<TStateModel = Record<string, unknown>> {- readonly #defaultState: TStateModel;+export class RecoverableState<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> {+ readonly #defaultState: () => TStateModel;#state: TStateModel | null = null;+ #initialized = false;+ #listening = false;readonly #persistenceEnabled: boolean;readonly #persistStateKey: string;- readonly #persistStateKvsName?: string;- readonly #persistStateKvsId?: string;- #keyValueStore: KeyValueStore | null = null;+ readonly #persistenceTimeoutMillis: number;+ readonly #configuration?: Configuration;+ #keyValueStore: KeyValueStore | PromiseLike<KeyValueStore> | null;readonly #log: CrawleeLogger;- readonly #serialize: (state: TStateModel) => string;- readonly #deserialize: (serializedState: string) => TStateModel;+ readonly #serialize: (state: TStateModel) => Promise<TPersistedState>;+ readonly #deserialize: (persistedState: TPersistedState) => Promise<TStateModel>;+ readonly #persistStateQuietly: (eventData?: Record<string, unknown>) => Promise<void>;/*** Initialize a new recoverable state object.** @param options Configuration options for the recoverable state*/- constructor(options: RecoverableStateOptions<TStateModel>) {- this.#defaultState = options.defaultState;+ constructor(options: RecoverableStateOptions<TStateModel, TPersistedState>) {+ const { defaultState } = options;+ this.#defaultState =+ typeof defaultState === 'function'+ ? (defaultState as () => TStateModel)+ : () => structuredClone(defaultState);+this.#persistStateKey = options.persistStateKey;this.#persistenceEnabled = options.persistenceEnabled ?? false;- this.#persistStateKvsName = options.persistStateKvsName;- this.#persistStateKvsId = options.persistStateKvsId;+ this.#persistenceTimeoutMillis = options.persistenceTimeoutMillis ?? DEFAULT_PERSISTENCE_TIMEOUT_MILLIS;+ this.#configuration = options.configuration;+ this.#keyValueStore = options.keyValueStore ?? null;this.#log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });- this.#serialize = options.serialize ?? JSON.stringify;- this.#deserialize = options.deserialize ?? JSON.parse;+ this.#serialize = this.#toConversion(options.serialize);+ this.#deserialize = this.#toConversion(options.deserialize);++ // The automatic persists, where a rejection has nowhere useful to go - the event manager does not catch+ // listener errors, and throwing from teardown would bury the outcome of the work it cleans up after.+ this.#persistStateQuietly = async (eventData) =>+ this.persistState(eventData).catch((error) =>+ this.#log.warning(`Failed to persist the state under key '${this.#persistStateKey}'.`, { error }),+ );+ }++ /** Normalizes a conversion option into a function. Absent conversions pass the value through unchanged. */+ #toConversion<TFrom, TTo>(conversion: StateConversion<TFrom, TTo> | undefined): (value: TFrom) => Promise<TTo> {+ if (conversion === undefined) {+ return async (value) => value as unknown as TTo;+ }++ if (typeof conversion === 'function') {+ return async (value) => conversion(value);+ }- this.persistState = this.persistState.bind(this);+ return async (value) => {+ const result = await conversion['~standard'].validate(value);++ if (result.issues) {+ throw new StateValidationError(this.#persistStateKey, result.issues);+ }++ return result.value;+ };}/*** Initialize the recoverable state.*- * This method must be called before using the recoverable state. It loads the saved state- * if persistence is enabled and registers the object to listen for PERSIST_STATE events.+ * If persistence is enabled, this method loads the saved state and registers the object to listen for+ * PERSIST_STATE events. A state established beforehand by {@apilink RecoverableState.reset} survives if there+ * is no record to restore.+ *+ * Calling this again after a {@apilink RecoverableState.teardown} starts a new persistence window - the+ * listener is registered again and the record reloaded.** @returns The loaded state object*/async initialize(): Promise<TStateModel> {- if (this.#state !== null && this.#state !== undefined) {+ if (this.#initialized) {return this.currentValue;}- if (!this.#persistenceEnabled) {- this.#state = this.#deserialize(this.#serialize(this.#defaultState));- return this.currentValue;- }-- let kvsIdentifier: { name: string } | { id: string } | null = null;-- if (this.#persistStateKvsName) {- kvsIdentifier = { name: this.#persistStateKvsName };- } else if (this.#persistStateKvsId) {- kvsIdentifier = { id: this.#persistStateKvsId };+ if (this.#persistenceEnabled) {+ this.#keyValueStore ??= KeyValueStore.open(null, {+ configuration: this.#configuration ?? serviceLocator.getConfiguration(),+ });+ await this.#resolveKeyValueStore();+ serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.#persistStateQuietly);+ this.#listening = true;}- this.#keyValueStore = await KeyValueStore.open(kvsIdentifier, {- configuration: serviceLocator.getConfiguration(),- });+ // Flipped before the record is loaded, so that a caller catching a `StateValidationError` is left with a+ // fully wired object running on the default state rather than a half-initialized one.+ this.#initialized = true;+ this.#state ??= this.#defaultState();- await this.loadSavedState();-- // Register for persist state events- const eventManager = serviceLocator.getEventManager();- eventManager.on(EventType.PERSIST_STATE, this.persistState);+ await this.#loadSavedState();return this.currentValue;}@@ -144,82 +193,157 @@ export class RecoverableState<TStateModel = Record<string, unknown>> {* Clean up resources used by the recoverable state.** If persistence is enabled, this method deregisters the object from PERSIST_STATE events- * and persists the current state one last time.+ * and persists the current state one last time, warning rather than throwing if that write fails - cleanup+ * runs when the work is already done, and failing it would bury whatever the caller was doing. The in-memory+ * state is left alone, and {@apilink RecoverableState.initialize} can be called again to open a new+ * persistence window.*/async teardown(): Promise<void> {- if (!this.#persistenceEnabled || !this.persistState) {+ this.#initialized = false;++ if (!this.#persistenceEnabled) {return;}- const eventManager = serviceLocator.getEventManager();- eventManager.off(EventType.PERSIST_STATE, this.persistState);- await this.persistState();+ serviceLocator.getEventManager().off(EventType.PERSIST_STATE, this.#persistStateQuietly);+ this.#listening = false;+ await this.#persistStateQuietly();}/*** Get the current state.+ *+ * Throws until the state has been established, by either {@apilink RecoverableState.initialize} or the+ * synchronous {@apilink RecoverableState.reset} - the latter being how a caller that cannot await in its+ * constructor gets a usable state right away.*/get currentValue(): TStateModel {if (this.#state === null) {- throw new Error('Recoverable state has not yet been loaded');+ throw new Error('Recoverable state has not yet been loaded - call initialize() or reset() first');}return this.#state;}/**- * Reset the state to the default values and clear any persisted state.+ * Reset the in-memory state to the default values, leaving any persisted record alone.*- * Resets the current state to the default state and, if persistence is enabled,- * clears the persisted state from the KeyValueStore.+ * Use {@apilink RecoverableState.resetStore} to clear the persisted record as well.*/- async reset(): Promise<void> {- this.#state = this.#deserialize(this.#serialize(this.#defaultState));+ reset(): void {+ this.#state = this.#defaultState();+ }- if (this.#persistenceEnabled) {- if (this.#keyValueStore === null) {- throw new Error('Recoverable state has not yet been initialized');- }+ /**+ * Clear the persisted state record, leaving the in-memory state alone.+ *+ * This is a between-lifecycles operation - its point is to stop the next {@apilink RecoverableState.initialize}+ * from restoring the record, so it throws while PERSIST_STATE events are still being handled, where the next+ * one would write the record straight back. Use {@apilink RecoverableState.reset} to reset the state itself,+ * or {@apilink RecoverableState.teardown} before clearing the record.+ *+ * A no-op if persistence is disabled or no KeyValueStore is available yet.+ */+ async resetStore(): Promise<void> {+ if (this.#listening) {+ throw new Error(+ `Cannot clear the state persisted under key '${this.#persistStateKey}' while it is still being persisted periodically - the next PERSIST_STATE event would write it straight back. Use reset() to reset the state itself, or teardown() before clearing the record.`,+ );+ }- await this.#keyValueStore.setValue(this.#persistStateKey, null);+ if (!this.#persistenceEnabled) {+ return;}++ const keyValueStore = await this.#resolveKeyValueStore();++ if (keyValueStore === null) {+ return;+ }++ await this.#withTimeout(+ async () => keyValueStore.setValue(this.#persistStateKey, null),+ 'Clearing the persisted state',+ );}/*** Persist the current state to the KeyValueStore.** This method is typically called in response to a PERSIST_STATE event, but can also be called- * directly when needed.+ * directly when needed. It is a no-op if persistence is disabled, if no KeyValueStore is available yet, or if+ * there is no state to write. A failed write only rejects here - the periodic and teardown ones warn instead.** @param eventData Optional data associated with a PERSIST_STATE event*/- async persistState(eventData?: { isMigrating: boolean }): Promise<void> {- this.#log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);+ async persistState(eventData?: Record<string, unknown>): Promise<void> {+ if (!this.#persistenceEnabled || this.#state === null) {+ return;+ }++ const keyValueStore = await this.#resolveKeyValueStore();- if (this.#keyValueStore === null || this.#state === null) {- throw new Error('Recoverable state has not yet been initialized');+ if (keyValueStore === null) {+ return;}- if (this.#persistenceEnabled) {- await this.#keyValueStore.setValue(this.#persistStateKey, this.#serialize(this.#state), {- contentType: 'text/plain', // HACK - the result is expected to be JSON, but we do this to avoid the implicit JSON.parse in `KeyValueStore.getValue`- });+ this.#log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);++ const serializedState = await this.#serialize(this.currentValue);++ await this.#withTimeout(+ async () => keyValueStore.setValue(this.#persistStateKey, serializedState),+ 'Persisting the state',+ );+ }++ /** Awaits a store handed over as a pending `open()`, keeping the resolved instance for later calls. */+ async #resolveKeyValueStore(): Promise<KeyValueStore | null> {+ if (this.#keyValueStore === null) {+ return null;}++ this.#keyValueStore = await this.#keyValueStore;++ return this.#keyValueStore;}/**- * Load the saved state from the KeyValueStore+ * Load the saved state from the KeyValueStore. Leaves the current state alone if there is no record to load.*/- private async loadSavedState(): Promise<void> {- if (this.#keyValueStore === null) {- throw new Error('Recoverable state has not yet been initialized');+ async #loadSavedState(): Promise<void> {+ if (!this.#persistenceEnabled) {+ return;}- const storedState = await this.#keyValueStore.getValue(this.#persistStateKey);+ const keyValueStore = await this.#resolveKeyValueStore();++ if (keyValueStore === null) {+ return;+ }++ const storedState = await this.#withTimeout(+ async () => keyValueStore.getValue(this.#persistStateKey),+ 'Loading the persisted state',+ );+if (storedState === null || storedState === undefined) {- this.#state = this.#deserialize(this.#serialize(this.#defaultState));- } else {- this.#state = this.#deserialize(storedState as string);+ return;}++ this.#state = await this.#deserialize(storedState as TPersistedState);+ }++ async #withTimeout<T>(operation: () => Promise<T>, description: string): Promise<T> {+ // `@apify/timeout` shares one `AbortController` across nested frames and `KeyValueStore` checks it on+ // every operation, so a teardown-time persist running inside an already-expired request handler timeout+ // would be aborted before it started. Hence a fresh timeout context.+ return timeoutStorage.exit(async () =>+ addTimeoutToPromise(+ operation,+ this.#persistenceTimeoutMillis,+ `${description} under key '${this.#persistStateKey}' timed out after ${this.#persistenceTimeoutMillis / 1000} seconds.`,+ ),+ );}}diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.jsonindex 9e001135d..2719d113b 100644--- a/packages/playwright-crawler/package.json+++ b/packages/playwright-crawler/package.json@@ -63,7 +63,8 @@"ow": "^2.0.0","string-comparison": "^1.3.0","tslib": "^2.8.1",- "type-fest": "^4.0.0"+ "type-fest": "^4.0.0",+ "zod": "^4.1.0"},"peerDependencies": {"idcac-playwright": "^0.2.0",diff --git a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.tsindex e385ef1ce..aece5ffa1 100644--- a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts+++ b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts@@ -3,6 +3,7 @@ import { RecoverableState } from '@crawlee/core';import LogisticRegression from 'ml-logistic-regression';import { Matrix } from 'ml-matrix';import stringComparison from 'string-comparison';+import { z } from 'zod';export type RenderingType = 'clientOnly' | 'static';@@ -63,6 +64,51 @@ export interface IRenderingTypePredictor {storeResult(requests: Request | Request[], renderingType: RenderingType): void;}+const renderingType = z.enum(['clientOnly', 'static'] as const satisfies readonly RenderingType[]);++const predictorState = z.object({+ logreg: z.instanceof(LogisticRegression),+ detectionResults: z.map(renderingType, z.map(z.string().optional(), z.array(z.array(z.string())))),+});++const persistedState = z.object({+ logreg: z+ .record(z.string(), z.unknown())+ .prefault(() => new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }).toJSON()),+ detectionResults: z+ .array(+ z.object({+ renderingType,+ urlPartsByLabel: z.array(+ z.object({+ label: z.string().optional(),+ urlParts: z.array(z.array(z.string())),+ }),+ ),+ }),+ )+ .prefault([]),+});++const stateCodec = z.codec(persistedState, predictorState, {+ decode: ({ logreg, detectionResults }) => ({+ logreg: LogisticRegression.load(logreg),+ detectionResults: new Map(+ detectionResults.map(({ renderingType, urlPartsByLabel }) => [+ renderingType,+ new Map(urlPartsByLabel.map(({ label, urlParts }) => [label, urlParts])),+ ]),+ ),+ }),+ encode: ({ logreg, detectionResults }) => ({+ logreg: logreg.toJSON(),+ detectionResults: Array.from(detectionResults.entries()).map(([renderingType, urlPartsByLabel]) => ({+ renderingType,+ urlPartsByLabel: Array.from(urlPartsByLabel.entries()).map(([label, urlParts]) => ({ label, urlParts })),+ })),+ }),+});+/*** Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.*@@ -71,44 +117,15 @@ export interface IRenderingTypePredictor {export class RenderingTypePredictor implements IRenderingTypePredictor {#detectionRatio: number;// kept as TS-private: tests reach for it at runtime- private state: RecoverableState<{- logreg: LogisticRegression;- detectionResults: Map<RenderingType, Map<string | undefined, URLComponents[]>>;- }>;+ private state: RecoverableState<z.infer<typeof predictorState>, z.input<typeof persistedState>>;constructor({ detectionRatio, persistenceOptions }: RenderingTypePredictorOptions) {this.#detectionRatio = detectionRatio;this.state = new RecoverableState({- defaultState: {- logreg: new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }),- detectionResults: new Map<RenderingType, Map<string | undefined, URLComponents[]>>(),- },- serialize: (state) =>- JSON.stringify({- logreg: state.logreg.toJSON(),- detectionResults: Array.from(state.detectionResults.entries()).map(- ([renderingType, urlPartsByLabel]) => ({- renderingType,- urlPartsByLabel: Array.from(urlPartsByLabel.entries()).map(([label, urlParts]) => ({- label,- urlParts,- })),- }),- ),- }),- deserialize: (serializedState) => {- const { logreg, detectionResults = [] } = JSON.parse(serializedState);-- return {- logreg: LogisticRegression.load(logreg),- detectionResults: new Map(- detectionResults.map((serializedItem: any) => [- serializedItem.renderingType,- new Map(serializedItem.urlPartsByLabel.map((item: any) => [item.label, item.urlParts])),- ]),- ),- };- },+ defaultState: () => stateCodec.decode({}),+ // The codec validates in the decode direction, so it is a Standard Schema as-is; encoding needs a call.+ deserialize: stateCodec,+ serialize: (state) => stateCodec.encode(state),persistStateKey: 'rendering-type-predictor-state',persistenceEnabled: true,...persistenceOptions,diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 4049d26ca..9ea7e68ab 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -807,6 +807,9 @@ importers:type-fest:specifier: ^4.0.0version: 4.41.0+ zod:+ specifier: ^4.1.0+ version: 4.3.6packages/puppeteer-crawler:dependencies:diff --git a/test/core/crawlers/adaptive_playwright_crawler.test.ts b/test/core/crawlers/adaptive_playwright_crawler.test.tsindex 678cd7ce4..923fb9010 100644--- a/test/core/crawlers/adaptive_playwright_crawler.test.ts+++ b/test/core/crawlers/adaptive_playwright_crawler.test.ts@@ -805,11 +805,8 @@ describe('AdaptivePlaywrightCrawler', () => {// Verify that the regression model was actually saved to the key-value storeconst store = await KeyValueStore.open();- const storedState = await store.getValue<string>('rendering-type-predictor-state');- expect(storedState).not.toBeNull();-- const parsedState = JSON.parse(storedState!);- expect(parsedState).toHaveProperty('logreg');+ const storedState = await store.getValue('rendering-type-predictor-state');+ expect(storedState).toHaveProperty('logreg');// Test that the persisted state can be successfully restored// by creating a new RenderingTypePredictor and seeing if it initializes without errordiff --git a/test/core/crawlers/rendering_type_predictor.test.ts b/test/core/crawlers/rendering_type_predictor.test.tsindex e95545314..5c55d28f8 100644--- a/test/core/crawlers/rendering_type_predictor.test.ts+++ b/test/core/crawlers/rendering_type_predictor.test.ts@@ -30,13 +30,9 @@ describe('RenderingTypePredictor', () => {// eslint-disable-next-line dot-notationawait predictor['state'].persistState(); // Access private state for persistence- // RecoverableState persists with a `text/plain` content type on purpose (it owns- // (de)serialization), so the frontend hands back the raw serialized string here.- const serializedState = await store.getValue<string>(persistStateKey);- expect(serializedState).not.toBeNull();- const parsedState = JSON.parse(serializedState!);- expect(parsedState).toHaveProperty('logreg');- expect(parsedState).toHaveProperty('detectionResults');+ const persistedState = await store.getValue(persistStateKey);+ expect(persistedState).toHaveProperty('logreg');+ expect(persistedState).toHaveProperty('detectionResults');// Create a new predictor and verify it restores the stateconst restoredPredictor = new RenderingTypePredictor({diff --git a/test/core/recoverable_state.test.ts b/test/core/recoverable_state.test.tsindex b6eb67ead..116555df2 100644--- a/test/core/recoverable_state.test.ts+++ b/test/core/recoverable_state.test.ts@@ -1,6 +1,15 @@-import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';--import { KeyValueStore, MemoryStorageBackend, serviceLocator } from '../../packages/core/src/index.js';+import { setTimeout as sleep } from 'node:timers/promises';++import { beforeEach, describe, expect, test, vi } from 'vitest';+import { z } from 'zod';++import {+ EventType,+ KeyValueStore,+ MemoryStorageBackend,+ serviceLocator,+ StateValidationError,+} from '../../packages/core/src/index.js';import { RecoverableState } from '../../packages/core/src/recoverable_state.js';interface TestState {@@ -34,7 +43,7 @@ describe('RecoverableState', () => {expect(recoverableState.currentValue).toEqual(defaultState);});- test('should throw error when accessing currentValue before initialization', () => {+ test('should throw when accessing currentValue before it has been established', () => {const recoverableState = new RecoverableState({defaultState,persistStateKey: 'test-key',@@ -44,6 +53,48 @@ describe('RecoverableState', () => {expect(() => recoverableState.currentValue).toThrow('Recoverable state has not yet been loaded');});+ test('should establish a deep copy of the default state on a synchronous reset', () => {+ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: false,+ });++ recoverableState.reset();++ expect(recoverableState.currentValue).toEqual(defaultState);+ expect(recoverableState.currentValue).not.toBe(defaultState);+ });++ test('should keep pre-initialization modifications when there is nothing to restore', async () => {+ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ });++ recoverableState.reset();+ recoverableState.currentValue.counter = 42;+ await recoverableState.initialize();++ expect(recoverableState.currentValue.counter).toBe(42);+ });++ test('should overwrite a pre-initialization state with a restored record', async () => {+ await (await KeyValueStore.open()).setValue('test-key', { ...defaultState, counter: 7 });++ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ });++ recoverableState.reset();+ recoverableState.currentValue.counter = 42;++ expect((await recoverableState.initialize()).counter).toBe(7);+ });+test('should allow state modification after initialization', async () => {const recoverableState = new RecoverableState({defaultState,@@ -76,8 +127,7 @@ describe('RecoverableState', () => {recoverableState.currentValue.counter = 42;recoverableState.currentValue.message = 'modified';- // Reset- await recoverableState.reset();+ recoverableState.reset();expect(recoverableState.currentValue).toEqual(defaultState);expect(recoverableState.currentValue).not.toBe(defaultState); // Should be a new copy@@ -94,6 +144,64 @@ describe('RecoverableState', () => {await expect(recoverableState.teardown()).resolves.not.toThrow();});+ test('should start listening again when initialized after a teardown', async () => {+ const events = serviceLocator.getEventManager();++ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ });++ await recoverableState.initialize();+ expect(events.listenerCount(EventType.PERSIST_STATE)).toBe(1);++ await recoverableState.teardown();+ expect(events.listenerCount(EventType.PERSIST_STATE)).toBe(0);++ await recoverableState.initialize();+ expect(events.listenerCount(EventType.PERSIST_STATE)).toBe(1);++ await recoverableState.teardown();+ });++ test('should warn rather than throw when a periodic persist fails', async () => {+ const store = await KeyValueStore.open();+ vi.spyOn(store, 'setValue').mockRejectedValue(new Error('store is on fire'));++ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ keyValueStore: store,+ });++ await recoverableState.initialize();++ // An unhandled rejection here would take the process down and make `EventManager.close()` reject.+ serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });+ await expect(serviceLocator.getEventManager().waitForAllListenersToComplete()).resolves.not.toThrow();++ await expect(recoverableState.persistState()).rejects.toThrow('store is on fire');+ });++ test('should warn rather than throw when the persist during teardown fails', async () => {+ const store = await KeyValueStore.open();+ vi.spyOn(store, 'setValue').mockRejectedValue(new Error('store is on fire'));++ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ keyValueStore: store,+ });++ await recoverableState.initialize();++ // `BasicCrawler.run()` tears down in a `finally`, so a throw here would replace the crawl's own outcome.+ await expect(recoverableState.teardown()).resolves.not.toThrow();+ });+test('should handle arrays and complex objects in deep copy', async () => {const stateWithArray = {items: [1, 2, { nested: 'value' }],@@ -112,11 +220,58 @@ describe('RecoverableState', () => {(recoverableState.currentValue.items[2] as any).nested = 'modified';// Reset should restore original values- await recoverableState.reset();+ recoverableState.reset();expect(recoverableState.currentValue.items).toEqual([1, 2, { nested: 'value' }]);});+ test('should preserve values that do not survive JSON in the default deep copy', async () => {+ const counts = new Map([['a', 1]]);++ const recoverableState = new RecoverableState({+ defaultState: {+ limit: Infinity,+ since: new Date('2020-01-01T00:00:00.000Z'),+ counts,+ },+ persistStateKey: 'test-key',+ persistenceEnabled: false,+ });++ const state = await recoverableState.initialize();++ expect(state.limit).toBe(Infinity);+ expect(state.since).toEqual(new Date('2020-01-01T00:00:00.000Z'));+ expect(state.counts).toEqual(counts);+ expect(state.counts).not.toBe(counts); // Should be a deep copy+ });++ test('should persist the state as JSON and restore it', async () => {+ const store = await KeyValueStore.open();++ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ });++ await recoverableState.initialize();+ recoverableState.currentValue.counter = 42;+ await recoverableState.persistState();++ const record = await store.getRecord('test-key');+ expect(record!.contentType).toMatch('application/json');+ expect(await store.getValue('test-key')).toEqual({ ...defaultState, counter: 42 });++ const restored = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ });++ expect(await restored.initialize()).toEqual({ ...defaultState, counter: 42 });+ });+test('should handle custom classes with serialize/deserialize', async () => {class CustomData {constructor(@@ -125,61 +280,384 @@ describe('RecoverableState', () => {) {}}- interface StateWithCustomClass {- data: CustomData;- name: string;- }+ const serialize = vi.fn((state: { data: CustomData; name: string }) => ({+ data: { value: state.data.value, count: state.data.count },+ name: state.name,+ }));++ const deserialize = vi.fn((persisted: { data: { value: string; count: number }; name: string }) => ({+ data: new CustomData(persisted.data.value, persisted.data.count),+ name: persisted.name,+ }));++ const build = () =>+ new RecoverableState({+ defaultState: () => ({ data: new CustomData('test', 42), name: 'example' }),+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ serialize,+ deserialize,+ });++ const recoverableState = build();+ await recoverableState.initialize();- const stateWithCustomClass: StateWithCustomClass = {- data: new CustomData('test', 42),- name: 'example',- };+ // With no record to load, the default comes from the factory - deserialize is not involved.+ expect(recoverableState.currentValue.data).toBeInstanceOf(CustomData);+ expect(recoverableState.currentValue.data.value).toBe('test');+ expect(recoverableState.currentValue.name).toBe('example');+ expect(deserialize).not.toHaveBeenCalled();- const serialize = vi.fn((state: StateWithCustomClass) =>- JSON.stringify({- data: {- value: state.data.value,- count: state.data.count,- },- name: state.name,- }),- );+ recoverableState.currentValue.data.value = 'updated';+ await recoverableState.persistState();- const deserialize = vi.fn((serialized: string): StateWithCustomClass => {- const parsed = JSON.parse(serialized);- return {- data: new CustomData(parsed.data.value, parsed.data.count),- name: parsed.name,- };+ expect(serialize).toHaveBeenCalled();+ expect(await (await KeyValueStore.open()).getValue('test-key')).toMatchObject({+ data: { value: 'updated' },+ });++ const restored = build();+ await restored.initialize();++ expect(deserialize).toHaveBeenCalled();+ expect(restored.currentValue.data).toBeInstanceOf(CustomData);+ expect(restored.currentValue.data.value).toBe('updated');+ });++ test('should call a defaultState factory afresh for every reset', async () => {+ const defaultStateFactory = vi.fn(() => ({ items: new Map([['a', 1]]) }));++ const recoverableState = new RecoverableState({+ defaultState: defaultStateFactory,+ persistStateKey: 'test-key',+ persistenceEnabled: false,});+ recoverableState.reset();+ const first = recoverableState.currentValue.items;+ recoverableState.reset();++ expect(recoverableState.currentValue.items).toEqual(first);+ expect(recoverableState.currentValue.items).not.toBe(first);+ expect(defaultStateFactory).toHaveBeenCalledTimes(2);+ });++ test('should not let modifications of a restored state reach the persisted record', async () => {+ const store = await KeyValueStore.open();+ await store.setValue('test-key', { ...defaultState, counter: 42 });+const recoverableState = new RecoverableState({- defaultState: stateWithCustomClass,+ defaultState,persistStateKey: 'test-key',persistenceEnabled: true,- serialize,- deserialize,});await recoverableState.initialize();+ recoverableState.currentValue.data.nested = 'mutated';- // Verify the custom class is properly handled- expect(recoverableState.currentValue.data).toBeInstanceOf(CustomData);- expect(recoverableState.currentValue.data.value).toBe('test');- expect(recoverableState.currentValue.name).toBe('example');+ expect(await store.getValue('test-key')).toMatchObject({ data: { nested: 'value' } });+ });- expect(deserialize).toHaveBeenCalled();+ describe('schemas as conversions', () => {+ const stateSchema = z.object({ counter: z.number(), message: z.string() });- recoverableState.currentValue.data.value = 'updated';- await recoverableState.persistState();+ test('should use the validated output of a deserialize schema', async () => {+ await (await KeyValueStore.open()).setValue('test-key', { counter: '42', message: 'hi', extra: 'nope' });- // RecoverableState persists with a `text/plain` content type on purpose (it owns- // (de)serialization), so the frontend hands back the raw serialized string here.- const persistedState = await (await KeyValueStore.open()).getValue<string>('test-key');- expect(JSON.parse(persistedState!)).toMatchObject({- data: { value: 'updated' },+ const recoverableState = new RecoverableState({+ defaultState: { counter: 0, message: '' },+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ // `z.coerce` and the stripping of undeclared keys are only visible if the validated output,+ // rather than the raw record, is what becomes the state.+ deserialize: z.object({ counter: z.coerce.number(), message: z.string() }),+ });++ expect(await recoverableState.initialize()).toEqual({ counter: 42, message: 'hi' });});- expect(serialize).toHaveBeenCalled();+ test('should throw a StateValidationError on an invalid record', async () => {+ await (await KeyValueStore.open()).setValue('test-key', { counter: 'not a number', message: 'hi' });++ const recoverableState = new RecoverableState({+ defaultState: { counter: 0, message: '' },+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ deserialize: stateSchema,+ });++ await expect(recoverableState.initialize()).rejects.toThrow(StateValidationError);+ });++ test('should stay usable and keep persisting after a caught validation error', async () => {+ const store = await KeyValueStore.open();+ await store.setValue('test-key', { counter: 'not a number', message: 'hi' });++ const recoverableState = new RecoverableState({+ defaultState: { counter: 0, message: '' },+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ deserialize: stateSchema,+ });++ await expect(recoverableState.initialize()).rejects.toThrow(StateValidationError);++ expect(recoverableState.currentValue).toEqual({ counter: 0, message: '' });++ recoverableState.currentValue.counter = 1;+ await recoverableState.persistState();+ expect(await store.getValue('test-key')).toEqual({ counter: 1, message: '' });+ });++ test('should not run a deserialize schema when there is no record', async () => {+ const validate = vi.fn();++ const recoverableState = new RecoverableState({+ defaultState: { counter: 0 },+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ deserialize: { '~standard': { version: 1, vendor: 'test', validate } },+ });++ await recoverableState.initialize();++ expect(validate).not.toHaveBeenCalled();+ });++ test('should validate on the way out when serialize is a schema', async () => {+ const recoverableState = new RecoverableState({+ defaultState: { counter: 0, message: '' },+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ serialize: stateSchema,+ });++ await recoverableState.initialize();+ (recoverableState.currentValue as any).counter = 'not a number';++ await expect(recoverableState.persistState()).rejects.toThrow(StateValidationError);+ });++ test('should support an asynchronous schema', async () => {+ await (await KeyValueStore.open()).setValue('test-key', { counter: 7 });++ const recoverableState = new RecoverableState({+ defaultState: { counter: 0 },+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ deserialize: {+ '~standard': {+ version: 1,+ vendor: 'test',+ validate: async (value) => ({ value: value as { counter: number } }),+ },+ },+ });++ expect(await recoverableState.initialize()).toEqual({ counter: 7 });+ });++ test('should accept a zod codec as deserialize, with encode as serialize', async () => {+ class Wrapped {+ constructor(readonly value: number) {}+ }++ const codec = z.codec(z.object({ counter: z.number() }), z.instanceof(Wrapped), {+ decode: ({ counter }) => new Wrapped(counter),+ encode: (wrapped) => ({ counter: wrapped.value }),+ });++ const store = await KeyValueStore.open();+ await store.setValue('test-key', { counter: 7 });++ const recoverableState = new RecoverableState({+ defaultState: () => new Wrapped(0),+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ deserialize: codec,+ serialize: (state) => codec.encode(state),+ });++ const loaded = await recoverableState.initialize();+ expect(loaded).toBeInstanceOf(Wrapped);+ expect(loaded.value).toBe(7);++ await recoverableState.persistState();+ expect(await store.getValue('test-key')).toEqual({ counter: 7 });+ });+ });++ describe('reset and resetStore', () => {+ test('reset should leave the persisted record alone', async () => {+ const store = await KeyValueStore.open();++ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ });++ await recoverableState.initialize();+ recoverableState.currentValue.counter = 42;+ await recoverableState.persistState();++ recoverableState.reset();++ expect(recoverableState.currentValue.counter).toBe(0);+ expect(await store.getValue('test-key')).toMatchObject({ counter: 42 });+ });++ test('reset should be undone by the next initialize, unless the record is cleared too', async () => {+ const build = () =>+ new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ });++ const first = build();+ await first.initialize();+ first.currentValue.counter = 42;+ await first.teardown();++ // Resetting in memory does not stop the record from coming back on the next load...+ first.reset();+ expect(await build().initialize()).toMatchObject({ counter: 42 });++ // ...clearing the record does.+ await first.resetStore();+ expect(await build().initialize()).toMatchObject({ counter: 0 });+ });++ test('resetStore should clear the record but leave the in-memory state alone', async () => {+ const store = await KeyValueStore.open();++ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ });++ await recoverableState.initialize();+ recoverableState.currentValue.counter = 42;+ await recoverableState.teardown();++ await recoverableState.resetStore();++ expect(await store.getValue('test-key')).toBeNull();+ expect(recoverableState.currentValue.counter).toBe(42);+ });++ test('resetStore should throw while PERSIST_STATE events are still being handled', async () => {+ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ });++ await recoverableState.initialize();++ await expect(recoverableState.resetStore()).rejects.toThrow('Use reset() to reset the state itself');++ await recoverableState.teardown();+ await expect(recoverableState.resetStore()).resolves.not.toThrow();+ });++ test('resetStore should be a no-op without a store', async () => {+ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ });++ await expect(recoverableState.resetStore()).resolves.not.toThrow();+ });+ });++ test('persistState should be a no-op when there is no state to write', async () => {+ const store = await KeyValueStore.open();+ const setValue = vi.spyOn(store, 'setValue');++ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ keyValueStore: store,+ });++ await expect(recoverableState.persistState()).resolves.not.toThrow();+ expect(setValue).not.toHaveBeenCalled();+ });++ test('persistState should be a no-op before initialization', async () => {+ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ });++ await expect(recoverableState.persistState()).resolves.not.toThrow();+ expect(await (await KeyValueStore.open()).getValue('test-key')).toBeNull();+ });++ describe('keyValueStore', () => {+ let namedStore: KeyValueStore;++ beforeEach(async () => {+ namedStore = await KeyValueStore.open({ name: 'named-store' });+ });++ const cases: [label: string, buildStore: () => KeyValueStore | PromiseLike<KeyValueStore>][] = [+ ['an open instance', () => namedStore],+ ['a pending open()', () => KeyValueStore.open({ name: 'named-store' })],+ ];++ test.for(cases)('should persist into the store given as %s', async ([, buildStore]) => {+ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ keyValueStore: buildStore(),+ });++ await recoverableState.initialize();+ recoverableState.currentValue.counter = 42;+ await recoverableState.persistState();++ expect(await namedStore.getValue('test-key')).toMatchObject({ counter: 42 });+ expect(await (await KeyValueStore.open()).getValue('test-key')).toBeNull();+ });++ test('should await a pending open() even without initialization', async () => {+ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ keyValueStore: KeyValueStore.open({ name: 'named-store' }),+ });++ recoverableState.reset();+ recoverableState.currentValue.counter = 42;+ await recoverableState.persistState();++ expect(await namedStore.getValue('test-key')).toMatchObject({ counter: 42 });+ });+ });++ test('should time out a persistence call that takes too long', async () => {+ const store = await KeyValueStore.open();+ vi.spyOn(store, 'setValue').mockImplementation(async () => sleep(1000));++ const recoverableState = new RecoverableState({+ defaultState,+ persistStateKey: 'test-key',+ persistenceEnabled: true,+ keyValueStore: store,+ persistenceTimeoutMillis: 50,+ });++ recoverableState.reset();++ await expect(recoverableState.persistState()).rejects.toThrow(/timed out after 0.05 seconds/);});});