Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { Readable } from 'node:stream';
import { addTimeoutToPromise, storage as timeoutStorage } from '@apify/timeout';
import type { Configuration } from './configuration.js';
import { StateValidationError } from './errors.js';
import { EventType } from './events/event-manager.js';
import type { CrawleeLogger } from './log.js';
import { serviceLocator } from './service-locator.js';
import { KeyValueStore } from './storages/key-value-store.js';
import type { Awaitable } from '@crawlee/types';
import type { StandardSchemaV1 } from '@standard-schema/spec';
const DEFAULT_PERSISTENCE_TIMEOUT_MILLIS = 60_000;
/**
* One direction of the conversion between the state model and its persisted form - either a plain function, or a
* [Standard Schema](https://standardschema.dev) whose validated output is the result.
*
* A schema that fails to validate makes {@apilink RecoverableState} throw a {@apilink StateValidationError}. Zod
* codecs work directly, as their validation *is* the decode direction; use `(state) => codec.encode(state)` for the
* other one.
*/
export type StateConversion<TFrom, TTo> = ((value: TFrom) => Awaitable<TTo>) | StandardSchemaV1<TFrom, TTo>;
/**
* A {@apilink StateConversion} for a caller that cannot await one - {@apilink Statistics}, whose `toJSON()` is
* synchronous, being the reason this exists.
*
* Only the function arm can be narrowed here: a Standard Schema is free to validate asynchronously, so a schema
* that does is rejected when it runs rather than when it is passed.
*/
export type SyncStateConversion<TFrom, TTo> = ((value: TFrom) => TTo) | StandardSchemaV1<TFrom, TTo>;
/**
* Applies a {@apilink SyncStateConversion}, throwing a {@apilink StateValidationError} for a schema that rejects
* the value.
*
* @internal
*/
export function convertStateSync<TFrom, TTo>(
conversion: SyncStateConversion<TFrom, TTo>,
value: TFrom,
persistStateKey: string,
): TTo {
if (typeof conversion === 'function') {
return conversion(value);
}
const result = conversion['~standard'].validate(value);
if ('then' in result) {
throw new Error(
`The state conversion for '${persistStateKey}' validated asynchronously, which this caller cannot await.`,
);
}
if (result.issues) {
throw new StateValidationError(persistStateKey, result.issues);
}
return result.value;
}
export interface RecoverableStatePersistenceOptions {
/**
* The key under which the state is stored in the KeyValueStore
*/
persistStateKey: string;
/**
* Flag to enable or disable state persistence
*/
persistenceEnabled?: boolean;
/**
* The KeyValueStore to persist into, defaulting to the default store. Accepts a pending
* {@apilink KeyValueStore.open} so that callers do not have to be async to point at a specific store.
*/
keyValueStore?: KeyValueStore | PromiseLike<KeyValueStore>;
/**
* Time limit for a single load or save of the state, in milliseconds.
* @default 60_000
*/
persistenceTimeoutMillis?: number;
}
/**
* The fields of {@apilink RecoverableStateOptions}, without the constraint tying `contentType` to the conversions.
*/
export interface RecoverableStateBaseOptions<
TStateModel = Record<string, unknown>,
TPersistedState = TStateModel,
> extends RecoverableStatePersistenceOptions {
/**
* The state used when no persisted state is found, and the state {@apilink RecoverableState.reset} restores.
*
* A plain value is deep-copied with `structuredClone` each time it is used, so pass a factory for a state
* that `structuredClone` cannot rebuild - one holding class instances, say, or one derived from a schema.
*/
defaultState: TStateModel | (() => TStateModel);
/**
* A logger instance for logging operations related to state persistence
*/
logger?: CrawleeLogger;
/**
* Configuration instance to use when opening the KeyValueStore
*/
configuration?: Configuration;
/**
* Optional conversion of the state to a plain JSON-serializable value before it is persisted.
* If not provided, the state is persisted as is.
*
* With {@apilink RecoverableStateBaseOptions.contentType} set, it has to produce what
* {@apilink KeyValueStore.setValue} accepts alongside an explicit content type - a `string`, a `Buffer` or a
* stream.
*/
serialize?: StateConversion<TStateModel, TPersistedState>;
/**
* Optional conversion of a persisted value back to the state model, and the place to validate a record before
* trusting it. If not provided, the persisted value is used as is.
*
* With {@apilink RecoverableStateBaseOptions.contentType} set, it receives a `Readable` of the record bytes
* instead of a parsed value.
*/
deserialize?: StateConversion<TPersistedState, TStateModel>;
/**
* Content type of the persisted record. Setting it hands the record encoding over to
* {@apilink RecoverableStateBaseOptions.serialize} and {@apilink RecoverableStateBaseOptions.deserialize}, both of
* which are then required - the default JSON codec is bypassed in both directions. Meant for a state too large
* for `JSON.stringify`, which `serialize` can then stream out instead.
*/
contentType?: string;
}
/**
* Options for configuring the RecoverableState
*/
export type RecoverableStateOptions<
TStateModel = Record<string, unknown>,
TPersistedState = TStateModel,
> = RecoverableStateBaseOptions<TStateModel, TPersistedState> &
(
| { contentType?: undefined }
| Required<
Pick<
RecoverableStateBaseOptions<TStateModel, TPersistedState>,
'serialize' | 'deserialize' | 'contentType'
>
>
);
/**
* A class for managing persistent recoverable state using a plain JavaScript object.
*
* This class facilitates state persistence to a `KeyValueStore`, allowing data to be saved and retrieved
* across migrations or restarts. It manages the loading, saving, and resetting of state data,
* with optional persistence capabilities.
*
* The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
* The class automatically hooks into the event system to persist state when needed.
*/
export class RecoverableState<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> {
readonly #defaultState: () => TStateModel;
#state: TStateModel | null = null;
#initialized = false;
#recordLoaded = false;
#listening = false;
readonly #persistenceEnabled: boolean;
readonly #persistStateKey: string;
readonly #persistenceTimeoutMillis: number;
readonly #configuration?: Configuration;
#keyValueStore: KeyValueStore | PromiseLike<KeyValueStore> | null;
readonly #log: CrawleeLogger;
readonly #serialize: (state: TStateModel) => Promise<TPersistedState>;
readonly #deserialize: (persistedState: TPersistedState) => Promise<TStateModel>;
readonly #contentType?: string;
readonly #persistStateQuietly: (eventData?: Record<string, unknown>) => Promise<void>;
/**
* Initialize a new recoverable state object.
*
* @param options Configuration options for the recoverable state
*/
constructor(options: RecoverableStateOptions<TStateModel, TPersistedState>) {
const { defaultState } = options;
this.#defaultState =
typeof defaultState === 'function'
? (defaultState as () => TStateModel)
: () => structuredClone(defaultState);
this.#persistStateKey = options.persistStateKey;
this.#persistenceEnabled = options.persistenceEnabled ?? false;
this.#persistenceTimeoutMillis = options.persistenceTimeoutMillis ?? DEFAULT_PERSISTENCE_TIMEOUT_MILLIS;
this.#configuration = options.configuration;
this.#keyValueStore = options.keyValueStore ?? null;
this.#log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
this.#contentType = options.contentType;
if (this.#contentType !== undefined && (options.serialize === undefined || options.deserialize === undefined)) {
throw new Error(
`A 'contentType' for the state persisted under key '${this.#persistStateKey}' requires both 'serialize' and 'deserialize' - the record is no longer JSON the default codec can handle.`,
);
}
this.#serialize = this.#toConversion(options.serialize);
this.#deserialize = this.#toConversion(options.deserialize);
// A pending open() may reject before anything here awaits it. Observe the rejection so it surfaces on
// first use rather than as an unhandled rejection - most immediately when persistence is disabled and
// nothing ever awaits the store.
if (this.#keyValueStore !== null) {
Promise.resolve(this.#keyValueStore).then(undefined, () => {});
}
// The automatic persists, where a rejection has nowhere useful to go - the event manager does not catch
// listener errors, and throwing from teardown would bury the outcome of the work it cleans up after.
this.#persistStateQuietly = async (eventData) =>
this.persistState(eventData).catch((error) =>
this.#log.warning(`Failed to persist the state under key '${this.#persistStateKey}'.`, { error }),
);
}
/** Normalizes a conversion option into a function. Absent conversions pass the value through unchanged. */
#toConversion<TFrom, TTo>(conversion: StateConversion<TFrom, TTo> | undefined): (value: TFrom) => Promise<TTo> {
if (conversion === undefined) {
return async (value) => value as unknown as TTo;
}
if (typeof conversion === 'function') {
return async (value) => conversion(value);
}
return async (value) => {
const result = await conversion['~standard'].validate(value);
if (result.issues) {
throw new StateValidationError(this.#persistStateKey, result.issues);
}
return result.value;
};
}
/**
* Initialize the recoverable state.
*
* If persistence is enabled, this method loads the saved state and registers the object to listen for
* PERSIST_STATE events. A state established beforehand by {@apilink RecoverableState.reset} survives if there
* is no record to restore.
*
* Calling this again after a {@apilink RecoverableState.teardown} starts a new persistence window - the
* listener is registered again. The record is not reloaded: the in-memory state is what the teardown wrote, and
* reloading it would only drop whatever the deserialization leaves out.
*
* @returns The loaded state object
*/
async initialize(): Promise<TStateModel> {
if (this.#initialized) {
return this.currentValue;
}
if (this.#persistenceEnabled) {
await this.#openKeyValueStore();
serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.#persistStateQuietly);
this.#listening = true;
}
// Flipped before the record is loaded, so that a caller catching a `StateValidationError` is left with a
// fully wired object running on the default state rather than a half-initialized one.
this.#initialized = true;
this.#state ??= this.#defaultState();
if (!this.#recordLoaded) {
this.#recordLoaded = true;
await this.#loadSavedState();
}
return this.currentValue;
}
/**
* Clean up resources used by the recoverable state.
*
* If persistence is enabled, this method deregisters the object from PERSIST_STATE events
* and persists the current state one last time, warning rather than throwing if that write fails - cleanup
* runs when the work is already done, and failing it would bury whatever the caller was doing. The in-memory
* state is left alone, and {@apilink RecoverableState.initialize} can be called again to open a new
* persistence window.
*/
async teardown(): Promise<void> {
this.#initialized = false;
if (!this.#persistenceEnabled) {
return;
}
serviceLocator.getEventManager().off(EventType.PERSIST_STATE, this.#persistStateQuietly);
this.#listening = false;
await this.#persistStateQuietly();
}
/**
* Get the current state.
*
* Throws until the state has been established, by either {@apilink RecoverableState.initialize} or the
* synchronous {@apilink RecoverableState.reset} - the latter being how a caller that cannot await in its
* constructor gets a usable state right away.
*/
get currentValue(): TStateModel {
if (this.#state === null) {
throw new Error('Recoverable state has not yet been loaded - call initialize() or reset() first');
}
return this.#state;
}
/**
* Reset the in-memory state to the default values, leaving any persisted record alone.
*
* Use {@apilink RecoverableState.resetStore} to clear the persisted record as well.
*/
reset(): void {
this.#state = this.#defaultState();
}
/**
* Clear the persisted state record, leaving the in-memory state alone.
*
* This is a between-lifecycles operation - its point is to stop the next {@apilink RecoverableState.initialize}
* from restoring the record, so it throws while PERSIST_STATE events are still being handled, where the next
* one would write the record straight back. Use {@apilink RecoverableState.reset} to reset the state itself,
* or {@apilink RecoverableState.teardown} before clearing the record.
*
* A no-op if persistence is disabled.
*/
async resetStore(): Promise<void> {
if (this.#listening) {
throw new Error(
`Cannot clear the state persisted under key '${this.#persistStateKey}' while it is still being persisted periodically - the next PERSIST_STATE event would write it straight back. Use reset() to reset the state itself, or teardown() before clearing the record.`,
);
}
if (!this.#persistenceEnabled) {
return;
}
const keyValueStore = await this.#openKeyValueStore();
await this.#withTimeout(
async () => keyValueStore.setValue(this.#persistStateKey, null),
'Clearing the persisted state',
);
}
/**
* Persist the current state to the KeyValueStore.
*
* This method is typically called in response to a PERSIST_STATE event, but can also be called
* directly when needed. It is a no-op if persistence is disabled, if no KeyValueStore is available yet, or if
* there is no state to write. A failed write only rejects here - the periodic and teardown ones warn instead.
*
* @param eventData Optional data associated with a PERSIST_STATE event
*/
async persistState(eventData?: Record<string, unknown>): Promise<void> {
if (!this.#persistenceEnabled || this.#state === null) {
return;
}
const keyValueStore = await this.#resolveKeyValueStore();
if (keyValueStore === null) {
return;
}
this.#log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
const serializedState = await this.#serialize(this.currentValue);
await this.#withTimeout(
async () =>
this.#contentType === undefined
? keyValueStore.setValue(this.#persistStateKey, serializedState)
: keyValueStore.setValue(this.#persistStateKey, serializedState, {
contentType: this.#contentType,
}),
'Persisting the state',
);
}
/** Falls back to the default store when none was supplied - persistence has to write somewhere. */
async #openKeyValueStore(): Promise<KeyValueStore> {
this.#keyValueStore ??= KeyValueStore.open(null, {
configuration: this.#configuration ?? serviceLocator.getConfiguration(),
});
return (await this.#resolveKeyValueStore())!;
}
/** Awaits a store handed over as a pending `open()`, keeping the resolved instance for later calls. */
async #resolveKeyValueStore(): Promise<KeyValueStore | null> {
if (this.#keyValueStore === null) {
return null;
}
this.#keyValueStore = await this.#keyValueStore;
return this.#keyValueStore;
}
/**
* Load the saved state from the KeyValueStore. Leaves the current state alone if there is no record to load.
*/
async #loadSavedState(): Promise<void> {
if (!this.#persistenceEnabled) {
return;
}
const keyValueStore = await this.#resolveKeyValueStore();
if (keyValueStore === null) {
return;
}
// With a content type, the record is whatever `serialize` produced, so the codec's parse is skipped and
// `deserialize` gets the bytes as a stream - the shape a streaming parser wants.
// TODO: read the record as a stream instead of buffering it and wrapping (https://github.com/apify/crawlee/issues/2929).
const storedState = await this.#withTimeout(async () => {
if (this.#contentType === undefined) {
return keyValueStore.getValue(this.#persistStateKey);
}
const record = await keyValueStore.getRecord(this.#persistStateKey);
if (record === null) {
return null;
}
return Readable.from(Buffer.isBuffer(record.value) ? record.value : Buffer.from(record.value));
}, 'Loading the persisted state');
if (storedState === null || storedState === undefined) {
return;
}
this.#state = await this.#deserialize(storedState as TPersistedState);
}
async #withTimeout<T>(operation: () => Promise<T>, description: string): Promise<T> {
// `@apify/timeout` shares one `AbortController` across nested frames and `KeyValueStore` checks it on
// every operation, so a teardown-time persist running inside an already-expired request handler timeout
// would be aborted before it started. Hence a fresh timeout context.
return timeoutStorage.exit(async () =>
addTimeoutToPromise(
operation,
this.#persistenceTimeoutMillis,
`${description} under key '${this.#persistStateKey}' timed out after ${this.#persistenceTimeoutMillis / 1000} seconds.`,
),
);
}
}