Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

34.6 KB
Raw Blame

Public API Report File for "@crawlee/core"

Do not edit this file. It is a report generated by API Extractor.


import type { AllowedHttpMethods } from '@crawlee/types';
import { ArgumentValidationError } from '@crawlee/utils';
import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
import type { Awaitable } from '@crawlee/types';
import { BaseHttpClient } from '@crawlee/http-client';
import type { BatchAddRequestsResult } from '@crawlee/types';
import type { BetterIntervalID } from '@apify/utilities';
import type { Constructor } from '@crawlee/types';
import { CrawleeLogger } from '@crawlee/types';
import type { CrawleeLoggerOptions } from '@crawlee/types';
import type { DatasetBackend } from '@crawlee/types';
import type { DatasetInfo } from '@crawlee/types';
import { Dictionary } from '@crawlee/types';
import { EnqueueStrategy } from '@crawlee/utils';
import type { KeyValueStoreBackend } from '@crawlee/types';
import type { LiteralUnion } from 'type-fest';
import { Log } from '@apify/log';
import log from '@apify/log';
import { Logger } from '@apify/log';
import { LoggerJson } from '@apify/log';
import type { LoggerOptions } from '@apify/log';
import { LoggerText } from '@apify/log';
import { LogLevel } from '@apify/log';
import type { LogOptions } from '@crawlee/types';
import type { ProcessedRequest } from '@crawlee/types';
import type { ProxyInfo } from '@crawlee/types';
import type { QueueOperationInfo } from '@crawlee/types';
import type { ReadonlyDeep } from 'type-fest';
import type { RequestQueueBackend } from '@crawlee/types';
import type { RequestQueueInfo } from '@crawlee/types';
import type { RequestSchema } from '@crawlee/types';
import type { StandardSchemaV1 } from '@standard-schema/spec';
import type * as storage from '@crawlee/types';
import { StorageBackend } from '@crawlee/types';
import { StorageIdentifier } from '@crawlee/types';
import { z } from 'zod';
import { ZodType } from 'zod';

// @public (undocumented)
export interface AddRequestsBatchedOptions extends RequestQueueOperationOptions {
    batchSize?: number;
    maxNewRequests?: number;
    waitBetweenBatchesMillis?: number;
    waitForAllRequestsToBeAdded?: boolean;
}

// @public (undocumented)
export interface AddRequestsBatchedResult {
    // (undocumented)
    addedRequests: ProcessedRequest[];
    requestsOverLimit?: Source[];
    waitForAllRequestsToBeAdded: Promise<ProcessedRequest[]>;
}

// @public
export class AfterCommitError extends NonRetryableError {
    constructor(cause: unknown);
}

// @public
export class ApifyLogAdapter extends BaseCrawleeLogger {
    constructor(apifyLog: Log, options?: Partial<CrawleeLoggerOptions>);
    // (undocumented)
    protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
    // (undocumented)
    logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
}

export { ArgumentValidationError }

// @public
export abstract class BaseCrawleeLogger implements CrawleeLogger {
    constructor(options?: Partial<CrawleeLoggerOptions>);
    // (undocumented)
    child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
    protected abstract createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
    // (undocumented)
    debug(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
    // (undocumented)
    deprecated(message: string): void;
    // (undocumented)
    error(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
    // (undocumented)
    exception(exception: Error, message: string, data?: Record<string, unknown>): void;
    // (undocumented)
    getOptions(): CrawleeLoggerOptions;
    // (undocumented)
    info(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
    abstract logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
    // (undocumented)
    perf(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
    // (undocumented)
    setOptions(options: Partial<CrawleeLoggerOptions>): void;
    // (undocumented)
    softFail(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
    // (undocumented)
    warning(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
    // (undocumented)
    warningOnce(message: string): void;
}

// @public
export const coerceBoolean: z.ZodPreprocess<z.ZodBoolean, unknown>;

// @public (undocumented)
export const coerceNumber: z.ZodPreprocess<z.ZodNumber, unknown>;

// @public (undocumented)
export interface ConfigField<T extends z.ZodType = z.ZodType> {
    // (undocumented)
    envVar?: string | string[];
    // (undocumented)
    schema: T;
}

// @public (undocumented)
export interface Configuration extends ResolvedConfigValues {
}

// @public
export class Configuration {
    constructor(options?: ConfigurationInput);
    protected static fields: Record<string, ConfigField>;
    static getGlobalConfiguration(): Configuration;
}

// @public (undocumented)
export type ConfigurationInput = FieldsInput<typeof crawleeConfigFields>;

// @public @deprecated (undocumented)
export type ConfigurationOptions = ConfigurationInput;

// @public (undocumented)
export const crawleeConfigFields: {
    purgeOnStart: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
    maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
    availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
    memoryMbytes: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
    persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
    internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
    systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
    headless: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
    xvfb: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
    chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
    defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
    disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
    logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof LogLevel>, unknown>>>;
    persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
    storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
    containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
};

export { CrawleeLogger }

export { CrawleeLoggerOptions }

// @public
export function createStorageTransaction(options?: StorageTransactionOptions): StorageTransaction;

// @public
export class CriticalError extends NonRetryableError {
}

// @public
export class Dataset<Data extends Dictionary = Dictionary> {
    [Symbol.asyncIterator](): AsyncGenerator<Data, void, undefined>;
    // (undocumented)
    readonly configuration: Configuration;
    drop(): Promise<void>;
    entries(options?: DatasetIteratorOptions): AsyncIterable<[number, Data]> & Promise<[number, Data][]>;
    export(options?: DatasetExportOptions): Promise<Data[]>;
    exportTo(key: string, options?: DatasetExportToOptions, contentType?: string): Promise<Data[]>;
    exportToCSV(key: string, options?: Omit<DatasetExportToOptions, 'fromDataset'>): Promise<void>;
    static exportToCSV(key: string, options?: DatasetExportToOptions): Promise<void>;
    exportToJSON(key: string, options?: Omit<DatasetExportToOptions, 'fromDataset'>): Promise<void>;
    static exportToJSON(key: string, options?: DatasetExportToOptions): Promise<void>;
    forEach(iteratee: DatasetConsumer<Data>, options?: DatasetIteratorOptions, index?: number): Promise<void>;
    getData(options?: DatasetDataOptions): Promise<DatasetContent<Data>>;
    static getData<Data extends Dictionary = Dictionary>(options?: DatasetDataOptions): Promise<DatasetContent<Data>>;
    getInfo(): Promise<DatasetInfo>;
    // (undocumented)
    readonly id: string;
    map<R>(iteratee: DatasetMapper<Data, R>, options?: DatasetIteratorOptions): Promise<R[]>;
    // (undocumented)
    readonly name?: string;
    static open<Data extends Dictionary = Dictionary>(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<Dataset<Data>>;
    purge(): Promise<void>;
    pushData(data: Data | Data[]): Promise<void>;
    reduce(iteratee: DatasetReducer<Data, Data>): Promise<Data | undefined>;
    reduce(iteratee: DatasetReducer<Data, Data>, memo: undefined, options: DatasetIteratorOptions): Promise<Data | undefined>;
    reduce<T>(iteratee: DatasetReducer<T, Data>, memo: T, options?: DatasetIteratorOptions): Promise<T>;
    get stats(): DatasetStats;
    values(options?: DatasetIteratorOptions): AsyncIterable<Data> & Promise<Data[]>;
}

// @public
export interface DatasetConsumer<Data> {
    // (undocumented)
    (item: Data, index: number): Awaitable<void>;
}

// @public (undocumented)
export interface DatasetContent<Data> {
    count: number;
    desc?: boolean;
    items: Data[];
    limit: number;
    offset: number;
    total: number;
}

// @public (undocumented)
export interface DatasetDataOptions {
    clean?: boolean;
    desc?: boolean;
    fields?: string[];
    limit?: number;
    offset?: number;
    skipEmpty?: boolean;
    skipHidden?: boolean;
    unwind?: string;
}

// @public (undocumented)
export interface DatasetExportOptions extends Omit<DatasetDataOptions, 'offset' | 'limit'> {
    collectAllKeys?: boolean;
}

// @public (undocumented)
export interface DatasetExportToOptions extends DatasetExportOptions {
    // (undocumented)
    fromDataset?: string | StorageIdentifier;
    // (undocumented)
    toKVS?: string | StorageIdentifier;
}

// @public (undocumented)
export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset' | 'limit' | 'clean' | 'skipHidden' | 'skipEmpty'> {
}

// @public
export interface DatasetMapper<Data, R> {
    (item: Data, index: number): Awaitable<R>;
}

// @public
export interface DatasetReducer<T, Data> {
    // (undocumented)
    (memo: T, item: Data, index: number): Awaitable<T>;
}

// @public
export interface DatasetStats {
    readCount: number;
    writeCount: number;
}

// @public
export interface DefaultStorageIdentifier {
    // (undocumented)
    alias?: never;
    // (undocumented)
    id?: never;
    // (undocumented)
    name?: never;
}

export { EnqueueStrategy }

// @public
export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';

// @public (undocumented)
export abstract class EventManager {
    constructor(options: EventManagerOptions);
    close(): Promise<void>;
    // (undocumented)
    emit(event: EventTypeName, ...args: unknown[]): void;
    // (undocumented)
    protected events: AsyncEventEmitter<{}>;
    init(): Promise<void>;
    // (undocumented)
    protected initialized: boolean;
    // (undocumented)
    protected intervals: Intervals;
    // (undocumented)
    isInitialized(): boolean;
    // (undocumented)
    protected log: CrawleeLogger;
    // (undocumented)
    off(event: EventTypeName, listener?: (...args: any[]) => any): void;
    // (undocumented)
    on(event: EventTypeName, listener: (...args: any[]) => any): void;
}

// @public (undocumented)
export interface EventManagerOptions {
    persistStateIntervalMillis: number;
}

// @public
export interface EventStatusMessageData {
    crawlerId: string;
    isStatusMessageTerminal?: boolean;
    level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
    message: string;
}

// @public (undocumented)
export enum EventType {
    // (undocumented)
    ABORTING = "aborting",
    // (undocumented)
    EXIT = "exit",
    // (undocumented)
    MIGRATING = "migrating",
    // (undocumented)
    PERSIST_STATE = "persistState",
    // (undocumented)
    STATUS_MESSAGE = "statusMessage",
    // (undocumented)
    SYSTEM_INFO = "systemInfo"
}

// @public (undocumented)
export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit' | 'statusMessage';

// @public
export type ExplicitStorageIdentifier = {
    id: string;
    name?: never;
    alias?: never;
} | {
    id?: never;
    name: string;
    alias?: never;
} | {
    id?: never;
    name?: never;
    alias: string;
};

// @public (undocumented)
export function field<T extends z.ZodType>(schema: T, envVar?: string | string[]): ConfigField<T>;

// @public (undocumented)
export type FieldsInput<F extends Record<string, ConfigField>> = {
    [K in keyof F]?: z.output<F[K]['schema']>;
};

// @public (undocumented)
export type FieldsOutput<F extends Record<string, ConfigField>> = {
    [K in keyof F]: z.output<F[K]['schema']>;
};

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface Intervals {
    // (undocumented)
    persistState?: BetterIntervalID;
    // (undocumented)
    systemInfo?: BetterIntervalID;
}

// @public
export interface IProxyConfiguration {
    newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined>;
}

// @public
export interface IRequestLoader {
    [Symbol.asyncIterator](): AsyncGenerator<Request_2>;
    checkReadiness(): Promise<RequestSourceStatus>;
    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;
    getHandledCount(): Promise<number>;
    getPendingCount(): Promise<number>;
    getTotalCount(): Promise<number>;
    markRequestAsHandled(request: Request_2): Promise<RequestQueueOperationInfo | void | null>;
    toTandem?(requestManager?: IRequestManager): Promise<IRequestManager>;
}

// @public
export interface IRequestManager extends IRequestLoader {
    // (undocumented)
    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
    // (undocumented)
    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
    extendRequestProcessingTime?(request: Request_2, extensionSecs: number): Promise<boolean>;
    purge?(): Promise<void>;
    reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
    recordPacingSignal(signal: PacingSignal): boolean;
    setExpectedRequestProcessingTime?(processingTimeSecs: number): Promise<void>;
}

// @public
export interface IStorage {
    // (undocumented)
    id: string;
    // (undocumented)
    name?: string;
}

// @public
export interface KeyConsumer {
    // (undocumented)
    (key: string, index: number, info: {
        size: number;
    }): Awaitable<void>;
}

// @public
export class KeyValueStore {
    [Symbol.asyncIterator]<T = unknown>(): AsyncGenerator<[string, T], void, undefined>;
    // (undocumented)
    readonly backend: KeyValueStoreBackend;
    // (undocumented)
    readonly configuration: Configuration;
    drop(): Promise<void>;
    entries<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<[string, T]> & Promise<[string, T][]>;
    forEachKey(iteratee: KeyConsumer, options?: KeyValueStoreIteratorOptions): Promise<void>;
    // (undocumented)
    getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
    // (undocumented)
    static getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
    getPublicUrl(key: string): Promise<string | undefined>;
    getRecord(key: string): Promise<KeyValueStoreRawRecord | null>;
    getValue<T = unknown>(key: string): Promise<T | null>;
    getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
    // (undocumented)
    readonly id: string;
    keys(options?: KeyValueStoreIteratorOptions): AsyncIterable<string> & Promise<string[]>;
    // (undocumented)
    readonly name?: string;
    static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<KeyValueStore>;
    purge(): Promise<void>;
    recordExists(key: string): Promise<boolean>;
    static recordExists(key: string): Promise<boolean>;
    setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
    get stats(): KeyValueStoreStats;
    values<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<T> & Promise<T[]>;
}

// @public (undocumented)
export interface KeyValueStoreIteratorOptions {
    prefix?: string;
}

// @public
export interface KeyValueStoreRawRecord {
    // (undocumented)
    contentType: string | null;
    // (undocumented)
    value: Buffer | ArrayBuffer;
}

// @public
export interface KeyValueStoreStats {
    deleteCount: number;
    listCount: number;
    readCount: number;
    writeCount: number;
}

// @public (undocumented)
export interface LoadSignalInfo {
    // (undocumented)
    actualRatio: number;
    // (undocumented)
    isOverloaded: boolean;
    // (undocumented)
    limitRatio: number;
}

// @public (undocumented)
export class LocalEventManager extends EventManager {
    constructor(options: LocalEventManagerOptions);
    // (undocumented)
    close(): Promise<void>;
    static fromConfiguration(configuration?: Configuration): LocalEventManager;
    init(): Promise<void>;
}

// @public (undocumented)
export interface LocalEventManagerOptions extends EventManagerOptions {
    systemInfoIntervalMillis: number;
}

export { Log }

export { log }

export { Logger }

export { LoggerJson }

export { LoggerOptions }

export { LoggerText }

export { LogLevel }

export { LogOptions }

// @public (undocumented)
export class MemoryStorageBackend implements storage.StorageBackend {
    constructor(options?: MemoryStorageOptions);
    // (undocumented)
    createDatasetBackend(options?: storage.StorageIdentifier): Promise<storage.DatasetBackend>;
    // (undocumented)
    createKeyValueStoreBackend(options?: storage.StorageIdentifier): Promise<storage.KeyValueStoreBackend>;
    // (undocumented)
    createRequestQueueBackend(options?: storage.StorageIdentifier): Promise<storage.RequestQueueBackend>;
    getStorageBackendCacheKey(): string;
    // (undocumented)
    readonly logger?: CrawleeLogger;
    purge(): Promise<void>;
    // (undocumented)
    storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
    teardown(): Promise<void>;
}

// @public (undocumented)
export interface MemoryStorageOptions {
    logger?: CrawleeLogger;
}

// @public
export class NonRetryableError extends Error {
}

// @public
export type PacingScope = LiteralUnion<'hostname' | 'registrableDomain', string>;

// @public
export type PacingSignal = {
    reason: 'rateLimited';
    url: string;
    waitMs?: number;
    scope?: PacingScope;
} | {
    reason: 'minInterval';
    url: string;
    intervalMs: number;
    scope: PacingScope;
} | {
    reason: 'minIntervalEverywhere';
    intervalMs: number;
    scope: PacingScope;
};

// @public
export function parseValue(body: Buffer | ArrayBuffer | string, contentTypeHeader: string | null): string | Buffer | ArrayBuffer | Record<string, unknown>;

// @public
export class ProxyConfiguration implements IProxyConfiguration {
    constructor(options?: ProxyConfigurationOptions);
    // (undocumented)
    readonly isManInTheMiddle = false;
    newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined>;
    newUrl(): Promise<string | undefined>;
}

// @public (undocumented)
export interface ProxyConfigurationFunction {
    // (undocumented)
    (): string | null | Promise<string | null>;
}

// @public (undocumented)
export interface ProxyConfigurationOptions {
    newUrlFunction?: ProxyConfigurationFunction;
    proxyUrls?: (string | null)[];
}

// Not exported by the entry point; reachable only as a referenced type.
// @public
interface PurgeDefaultStorageOptions {
    // (undocumented)
    configuration?: Configuration;
    onlyPurgeOnce?: boolean;
    // (undocumented)
    storageBackend?: StorageBackend;
}

// @public
export function purgeDefaultStorages(options?: PurgeDefaultStorageOptions): Promise<void>;

// @public
export function purgeDefaultStorages(configuration?: Configuration, storageBackend?: StorageBackend): Promise<void>;

// @public (undocumented)
export interface RecordOptions {
    contentType?: string;
}

// @public
export class RecoverableState<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> {
    constructor(options: RecoverableStateOptions<TStateModel, TPersistedState>);
    get currentValue(): TStateModel;
    initialize(): Promise<TStateModel>;
    persistState(eventData?: Record<string, unknown>): Promise<void>;
    reset(): void;
    resetStore(): Promise<void>;
    teardown(): Promise<void>;
}

// @public
export interface RecoverableStateBaseOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> extends RecoverableStatePersistenceOptions {
    configuration?: Configuration;
    contentType?: string;
    defaultState: TStateModel | (() => TStateModel);
    deserialize?: StateConversion<TPersistedState, TStateModel>;
    logger?: CrawleeLogger;
    serialize?: StateConversion<TStateModel, TPersistedState>;
}

// @public
export type RecoverableStateOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> = RecoverableStateBaseOptions<TStateModel, TPersistedState> & ({
    contentType?: undefined;
} | Required<Pick<RecoverableStateBaseOptions<TStateModel, TPersistedState>, 'serialize' | 'deserialize' | 'contentType'>>);

// @public (undocumented)
export interface RecoverableStatePersistenceOptions {
    keyValueStore?: KeyValueStore | PromiseLike<KeyValueStore>;
    persistenceEnabled?: boolean;
    persistenceTimeoutMillis?: number;
    persistStateKey: string;
}

// @public
class Request_2<UserData extends Dictionary = Dictionary> {
    constructor(options: RequestOptions<UserData>);
    errorMessages: string[];
    static fromSchema<UserData extends Dictionary = Dictionary>(schema: RequestSchema): Request_2<UserData>;
    handledAt?: string;
    headers?: Record<string, string>;
    id?: string;
    intoFetchAPIRequest(): Request;
    get label(): string | undefined;
    set label(value: string | undefined);
    loadedUrl?: string;
    method: AllowedHttpMethods;
    noRetry: boolean;
    payload?: string;
    retryCount: number;
    uniqueKey: string;
    url: string;
    userData: UserData;
}
export { Request_2 as Request }

// @public
export class RequestList implements IRequestLoader {
    // (undocumented)
    [Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
    // (undocumented)
    checkReadiness(): Promise<RequestLoaderStatus>;
    // (undocumented)
    fetchNextRequest(): Promise<Request_2 | null>;
    // (undocumented)
    getHandledCount(): Promise<number>;
    getPendingCount(): Promise<number>;
    getState(): RequestListState;
    getTotalCount(): Promise<number>;
    // (undocumented)
    markRequestAsHandled(request: Request_2): Promise<void>;
    static open(listNameOrOptions: string | null | RequestListOptions, sources?: RequestListSource[], options?: RequestListOptions): Promise<RequestList>;
    // (undocumented)
    persistState(): Promise<void>;
    teardown(): Promise<void>;
    toTandem(requestManager?: IRequestManager): Promise<IRequestManager>;
}

// @public (undocumented)
export interface RequestListOptions {
    httpClient?: BaseHttpClient;
    keepDuplicateUrls?: boolean;
    persistRequestsKey?: string;
    persistStateKey?: string;
    proxyConfiguration?: IProxyConfiguration;
    sources?: RequestListSource[];
    sourcesFunction?: RequestListSourcesFunction;
    state?: RequestListState;
}

// @public (undocumented)
export type RequestListSource = string | Source;

// @public (undocumented)
export type RequestListSourcesFunction = () => Promise<RequestListSource[]>;

// @public
export interface RequestListState {
    inProgress: string[];
    nextIndex: number;
    nextUniqueKey: string | null;
}

// @public
export type RequestLoaderStatus = Exclude<RequestSourceStatus, {
    status: 'stalled';
}>;

// @public
export class RequestManagerTandem implements IRequestManager {
    // (undocumented)
    [Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
    constructor(requestLoader: IRequestLoader, requestManager: IRequestManager | (() => IRequestManager | Promise<IRequestManager>));
    // (undocumented)
    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
    // (undocumented)
    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
    checkReadiness(): Promise<RequestSourceStatus>;
    // (undocumented)
    extendRequestProcessingTime(request: Request_2, extensionSecs: number): Promise<boolean>;
    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;
    // (undocumented)
    getHandledCount(): Promise<number>;
    // (undocumented)
    getPendingCount(): Promise<number>;
    // (undocumented)
    getTotalCount(): Promise<number>;
    // (undocumented)
    markRequestAsHandled(request: Request_2): Promise<RequestQueueOperationInfo | void | null>;
    purge(): Promise<void>;
    // (undocumented)
    reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
    recordPacingSignal(signal: PacingSignal): boolean;
    setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void>;
}

// @public
export interface RequestOptions<UserData extends Dictionary = Dictionary> {
    alwaysEnqueue?: boolean;
    crawlDepth?: number;
    headers?: Record<string, string>;
    keepUrlFragment?: boolean;
    label?: string;
    maxRetries?: number;
    method?: AllowedHttpMethods | Lowercase<AllowedHttpMethods>;
    noRetry?: boolean;
    payload?: string;
    sessionId?: string;
    skipNavigation?: boolean;
    uniqueKey?: string;
    url: string;
    useExtendedUniqueKey?: boolean;
    userData?: UserData;
}

// @public
export class RequestQueue implements IStorage, IRequestManager {
    // (undocumented)
    [Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
    addRequests(requestsLike: RequestsLike, options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
    addRequestsBatched(requests: ReadonlyDeep<RequestsLike>, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
    // (undocumented)
    readonly backend: RequestQueueBackend;
    checkReadiness(): Promise<RequestLoaderStatus>;
    drop(): Promise<void>;
    extendRequestProcessingTime(request: Request_2, extensionSecs: number): Promise<boolean>;
    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;
    getHandledCount(): Promise<number>;
    getInfo(): Promise<RequestQueueInfo>;
    getPendingCount(): Promise<number>;
    getRequest<T extends Dictionary = Dictionary>(uniqueKey: string): Promise<Request_2<T> | null>;
    getTotalCount(): Promise<number>;
    // (undocumented)
    readonly id: string;
    // (undocumented)
    readonly log: CrawleeLogger;
    markRequestAsHandled(request: Request_2): Promise<RequestQueueOperationInfo | null>;
    // (undocumented)
    readonly name?: string;
    static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<RequestQueue>;
    purge(): Promise<void>;
    reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
    recordPacingSignal(_signal: PacingSignal): boolean;
    setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void>;
    get stats(): RequestQueueStats;
}

// @public (undocumented)
export interface RequestQueueOperationInfo extends QueueOperationInfo {
    // (undocumented)
    forefront: boolean;
    // (undocumented)
    uniqueKey: string;
}

// @public (undocumented)
export interface RequestQueueOperationOptions {
    forefront?: boolean;
}

// @public
export interface RequestQueueStats {
    headItemReadCount: number;
    writeCount: number;
}

// @public (undocumented)
export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];

// @public
export type RequestSourceStatus = {
    status: 'ready';
} | {
    status: 'waiting';
    readyAt?: number;
} | {
    status: 'stalled';
    reason: string;
} | {
    status: 'finished';
};

// @public
export class RequestValidationError extends NonRetryableError {
    constructor(label: string | symbol, issues: readonly SchemaIssue[]);
    // (undocumented)
    readonly issues: readonly SchemaIssue[];
    // (undocumented)
    readonly label: string | symbol;
}

// @public (undocumented)
export type ResolvedConfigValues = FieldsOutput<typeof crawleeConfigFields>;

// @public
export interface SchemaIssue {
    // (undocumented)
    readonly message: string;
    // (undocumented)
    readonly path?: readonly (PropertyKey | {
        key: PropertyKey;
    })[];
}

// @public
export function serializeValue(value: unknown, contentType?: string): {
    value: Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
    contentType: string;
};

// @public
export class ServiceConflictError extends Error {
    constructor(serviceName: string, newValue: unknown, existingValue: unknown);
}

// @public
export class ServiceLocator implements ServiceLocatorInterface {
    constructor(configuration?: Configuration, eventManager?: EventManager, storageBackend?: StorageBackend, logger?: CrawleeLogger);
    // (undocumented)
    getChildLog(prefix: string): CrawleeLogger;
    // (undocumented)
    getConfiguration(): Configuration;
    // (undocumented)
    getEventManager(): EventManager;
    // (undocumented)
    getLogger(): CrawleeLogger;
    // (undocumented)
    getStorageBackend(): StorageBackend;
    // (undocumented)
    setConfiguration(configuration: Configuration): void;
    // (undocumented)
    setEventManager(eventManager: EventManager): void;
    // (undocumented)
    setLogger(logger: CrawleeLogger): void;
    // (undocumented)
    setStorageBackend(storageBackend: StorageBackend): void;
}

// @public (undocumented)
export const serviceLocator: ServiceLocatorInterface;

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface ServiceLocatorInterface {
    getChildLog(prefix: string): CrawleeLogger;
    getConfiguration(): Configuration;
    getEventManager(): EventManager;
    getLogger(): CrawleeLogger;
    getStorageBackend(): StorageBackend;
    setConfiguration(configuration: Configuration): void;
    setEventManager(eventManager: EventManager): void;
    setLogger(logger: CrawleeLogger): void;
    setStorageBackend(storageBackend: StorageBackend): void;
}

// @public
export class SessionError extends Error {
    constructor(message?: string);
}

// @public
export class SessionRetiredError extends SessionError {
}

// @public (undocumented)
export type Source = (Partial<RequestOptions> & {
    requestsFromUrl?: string;
    regex?: RegExp;
}) | Request_2;

// @public
export type StateConversion<TFrom, TTo> = ((value: TFrom) => Awaitable<TTo>) | StandardSchemaV1<TFrom, TTo>;

// @public
export class StateValidationError extends Error {
    constructor(persistStateKey: string, issues: readonly SchemaIssue[]);
    // (undocumented)
    readonly issues: readonly SchemaIssue[];
    // (undocumented)
    readonly persistStateKey: string;
}

export { StorageBackend }

export { StorageIdentifier }

// @public
export class StorageInstanceManager {
    clearCache(): void;
    openStorage<TStorage extends IStorage>(cls: Constructor<TStorage>, input: (ExplicitStorageIdentifier | DefaultStorageIdentifier) & {
        backendOpener: () => Promise<DatasetBackend | KeyValueStoreBackend | RequestQueueBackend>;
        backendCacheKey: string;
    }): Promise<TStorage>;
    removeFromCache(instance: IStorage): void;
}

// @public
export interface StorageOpenOptions {
    configuration?: Configuration;
    httpClient?: BaseHttpClient;
    proxyConfiguration?: IProxyConfiguration;
    storageBackend?: StorageBackend;
}

// @public
export class StorageTransaction implements StorageTransactionView {
    afterCommit(callback: (error?: Error) => Awaitable<void>): void;
    commit(): Promise<void>;
    // (undocumented)
    get datasetItems(): {
        item: Dictionary;
        datasetId: string;
    }[];
    dispose(): void;
    // (undocumented)
    get enqueuedUrls(): {
        url: string;
        label?: string;
    }[];
    get isActive(): boolean;
    // (undocumented)
    get keyValueStoreChanges(): Record<string, Record<string, {
        changedValue: unknown;
        options?: RecordOptions;
    }>>;
    rollback(): void;
    run<T>(callback: () => Awaitable<T>): Promise<T>;
    // (undocumented)
    get state(): StorageTransactionState;
}

// @public (undocumented)
export interface StorageTransactionOptions {
    commitTimeoutMillis?: number;
    policy?: Partial<StorageWritePolicy>;
}

// @public (undocumented)
export type StorageTransactionState = 'open' | 'committing' | 'committed' | 'failed' | 'rolledBack';

// @public
export interface StorageTransactionView {
    readonly datasetItems: {
        item: Dictionary;
        datasetId: string;
    }[];
    readonly enqueuedUrls: {
        url: string;
        label?: string;
    }[];
    readonly keyValueStoreChanges: Record<string, Record<string, {
        changedValue: unknown;
        options?: RecordOptions;
    }>>;
    // (undocumented)
    readonly state: StorageTransactionState;
}

// @public
export type StorageWriteMode = 'deferred' | 'writeThrough';

// @public
export interface StorageWritePolicy {
    requestQueue: StorageWriteMode;
}

// @public
export type SyncStateConversion<TFrom, TTo> = ((value: TFrom) => TTo) | StandardSchemaV1<TFrom, TTo>;

// @public
export interface SystemInfo {
    // (undocumented)
    cpuInfo: LoadSignalInfo;
    // (undocumented)
    eventLoopInfo: LoadSignalInfo;
    isSystemIdle: boolean;
    loadSignalInfo?: Record<string, LoadSignalInfo>;
    // (undocumented)
    memCurrentBytes?: number;
    // (undocumented)
    memInfo: LoadSignalInfo;
    // (undocumented)
    memTotalBytes?: number;
    // (undocumented)
    storageBackendInfo: LoadSignalInfo;
}

// @public
export function useState<State extends Dictionary = Dictionary>(name?: string, defaultValue?: State, options?: UseStateOptions): Promise<State>;

// @public (undocumented)
export interface UseStateOptions {
    // (undocumented)
    configuration?: Configuration;
    keyValueStoreName?: string | null;
}

// @public
export function withDirectStorageAccess<T>(callback: () => Awaitable<T>): Promise<T>;

// @public
export function withStorageTransaction<T>(callback: (transaction: StorageTransaction) => Awaitable<T>, options?: StorageTransactionOptions): Promise<T>;

// (No @packageDocumentation comment for this package)