Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Public API Report File for "@crawlee/types"
Do not edit this file. It is a report generated by API Extractor.
import type { Readable } from 'node:stream';
// @public (undocumented)
export type AllowedHttpMethods = 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH' | 'get' | 'head' | 'post' | 'put' | 'delete' | 'trace' | 'options' | 'connect' | 'patch';
// @public (undocumented)
export type Awaitable<T> = T | PromiseLike<T>;
// @public
export interface BaseHttpClient {
sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
}
// @public (undocumented)
export interface BatchAddRequestsResult {
// (undocumented)
processedRequests: ProcessedRequest[];
// (undocumented)
unprocessedRequests: UnprocessedRequest[];
}
// @public (undocumented)
export type Constructor<T = unknown> = new (...args: any[]) => T;
// @public (undocumented)
export interface Cookie {
domain?: string;
expires?: number;
httpOnly?: boolean;
name: string;
path?: string;
priority?: 'Low' | 'Medium' | 'High';
sameParty?: boolean;
sameSite?: 'Strict' | 'Lax' | 'None';
secure?: boolean;
sourcePort?: number;
sourceScheme?: 'Unset' | 'NonSecure' | 'Secure';
url?: string;
value: string;
}
// @public
export interface CookieJar {
// (undocumented)
clone(): Promise<CookieJar>;
// (undocumented)
getCookies(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<SessionCookie[]>;
// (undocumented)
getCookieString(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string>;
// (undocumented)
getSetCookieStrings(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string[] | undefined>;
// (undocumented)
serialize(): Promise<SerializedCookieJar>;
// (undocumented)
setCookie(cookie: string | SessionCookie, url: string | URL, options?: CookieJarSetCookieOptions): Promise<SessionCookie | undefined>;
// (undocumented)
toJSON(): SerializedCookieJar | undefined;
}
// @public
export interface CookieJarGetCookiesOptions {
// (undocumented)
allPaths?: boolean;
// (undocumented)
expire?: boolean;
// (undocumented)
http?: boolean;
// (undocumented)
sameSiteContext?: 'none' | 'lax' | 'strict';
// (undocumented)
sort?: boolean;
}
// @public
export interface CookieJarSetCookieOptions {
// (undocumented)
http?: boolean;
// (undocumented)
ignoreError?: boolean;
// (undocumented)
loose?: boolean;
// (undocumented)
now?: Date;
// (undocumented)
sameSiteContext?: 'strict' | 'lax' | 'none';
}
// @public
export interface CrawleeLogger {
child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
debug(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
deprecated(message: string): void;
error(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
exception(exception: Error, message: string, data?: Record<string, unknown>): void;
getOptions(): CrawleeLoggerOptions;
info(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
perf(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
setOptions(options: Partial<CrawleeLoggerOptions>): void;
softFail(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
warning(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
warningOnce(message: string): void;
}
// @public
export interface CrawleeLoggerOptions {
prefix?: string | null;
}
// @public (undocumented)
export interface DatasetBackend<Data extends Dictionary = Dictionary> {
drop(): Promise<void>;
getData(options?: DatasetBackendListOptions): Promise<PaginatedList<Data>>;
getMetadata(): Promise<DatasetInfo>;
purge(): Promise<void>;
pushData(items: Data[]): Promise<void>;
}
// @public (undocumented)
export interface DatasetBackendListOptions {
// (undocumented)
desc?: boolean;
// (undocumented)
limit?: number;
// (undocumented)
offset?: number;
}
// @public (undocumented)
export interface DatasetInfo {
// (undocumented)
accessedAt: Date;
// (undocumented)
createdAt: Date;
// (undocumented)
id: string;
// (undocumented)
itemCount: number;
// (undocumented)
modifiedAt: Date;
// (undocumented)
name?: string;
}
// @public (undocumented)
export type Dictionary<T = any> = Record<PropertyKey, T>;
// @public
export interface HttpRequest {
// (undocumented)
body?: Readable;
// (undocumented)
cookieJar?: CookieJar;
// (undocumented)
encoding?: BufferEncoding;
// (undocumented)
followRedirect?: boolean | ((response: any) => boolean);
// (undocumented)
headers?: Headers;
// (undocumented)
method?: AllowedHttpMethods;
// (undocumented)
proxyUrl?: string;
// (undocumented)
signal?: AbortSignal;
// (undocumented)
timeout?: number;
// (undocumented)
url: string | URL;
}
// @public
export interface HttpRequestOptions extends HttpRequest {
form?: Record<string, string>;
json?: unknown;
password?: string;
searchParams?: SearchParams;
username?: string;
}
// @public
export interface IBrowserPool<Page = unknown> {
closePage(page: Page, options?: {
error?: Error;
}): Promise<void>;
extractPageState(page: Page): Promise<PageState>;
injectPageState(page: Page, state: PageState): Promise<void>;
newPage(options?: NewPageOptions): Promise<Page>;
}
// @public
export interface ISession {
// (undocumented)
cookieJar: CookieJar;
// (undocumented)
fingerprint?: SessionFingerprint;
// (undocumented)
readonly id: string;
isBlocked(): boolean;
isUsable(): boolean;
markBad(): void;
markGood(): void;
// (undocumented)
proxyInfo?: ProxyInfo;
retire(): void;
}
// @public
export interface ISessionPool {
getSession(sessionId?: string): Promise<ISession | undefined>;
}
// @public
export interface KeyValueStoreBackend {
deleteValue(key: string): Promise<void>;
drop(): Promise<void>;
getMetadata(): Promise<KeyValueStoreInfo>;
getPublicUrl(key: string): Promise<string | undefined>;
getValue(key: string): Promise<KeyValueStoreRecord | undefined>;
listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreListKeysResult>;
purge(): Promise<void>;
recordExists(key: string): Promise<boolean>;
setValue(record: KeyValueStoreInputRecord): Promise<void>;
}
// @public (undocumented)
export interface KeyValueStoreInfo {
// (undocumented)
accessedAt: Date;
// (undocumented)
createdAt: Date;
// (undocumented)
id: string;
// (undocumented)
modifiedAt: Date;
// (undocumented)
name?: string;
}
// @public
export interface KeyValueStoreInputRecord {
// (undocumented)
contentType?: string;
// (undocumented)
key: string;
// (undocumented)
value: KeyValueStoreRecordInputValue;
}
// @public (undocumented)
export interface KeyValueStoreItemData {
contentType: string;
// (undocumented)
key: string;
// (undocumented)
size: number;
}
// @public (undocumented)
export interface KeyValueStoreListKeysOptions {
exclusiveStartKey?: string;
limit?: number;
prefix?: string;
}
// @public
export interface KeyValueStoreListKeysResult {
count: number;
exclusiveStartKey?: string;
isTruncated: boolean;
items: KeyValueStoreItemData[];
limit: number;
nextExclusiveStartKey?: string;
}
// @public
export interface KeyValueStoreRecord {
// (undocumented)
contentType?: string;
// (undocumented)
key: string;
// (undocumented)
value: Buffer | ArrayBuffer;
}
// @public
export type KeyValueStoreRecordInputValue = Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
// @public
export interface LogOptions {
once?: boolean;
}
// @public
export interface NewPageOptions {
id?: string;
session?: ISession;
}
// @public
export interface PageState {
cookies: Cookie[];
}
// @public
export interface PaginatedList<Data> {
count: number;
desc?: boolean;
items: Data[];
limit: number;
offset: number;
total: number;
}
// @public (undocumented)
export interface ProcessedRequest {
// (undocumented)
requestId: string;
// (undocumented)
uniqueKey: string;
// (undocumented)
wasAlreadyHandled: boolean;
// (undocumented)
wasAlreadyPresent: boolean;
}
// @public
export interface ProxyInfo {
hostname: string;
ignoreTlsErrors?: boolean;
password: string;
port: number | string;
url: string;
username?: string;
}
// @public
export interface QueueOperationInfo {
requestId: string;
wasAlreadyHandled: boolean;
wasAlreadyPresent: boolean;
}
// @public
export interface RequestQueueBackend {
addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
drop(): Promise<void>;
extendRequestProcessingTime?(requestId: string, extensionSecs: number): Promise<boolean>;
fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;
getMetadata(): Promise<RequestQueueInfo>;
getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
isEmpty(): Promise<boolean>;
isFinished(): Promise<boolean>;
markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | undefined>;
purge(): Promise<void>;
reclaimRequest(request: UpdateRequestSchema, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | undefined>;
setExpectedRequestProcessingTime?(processingTimeSecs: number): Promise<void>;
}
// @public (undocumented)
export interface RequestQueueInfo {
// (undocumented)
accessedAt: Date;
// (undocumented)
createdAt: Date;
// (undocumented)
handledRequestCount: number;
// (undocumented)
id: string;
// (undocumented)
modifiedAt: Date;
// (undocumented)
name?: string;
// (undocumented)
pendingRequestCount: number;
// (undocumented)
totalRequestCount: number;
}
// @public
export interface RequestQueueOperationOptions {
forefront?: boolean;
}
// @public (undocumented)
export interface RequestSchema {
// (undocumented)
errorMessages?: string[] | null;
// (undocumented)
handledAt?: string | null;
// (undocumented)
headers?: Dictionary<string> | null;
// (undocumented)
id?: string;
// (undocumented)
loadedUrl?: string | null;
// (undocumented)
method?: AllowedHttpMethods;
// (undocumented)
noRetry?: boolean | null;
// (undocumented)
payload?: string | null;
// (undocumented)
retryCount?: number;
// (undocumented)
uniqueKey: string;
// (undocumented)
url: string;
// (undocumented)
userData?: Dictionary;
}
// @public (undocumented)
export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
// @public (undocumented)
export interface SendRequestOptions {
// (undocumented)
cookieJar?: CookieJar;
ignoreTlsErrors?: boolean;
proxyUrl?: string;
// (undocumented)
session?: ISession;
signal?: AbortSignal;
timeoutMillis?: number;
}
// @public
export interface SerializedCookieJar {
// (undocumented)
[key: string]: unknown;
// (undocumented)
cookies: Record<string, unknown>[];
// (undocumented)
rejectPublicSuffixes: boolean;
// (undocumented)
storeType: string | null;
// (undocumented)
version: string;
}
// @public
export interface SessionCookie {
// (undocumented)
canonicalizedDomain(): string | undefined;
// (undocumented)
cdomain(): string | undefined;
// (undocumented)
clone(): SessionCookie | undefined;
// (undocumented)
cookieString(): string;
// (undocumented)
creation: Date | 'Infinity' | null;
// (undocumented)
creationIndex: number;
// (undocumented)
domain: string | null;
// (undocumented)
expires: Date | 'Infinity' | null;
// (undocumented)
expiryDate(now?: Date): Date | undefined;
// (undocumented)
expiryTime(now?: Date): number | undefined;
// (undocumented)
extensions: string[] | null;
// (undocumented)
hostOnly: boolean | null;
// (undocumented)
httpOnly: boolean;
// (undocumented)
isPersistent(): boolean;
// (undocumented)
key: string;
// (undocumented)
lastAccessed: Date | 'Infinity' | null;
// (undocumented)
maxAge: number | 'Infinity' | '-Infinity' | null;
// (undocumented)
path: string | null;
// (undocumented)
pathIsDefault: boolean | null;
// (undocumented)
sameSite: string | undefined;
// (undocumented)
secure: boolean;
// (undocumented)
setExpires(exp: string | Date): void;
// (undocumented)
setMaxAge(age: number): void;
// (undocumented)
toJSON(): Record<string, unknown>;
// (undocumented)
toString(): string;
// (undocumented)
TTL(now?: number): number;
// (undocumented)
validate(): boolean;
// (undocumented)
value: string;
}
// @public
export interface SessionFingerprint {
browser?: 'chrome' | 'firefox' | 'safari' | 'edge';
device?: 'desktop' | 'mobile';
platform?: 'windows' | 'macos' | 'linux' | 'android' | 'ios';
}
// @public
export interface SetStatusMessageOptions {
isStatusMessageTerminal?: boolean;
level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
}
// @public
export interface StorageBackend {
createDatasetBackend(options?: StorageIdentifier): Promise<DatasetBackend>;
createKeyValueStoreBackend(options?: StorageIdentifier): Promise<KeyValueStoreBackend>;
createRequestQueueBackend(options?: StorageIdentifier): Promise<RequestQueueBackend>;
getStorageBackendCacheKey?(): string;
purge?(): Promise<void>;
// (undocumented)
stats?: {
rateLimitErrors: number[];
};
storageExists?(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
// (undocumented)
teardown?(): Promise<void>;
}
// @public
export type StorageIdentifier = {
id: string;
name?: never;
alias?: never;
} | {
id?: never;
name: string;
alias?: never;
} | {
id?: never;
name?: never;
alias: string;
} | {
id?: never;
name?: never;
alias?: never;
};
// @public (undocumented)
export interface UnprocessedRequest {
// (undocumented)
method?: AllowedHttpMethods;
// (undocumented)
uniqueKey: string;
// (undocumented)
url: string;
}
// @public (undocumented)
export interface UpdateRequestSchema extends RequestSchema {
// (undocumented)
id: string;
}
// (No @packageDocumentation comment for this package)