Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

11.7 KB
Raw Blame

Public API Report File for "@crawlee/http"

Do not edit this file. It is a report generated by API Extractor.


import type { AddRequestsBatchedResult } from '@crawlee/basic';
import type { AnyNode } from 'domhandler';
import type { Awaitable } from '@crawlee/types';
import { BasicCrawler } from '@crawlee/basic';
import { BasicCrawlerOptions } from '@crawlee/basic';
import type { Cheerio } from 'cheerio';
import type { CheerioAPI } from 'cheerio';
import type { ConcurrencySystemOptions } from '@crawlee/basic';
import { ContextPipeline } from '@crawlee/basic';
import type { CrawlingContext } from '@crawlee/basic';
import type { CrawlingRequest } from '@crawlee/basic';
import type { Dictionary } from '@crawlee/types';
import type { EnqueueLinksOptions } from '@crawlee/basic';
import { ErrorHandler } from '@crawlee/basic';
import type { ExtractLinksOptions } from '@crawlee/basic';
import { GetUserDataFromRequest } from '@crawlee/basic';
import type { JsonValue } from 'type-fest';
import type { LoadedRequest } from '@crawlee/basic';
import { RequestHandler } from '@crawlee/basic';
import type { RequireContextPipeline } from '@crawlee/basic';
import { RouterHandler } from '@crawlee/basic';
import { RouterRoutes } from '@crawlee/basic';
import { RouteSchemas } from '@crawlee/basic';
import { RoutesFromSchemas } from '@crawlee/basic';

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface CrawlingContextWithResponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
    request: LoadedRequest<CrawlingRequest<UserData>>;
    response: Response;
}

// @public
export function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;

// @public (undocumented)
export function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;

// @public (undocumented)
export function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;

// @public
export function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;

// @public (undocumented)
export function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;

// @public (undocumented)
export function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;

// @public
export class DOMCrawler<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<Parsed, Selection, Single> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<DOMCrawlingContext<Parsed, Selection, Single>['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<DOMCrawlingContext<Parsed, Selection, Single>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
    constructor(options: DOMCrawlerOptions<Parsed, Selection, Single, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
    // (undocumented)
    protected buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed, Selection, Single>>;
}

// @public (undocumented)
export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<Parsed, Selection, Single> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, any>, StatisticStateExtension extends object = {}> extends Omit<HttpCrawlerOptions<DOMCrawlingContext<Parsed, Selection, Single>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'contextPipelineBuilder'> {
    parser: DOMParser_2<Parsed, Selection, Single>;
}

// @public (undocumented)
export type DOMCrawlingContext<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any> = Omit<InternalHttpCrawlingContext<UserData, JSONData>, keyof DOMCrawlingHelpers> & Parsed & DOMCrawlingHelpers<Selection, Single>;

// @public (undocumented)
export interface DOMCrawlingHelpers<Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown> {
    enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;
    extractLinks(options?: ExtractLinksOptions): Promise<string[]>;
    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
    querySelectorAll(selector: string, timeoutMs?: number): Promise<Selection>;
    querySelectorOne(selector: string, timeoutMs?: number): Promise<Single>;
    waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
}

// @public
interface DOMParser_2<Parsed extends DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown> {
    cleanup?(parsed: Parsed): Awaitable<void>;
    extractLinks(parsed: Parsed, selector: string, baseUrl: string): Awaitable<string[]>;
    readonly mutable?: boolean;
    // (undocumented)
    parse(context: InternalHttpCrawlingContext): Awaitable<Parsed>;
    readonly placeholderMembers: Record<keyof Parsed & string, true>;
    querySelectorAll(parsed: Parsed, selector: string): Awaitable<Selection>;
    querySelectorOne(parsed: Parsed, selector: string): Awaitable<Single | undefined>;
    toCheerio?(parsed: Parsed): Awaitable<CheerioAPI>;
}
export { DOMParser_2 as DOMParser }

// @public
export interface DOMParseResult {
    // (undocumented)
    body: string;
}

// @public
export class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
    constructor(options?: FileDownloadOptions);
}

// @public (undocumented)
export interface FileDownloadCrawlingContext<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
    // (undocumented)
    contentType: {
        type: string;
        encoding: BufferEncoding;
    };
    // (undocumented)
    request: LoadedRequest<CrawlingRequest<UserData>>;
    // (undocumented)
    response: Response;
}

// @public (undocumented)
export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, FileDownloadCrawlingContext<UserData> & ContextExtension>;

// @public (undocumented)
export interface FileDownloadOptions extends Omit<BasicCrawlerOptions<FileDownloadCrawlingContext>, 'contextPipelineBuilder'> {
    ignoreTlsErrors?: boolean;
}

// @public (undocumented)
export type FileDownloadRequestHandler<UserData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData>>;

// @public
export const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS: ConcurrencySystemOptions;

// @public
export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
    constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
    // (undocumented)
    protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;
}

// @public (undocumented)
export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
    additionalMimeTypes?: string[];
    forceResponseEncoding?: string;
    ignoreTlsErrors?: boolean;
    navigationTimeoutSecs?: number;
    postNavigationHooks?: InternalHttpHook<CrawlingContextWithResponse, ContextExtension>[];
    preNavigationHooks?: InternalHttpHook<CrawlingContext<any>, ContextExtension>[];
    saveResponseCookies?: boolean;
    suggestResponseEncoding?: string;
}

// @public (undocumented)
export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
    // (undocumented)
    querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
    // (undocumented)
    querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
}

// @public (undocumented)
export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, HttpCrawlingContext<UserData, JSONData> & ContextExtension>;

// @public (undocumented)
export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;

// @public (undocumented)
export interface InternalHttpCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData> {
    body: string | Buffer;
    contentType: {
        type: string;
        encoding: BufferEncoding;
    };
    json: JSONData;
    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
    querySelectorAll(selector: string, timeoutMs?: number): Promise<ArrayLike<unknown>>;
    querySelectorOne(selector: string, timeoutMs?: number): Promise<unknown>;
    waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
}

// @public (undocumented)
export type InternalHttpHook<Context, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;


export * from "@crawlee/basic";

// (No @packageDocumentation comment for this package)