Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import type { Readable } from 'node:stream';
import type { CookieJar } from './cookies.js';
import type { ISession } from './session.js';
import type { AllowedHttpMethods } from './utility-types.js';
export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
/**
* HTTP Request as accepted by {@apilink BaseHttpClient} methods.
*/
export interface HttpRequest {
url: string | URL;
method?: AllowedHttpMethods;
headers?: Headers;
body?: Readable;
signal?: AbortSignal;
timeout?: number;
cookieJar?: CookieJar;
followRedirect?: boolean | ((response: any) => boolean); // TODO BC with got - specify type better in 4.0
encoding?: BufferEncoding;
proxyUrl?: string;
}
/**
* Additional options for HTTP requests that need to be handled separately before passing to {@apilink BaseHttpClient}.
*/
export interface HttpRequestOptions extends HttpRequest {
/** Search (query string) parameters to be appended to the request URL */
searchParams?: SearchParams;
/** A form to be sent in the HTTP request body (URL encoding will be used) */
form?: Record<string, string>;
/** Arbitrary object to be JSON-serialized and sent as the HTTP request body */
json?: unknown;
/** Basic HTTP Auth username */
username?: string;
/** Basic HTTP Auth password */
password?: string;
}
export interface SendRequestOptions {
session?: ISession;
cookieJar?: CookieJar;
/** Timeout for the HTTP request in milliseconds. */
timeoutMillis?: number;
/** An AbortSignal to cancel the HTTP request. */
signal?: AbortSignal;
/**
* Overrides the proxy URL set in the `session` for this request.
*
* Note that setting this manually can interfere with session proxy rotation.
*/
proxyUrl?: string;
/**
* When `true`, TLS certificate errors are ignored for this request. Also enabled by
* `session.proxyInfo.ignoreTlsErrors` (MITM proxies). Best-effort: clients that cannot
* disable TLS verification (e.g. the native fetch fallback) ignore it.
*/
ignoreTlsErrors?: boolean;
}
/**
* Interface for user-defined HTTP clients to be used for plain HTTP crawling and for sending additional requests during a crawl.
*/
export interface BaseHttpClient {
/**
* Perform an HTTP Request and return the complete response.
*/
sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
}