Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

8.5 KB
Raw Blame
import type {
    BaseHttpClient as BaseHttpClientInterface,
    CookieJar,
    CrawleeLogger,
    SendRequestOptions,
    SessionFingerprint,
} from '@crawlee/types';

/**
 * Per-request options handed to a concrete client's `fetch` implementation.
 */
export interface CustomFetchOptions {
    /**
     * Effective proxy URL for this request. `sendRequest` populates this from
     * the explicit `SendRequestOptions.proxyUrl` override when set, falling back
     * to `session.proxyInfo.url`.
     */
    proxyUrl?: string;

    /**
     * Effective cookie jar for this request. `sendRequest` populates this from
     * the explicit `SendRequestOptions.cookieJar` override when set, falling
     * back to `session.cookieJar` (or a fresh jar when neither is provided).
     */
    cookieJar?: CookieJar;

    /**
     * Hints about which browser-like setup this request should be impersonating —
     * `browser`, `platform`, `device`, and an opaque `details` slot for richer
     * payloads (e.g. a full browser fingerprint from `fingerprint-generator`).
     * These are *suggestions*, not requirements: each client applies what it can
     * (e.g. impit maps `browser` to its impersonation profile) and ignores the
     * rest on a best-effort basis. Sourced from `SendRequestOptions.session.fingerprint`.
     */
    fingerprint?: SessionFingerprint;

    /**
     * When `true`, TLS certificate errors should be ignored for this request.
     * Set when `SendRequestOptions.ignoreTlsErrors` is passed (e.g. from the
     * `ignoreTlsErrors` crawler option) or when the session's proxy is a MITM
     * proxy (`session.proxyInfo.ignoreTlsErrors`). Best-effort: clients that
     * cannot disable TLS verification ignore it.
     */
    ignoreTlsErrors?: boolean;
}

/**
 * Base HTTP client that provides fetch-like `sendRequest` with Crawlee-managed
 * behaviors (redirect handling, proxy and cookie handling). Concrete clients
 * implement only the low-level network call in `fetch`.
 */
export abstract class BaseHttpClient implements BaseHttpClientInterface {
    #log?: CrawleeLogger;

    constructor(options?: { logger?: CrawleeLogger }) {
        this.#log = options?.logger;
    }

    /**
     * Perform the raw network request and return a single Response without any
     * automatic redirect following or special error handling.
     */
    protected abstract fetch(input: Request, init?: RequestInit & CustomFetchOptions): Promise<Response>;

    async #applyCookies(request: Request, cookieJar: CookieJar): Promise<Request> {
        try {
            const requestCookies = request.headers.get('cookie') ?? '';

            if (!requestCookies) {
                // Fast path: no header cookies, use the jar directly.
                const cookieString = await cookieJar.getCookieString(request.url);
                if (cookieString) {
                    request.headers.set('cookie', cookieString);
                }
                return request;
            }

            // Merge jar cookies with request Cookie header. Clone the jar so we
            // don't persist the header-only cookies into the session.
            const merged = await cookieJar.clone();

            await Promise.all(
                requestCookies
                    .split(/; */)
                    .filter(Boolean)
                    .map((pair) => merged.setCookie(pair, request.url)),
            );
            const cookieString = await merged.getCookieString(request.url);

            if (cookieString) {
                request.headers.set('cookie', cookieString);
            }
        } catch (e) {
            this.#log?.warning(`Failed to get cookies for URL "${request.url}": ${(e as Error).message}`);
        }

        return request;
    }

    async #setCookies(response: Response, cookieJar: CookieJar): Promise<void> {
        const setCookieHeaders = response.headers.getSetCookie();

        for (const header of setCookieHeaders) {
            try {
                await cookieJar.setCookie(header, response.url);
            } catch (e) {
                this.#log?.warning(`Failed to set cookie for URL "${response.url}": ${(e as Error).message}`);
            }
        }
    }

    async #resolveRequestContext(options?: SendRequestOptions): Promise<{
        proxyUrl?: string;
        cookieJar: CookieJar;
        signal?: AbortSignal;
        fingerprint?: SessionFingerprint;
        ignoreTlsErrors?: boolean;
    }> {
        const proxyUrl = options?.proxyUrl ?? options?.session?.proxyInfo?.url;
        const cookieJar = options?.cookieJar ?? options?.session?.cookieJar ?? (await this.#createDefaultCookieJar());
        const signal = this.#createAbortSignal(options?.signal, options?.timeoutMillis);
        return {
            proxyUrl,
            cookieJar,
            signal,
            fingerprint: options?.session?.fingerprint,
            ignoreTlsErrors: options?.ignoreTlsErrors || options?.session?.proxyInfo?.ignoreTlsErrors,
        };
    }

    async #createDefaultCookieJar(): Promise<CookieJar> {
        const { CookieJar: ToughCookieJar } = await import('tough-cookie');
        return new ToughCookieJar();
    }

    #createAbortSignal(signal?: AbortSignal, timeoutMillis?: number): AbortSignal | undefined {
        if (signal && timeoutMillis) {
            return AbortSignal.any([signal, AbortSignal.timeout(timeoutMillis)]);
        }
        if (signal) {
            return signal;
        }
        return timeoutMillis ? AbortSignal.timeout(timeoutMillis) : undefined;
    }

    #isRedirect(response: Response): boolean {
        const status = response.status;
        return status >= 300 && status < 400 && !!response.headers.get('location');
    }

    #buildRedirectRequest(currentRequest: Request, response: Response, initialRequest: Request): Request {
        const location = response.headers.get('location')!;
        const nextUrl = new URL(location, response.url ?? currentRequest.url);

        const prevMethod = (currentRequest.method ?? 'GET').toUpperCase();
        let nextMethod = prevMethod;
        let nextBody: BodyInit | null = null;

        if (
            response.status === 303 ||
            ((response.status === 301 || response.status === 302) && prevMethod === 'POST')
        ) {
            nextMethod = 'GET';
            nextBody = null;
        } else {
            const clonedRequest = initialRequest.clone();
            nextBody = clonedRequest.body;
        }

        const nextHeaders = new Headers();
        currentRequest.headers.forEach((value, key) => nextHeaders.set(key, value));

        // Like native `fetch`, do not forward credentials to a different origin
        if (nextUrl.origin !== new URL(currentRequest.url).origin) {
            for (const header of ['authorization', 'proxy-authorization', 'cookie']) {
                nextHeaders.delete(header);
            }
        }

        return new Request(nextUrl.toString(), {
            method: nextMethod,
            headers: nextHeaders,
            body: nextBody,
            credentials: (currentRequest as any).credentials,
            redirect: 'manual',
            // Node-specific option required for a stream body, which the replayed body always is
            duplex: 'half',
        } as RequestInit);
    }

    /**
     * Public fetch-like method that handles redirects and uses provided proxy and cookie jar.
     */
    async sendRequest(initialRequest: Request, options?: SendRequestOptions): Promise<Response> {
        const maxRedirects = 10;
        let currentRequest = initialRequest;
        let redirectCount = 0;

        const { proxyUrl, cookieJar, signal, fingerprint, ignoreTlsErrors } =
            await this.#resolveRequestContext(options);
        currentRequest = initialRequest.clone();

        while (true) {
            // Like `fetch`, set the jar cookies on a copy, so that the next redirect hop does not inherit them
            const response = await this.fetch(await this.#applyCookies(new Request(currentRequest), cookieJar), {
                signal,
                proxyUrl,
                cookieJar,
                fingerprint,
                ignoreTlsErrors,
                redirect: 'manual',
            });

            await this.#setCookies(response, cookieJar);

            if (this.#isRedirect(response)) {
                if (redirectCount++ >= maxRedirects) {
                    throw new Error(`Too many redirects (${maxRedirects}) while requesting ${currentRequest.url}`);
                }
                currentRequest = this.#buildRedirectRequest(currentRequest, response, initialRequest);
                continue;
            }

            return response;
        }
    }
}