Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { serviceLocator } from '@crawlee/core';
import type { CustomFetchOptions } from '@crawlee/http-client';
import { BaseHttpClient, ResponseWithUrl } from '@crawlee/http-client';
import type { CrawleeLogger, SessionFingerprint } from '@crawlee/types';
import { Impit, type Browser as ImpitBrowser, type ImpitOptions } from 'impit';
import type { CookieJar as ToughCookieJar } from 'tough-cookie';
import { LruCache } from '@apify/datastructures';
// Concrete impit impersonation profiles per browser family. The plain `chrome` /
// `firefox` aliases fall back to the oldest available version, which is a
// fingerprint giveaway — we pick one of these explicitly instead. Keep in sync
// with impit's `Browser` type when bumping the dependency.
const IMPIT_VERSIONS_BY_BROWSER: Partial<Record<NonNullable<SessionFingerprint['browser']>, ImpitBrowser[]>> = {
chrome: [
'chrome100',
'chrome101',
'chrome104',
'chrome107',
'chrome110',
'chrome116',
'chrome124',
'chrome125',
'chrome131',
'chrome136',
'chrome142',
],
firefox: ['firefox128', 'firefox133', 'firefox135', 'firefox144'],
};
export const Browser = {
'Chrome': 'chrome',
'Firefox': 'firefox',
} as const;
/**
* A HTTP client implementation based on the `impit` library.
*/
export class ImpitHttpClient extends BaseHttpClient {
#impitOptions: ImpitOptions;
#cacheClients: boolean;
/**
* Enables reuse of `impit` clients for the same set of options.
* This is useful for performance reasons, as creating
* a new client for each request breaks TCP connection
* (and other resources) reuse.
*/
#clientCache: LruCache<{ client: Impit; cookieJar: ToughCookieJar }> = new LruCache({ maxLength: 10 });
/**
* Stable impit impersonation version per fingerprint object, so the same
* session keeps impersonating the same browser version across requests
* instead of rerolling on every call.
*/
#impitBrowserByFingerprint = new WeakMap<SessionFingerprint, ImpitBrowser>();
#getClient(options: ImpitOptions): Impit {
if (!this.#cacheClients) {
return new Impit(options);
}
const { cookieJar, ...rest } = options;
const cacheKey = JSON.stringify(rest);
const existingClient = this.#clientCache.get(cacheKey);
if (existingClient && (!cookieJar || existingClient.cookieJar === cookieJar)) {
return existingClient.client;
}
const client = new Impit(options);
this.#clientCache.add(cacheKey, { client, cookieJar: cookieJar as ToughCookieJar });
return client;
}
/**
* @param options.cacheClients Whether to cache `impit` clients between requests. Defaults to `true`.
*/
constructor(
options?: Omit<ImpitOptions, 'proxyUrl' | 'timeout'> & { cacheClients?: boolean; logger?: CrawleeLogger },
) {
super({ logger: options?.logger });
const { cacheClients = true, logger, ...impitOptions } = options ?? {};
if (impitOptions.browser) {
(logger ?? serviceLocator.getLogger()).warningOnce(
'The `browser` option of ImpitHttpClient overrides the browser fingerprint of the session. ' +
'Using it with Crawlee crawlers can lead to incompatible Session setups.',
);
}
this.#impitOptions = impitOptions;
this.#cacheClients = cacheClients;
}
/**
* @inheritDoc
*/
protected override async fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response> {
const { proxyUrl, redirect, signal, fingerprint, ignoreTlsErrors } = options ?? {};
const impitBrowser = this.#impitOptions.browser ? undefined : this.#resolveImpitBrowser(fingerprint);
const impit = this.#getClient({
...this.#impitOptions,
...(impitBrowser ? { browser: impitBrowser } : {}),
// The per-request flag (from the crawler option or a MITM proxy session)
// can only enable ignoring, never override a constructor-level `true`.
...(ignoreTlsErrors ? { ignoreTlsErrors: true } : {}),
proxyUrl,
followRedirects: redirect === 'follow',
});
// The caller's signal carries the deadline (for crawlers, the navigation window that `extendTimeout()` can
// push out), so impit's 30 s default must not cut the request short. Without a signal, the default stays.
const response = await impit.fetch(request, {
signal: signal ?? undefined,
...(signal ? { timeout: null } : {}),
});
// Once the headers are in, an aborted signal only stops impit's body reads, while the download itself keeps
// going. Piping the body through the signal cancels the download too.
const body =
signal && response.body ? response.body.pipeThrough(new TransformStream(), { signal }) : response.body;
// todo - cast shouldn't be needed here, impit returns `Uint8Array`
return new ResponseWithUrl(body, response);
}
#resolveImpitBrowser(fingerprint?: SessionFingerprint): ImpitBrowser | undefined {
if (!fingerprint?.browser) return undefined;
const cached = this.#impitBrowserByFingerprint.get(fingerprint);
if (cached) return cached;
// impit can only impersonate Chrome and Firefox. Map other (Chromium-based or
// unsupported) families like `edge`/`safari` onto Chrome so the request still
// carries realistic browser headers instead of impit's bare `*/*` defaults.
const versions = IMPIT_VERSIONS_BY_BROWSER[fingerprint.browser] ?? IMPIT_VERSIONS_BY_BROWSER.chrome!;
const picked = versions[Math.floor(Math.random() * versions.length)];
this.#impitBrowserByFingerprint.set(fingerprint, picked);
return picked;
}
}