Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { type CrawleeLogger, serviceLocator } from '@crawlee/core';
import type { Cookie, Dictionary } from '@crawlee/types';
import { nanoid } from 'nanoid';
import { TypedEmitter } from 'tiny-typed-emitter';
import { tryCancel } from '@apify/timeout';
import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
import type { LaunchContext } from '../launch-context.js';
import type { UnwrapPromise } from '../utils.js';
import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './browser-plugin.js';
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
/**
* The subset of the browser-pool `LaunchContext` that {@apilink IBrowserController} exposes.
* Other fields are only available on the concrete `LaunchContext` class.
*/
export interface IBrowserLaunchContext {
/**
* The proxy URL the browser was launched with, if any.
*/
proxyUrl?: string;
/**
* The fingerprint applied to the browser, if fingerprinting is enabled.
* Typed as `unknown` here; cast to the concrete `LaunchContext` if you
* need the structured shape.
*/
fingerprint?: unknown;
/**
* `true` if each page in this browser uses its own context.
*/
useIncognitoPages?: boolean;
/**
* The actual options the browser was launched with, after pre-launch hooks.
*/
launchOptions?: Dictionary | undefined;
}
/**
* The minimal public contract of a browser controller.
*
* Coordination with the pool (page-counting, `activate`, `assignBrowser`, lifecycle
* promises, …) is intentionally **not** part of this contract.
*
* @category Browser management
*/
export interface IBrowserController<Page = unknown> {
/**
* A stable identifier for this controller instance. Useful for tracking
* which browser served which request.
*/
readonly id: string;
/**
* The configuration the underlying browser was launched with — proxy URL,
* fingerprint, session, launcher-specific options, etc.
*/
readonly launchContext: IBrowserLaunchContext;
/**
* The raw browser handle from the underlying automation library
* (Puppeteer `Browser`, Playwright `Browser`/`BrowserContext`, …).
* Escape hatch for things the controller does not expose directly.
*/
readonly browser: unknown;
/**
* Reads cookies for the given page.
*/
getCookies(page: Page): Promise<Cookie[]>;
/**
* Writes cookies for the given page.
*/
setCookies(page: Page, cookies: Cookie[]): Promise<void>;
/**
* Gracefully closes the browser this controller owns. After this resolves,
* the controller is no longer usable.
*/
close(): Promise<void>;
}
export interface BrowserControllerEvents<
Library extends CommonLibrary,
LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],
LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,
NewPageOptions = Parameters<LaunchResult['newPage']>[0],
NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
> {
[BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED]: (
controller: BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>,
) => void;
}
/**
* The `BrowserController` serves two purposes. First, it is the base class that
* specialized controllers like `PuppeteerController` or `PlaywrightController`
* extend. Second, it defines the public interface of the specialized classes
* which provide only private methods. Therefore, we do not keep documentation
* for the specialized classes, because it's the same for all of them.
* @hideconstructor
*/
export abstract class BrowserController<
Library extends CommonLibrary = CommonLibrary,
LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],
LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,
NewPageOptions = Parameters<LaunchResult['newPage']>[0],
NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
>
extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>>
implements IBrowserController<NewPageResult>
{
readonly id = nanoid();
/**
* Kept `protected` rather than `#private` because the concrete controllers in this package are
* separate classes — an ES private field would not be visible to them.
*
* @internal
*/
protected readonly log!: CrawleeLogger;
/**
* The `BrowserPlugin` instance used to launch the browser.
*/
readonly browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
/**
* Browser representation of the underlying automation library.
*/
browser: LaunchResult = undefined!;
/**
* The configuration the browser was launched with.
*/
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult> = undefined!;
/**
* The proxy URL used by the browser controller.
* `undefined` if no proxy is used
*/
proxyUrl?: string;
/** @internal */
isActive = false;
activePages = 0;
readonly #closedPages = new WeakSet<object>();
readonly #pageTeardowns = new WeakMap<object, () => Promise<void>>();
/** @internal */
totalPages = 0;
/** @internal */
lastPageOpenedAt = Date.now();
#activate!: () => void;
#isActivePromise = new Promise<void>((resolve) => {
this.#activate = resolve;
});
/**
* Returns a promise that resolves once the browser controller has been activated.
*/
async waitForActive(): Promise<void> {
await this.#isActivePromise;
}
#commitBrowser!: () => void;
#hasBrowserPromise = new Promise<void>((resolve) => {
this.#commitBrowser = resolve;
});
constructor(browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>) {
super();
this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
this.browserPlugin = browserPlugin;
}
/**
* Activates the BrowserController. If you try to open new pages before
* activation, the pages will get queued and will only be opened after
* activate is called.
* @ignore
*/
activate(): void {
if (!this.browser) {
throw new Error('Cannot activate BrowserController without an assigned browser.');
}
this.#activate();
this.isActive = true;
}
/**
* @ignore
*/
assignBrowser(
browser: LaunchResult,
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>,
): void {
if (this.browser) {
throw new Error('BrowserController already has a browser instance assigned.');
}
this.browser = browser;
this.launchContext = launchContext;
this.#commitBrowser();
}
/**
* Gracefully closes the browser and makes sure
* there will be no lingering browser processes.
*
* Emits 'browserClosed' event.
*/
async close(): Promise<void> {
await this.#hasBrowserPromise;
try {
await this._close();
// TODO: shouldn't this go in a finally instead?
this.isActive = false;
} catch (error) {
this.log.debug(`Could not close browser.\nCause: ${(error as Error).message}`, { id: this.id });
}
this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
const killTimer = setTimeout(() => {
this._kill().catch((err) => {
this.log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
});
}, PROCESS_KILL_TIMEOUT_MILLIS);
killTimer!.unref();
}
/**
* Immediately kills the browser process.
*
* Emits 'browserClosed' event.
*/
async kill(): Promise<void> {
await this.#hasBrowserPromise;
await this._kill();
this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
}
/**
* Opens new browser page.
* @ignore
*/
async newPage(pageOptions?: NewPageOptions): Promise<NewPageResult> {
this.activePages++;
this.totalPages++;
await this.#isActivePromise;
const page = await this._newPage(pageOptions);
tryCancel();
this.lastPageOpenedAt = Date.now();
return page;
}
/**
* Accounts for a page no longer being open. Callable from either side and safe to call twice:
* the page's own `close` event, or the pool when it gives up on a close that never settles.
* Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
* @internal
*/
registerPageClosed(page: NewPageResult): void {
if (this.#closedPages.has(page as object)) return;
this.#closedPages.add(page as object);
this.activePages--;
const teardown = this.#pageTeardowns.get(page as object);
if (!teardown) return;
this.#pageTeardowns.delete(page as object);
teardown().catch((error: Error) => {
this.log.debug(`Could not clean up after a closed page.\nCause:${error.message}`, { id: this.id });
});
}
/**
* Registers cleanup belonging to a single page: an anonymizing proxy server, an incognito
* context. It runs from `registerPageClosed`, so it happens whether the page closed on its own
* or the pool gave up on a close that never settled, and it runs at most once.
*
* Hanging it off the page's `close` event instead leaks it in exactly the case this controller
* now handles, because that event does not arrive for a page the browser never destroyed.
* @ignore
*/
protected registerPageTeardown(page: NewPageResult, teardown: () => Promise<void>): void {
this.#pageTeardowns.set(page as object, teardown);
}
async setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void> {
return this._setCookies(page, cookies);
}
async getCookies(page: NewPageResult): Promise<Cookie[]> {
return this._getCookies(page);
}
protected abstract _close(): Promise<void>;
protected abstract _kill(): Promise<void>;
protected abstract _newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
protected abstract _setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
protected abstract _getCookies(page: NewPageResult): Promise<Cookie[]>;
/**
* @internal
*/
abstract normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
}