Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { type CrawleeLogger, SessionError, serviceLocator } from '@crawlee/core';
import type { IBrowserPool, NewPageOptions, PageState } from '@crawlee/types';
import { parseArgument, schemas } from '@crawlee/utils/internal';
import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';
import { FingerprintGenerator } from 'fingerprint-generator';
import { FingerprintInjector } from 'fingerprint-injector';
import { nanoid } from 'nanoid';
import pLimit from 'p-limit';
import QuickLRU from 'quick-lru';
import { TypedEmitter } from 'tiny-typed-emitter';
import { z } from 'zod';
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
import type { BrowserController } from './abstract-classes/browser-controller.js';
import type { BrowserPlugin } from './abstract-classes/browser-plugin.js';
import { BROWSER_POOL_EVENTS } from './events.js';
import {
createFingerprintPreLaunchHook,
createPostPageCreateHook,
createPrePageCreateHook,
} from './fingerprinting/hooks.js';
import type { FingerprintGeneratorOptions } from './fingerprinting/types.js';
import type { LaunchContext } from './launch-context.js';
import type { InferBrowserPluginArray, UnwrapPromise } from './utils.js';
const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
const SLOT_POLL_INTERVAL_MILLIS = 500;
const browserPoolOptionsSchema = z.strictObject({
browserPlugins: schemas.anyArray.refine((value) => value.length >= 1, 'Expected a non-empty array'),
maxOpenPagesPerBrowser: schemas.anyNumber.default(20),
maxOpenBrowsers: schemas.anyNumber.default(Infinity),
retireBrowserAfterPageCount: schemas.anyNumber.default(100),
operationTimeoutSecs: schemas.anyNumber.default(15),
closeInactiveBrowserAfterSecs: schemas.anyNumber.default(300),
retireInactiveBrowserAfterSecs: schemas.anyNumber.default(10),
preLaunchHooks: schemas.anyArray.default(() => []),
postLaunchHooks: schemas.anyArray.default(() => []),
prePageCreateHooks: schemas.anyArray.default(() => []),
postPageCreateHooks: schemas.anyArray.default(() => []),
prePageCloseHooks: schemas.anyArray.default(() => []),
postPageCloseHooks: schemas.anyArray.default(() => []),
useFingerprints: z.boolean().default(true),
fingerprintOptions: schemas.anyObject.default(() => ({})),
});
export interface BrowserPoolEvents<BC extends BrowserController, Page> {
[BROWSER_POOL_EVENTS.PAGE_CREATED]: (page: Page) => void | Promise<void>;
[BROWSER_POOL_EVENTS.PAGE_CLOSED]: (page: Page) => void | Promise<void>;
[BROWSER_POOL_EVENTS.BROWSER_RETIRED]: (browserController: BC) => void | Promise<void>;
[BROWSER_POOL_EVENTS.BROWSER_LAUNCHED]: (browserController: BC) => void | Promise<void>;
}
/**
* Settings for the fingerprint generator and virtual session management system.
*
* > To set the specific fingerprint generation options (operating system, device type, screen dimensions), use the `fingerprintGeneratorOptions` property.
*/
export interface FingerprintOptions {
/**
* Customizes the fingerprint generation by setting e.g. the device type, operating system or screen size.
*/
fingerprintGeneratorOptions?: FingerprintGeneratorOptions;
/**
* Enables the virtual session management system. This ties every Crawlee session with a specific browser fingerprint,
* so your scraping activity seems more natural to the target website.
* @default true
*/
useFingerprintCache?: boolean;
/**
* The maximum number of fingerprints that can be stored in the cache.
*
* Only relevant if `useFingerprintCache` is set to `true`.
* @default 10000
*/
fingerprintCacheSize?: number;
}
export interface BrowserPoolOptions<Plugin extends BrowserPlugin = BrowserPlugin> {
/**
* Browser plugins are wrappers of browser automation libraries that
* allow `BrowserPool` to control browsers with those libraries.
* `browser-pool` comes with a `PuppeteerPlugin` and a `PlaywrightPlugin`.
*/
browserPlugins: readonly Plugin[];
/**
* Sets the maximum number of pages that can be open in a browser at the
* same time. Once reached, a new browser will be launched to handle the excess.
*
* @default 20
*/
maxOpenPagesPerBrowser?: number;
/**
* Maximum number of browser processes alive at once. A browser counts from the moment its launch starts until it
* actually closes — a retired browser that is still finishing its last pages still occupies a slot. Once the
* limit is reached, {@apilink BrowserPool.newPage|newPage} waits for a slot (a browser closing, or an open
* browser with page capacity to spare) instead of launching another. Set it to a remote browser service's
* concurrent-session limit to avoid `429` errors.
*
* @default Infinity
*/
maxOpenBrowsers?: number;
/**
* Browsers tend to get bloated after processing a lot of pages. This option
* configures the maximum number of processed pages after which the browser will
* automatically retire and close. A new browser will launch in its place.
* The browser might be retired sooner if the connected {@apilink Session} is retired.
* You can change session retirement behavior using {@apilink SessionPoolOptions}.
*
* @default 100
*/
retireBrowserAfterPageCount?: number;
/**
* As we know from experience, async operations of the underlying libraries,
* such as launching a browser or opening a new page, can get stuck.
* To prevent `BrowserPool` from getting stuck, we add a timeout
* to those operations and you can configure it with this option.
*
* @default 15
*/
operationTimeoutSecs?: number;
/**
* Browsers normally close immediately after their last page is processed.
* However, there could be situations where this does not happen. Browser Pool
* makes sure all inactive browsers are closed regularly, to free resources.
*
* @default 300
*/
closeInactiveBrowserAfterSecs?: number;
/**
* Browsers are marked as retired after they have been inactive for a certain
* amount of time. This option sets the interval at which the browsers
* are checked and retired if they are inactive.
*
* Retired browsers are closed after all their pages are closed.
*
* @default 10
*/
retireInactiveBrowserAfterSecs?: number;
/**
* @default true
*/
useFingerprints?: boolean;
fingerprintOptions?: FingerprintOptions;
}
/**
* Pre-launch hooks are executed just before a browser is launched and provide
* a good opportunity to dynamically change the launch options.
* The hooks are called with two arguments:
* `pageId`: `string` and `launchContext`: {@apilink LaunchContext}
*/
export type PreLaunchHook<LC extends LaunchContext> = (pageId: string, launchContext: LC) => void | Promise<void>;
/**
* Post-launch hooks are executed as soon as a browser is launched.
* The hooks are called with two arguments:
* `pageId`: `string` and `browserController`: {@apilink BrowserController}
* To guarantee order of execution before other hooks in the same browser,
* the {@apilink BrowserController} methods cannot be used until the post-launch
* hooks complete. If you attempt to call `await browserController.close()` from
* a post-launch hook, it will deadlock the process. This API is subject to change.
*/
export type PostLaunchHook<BC extends BrowserController> = (
pageId: string,
browserController: BC,
) => void | Promise<void>;
/**
* Pre-page-create hooks are executed just before a new page is created. They
* are useful to make dynamic changes to the browser before opening a page.
* The hooks are called with three arguments:
* `pageId`: `string`, `browserController`: {@apilink BrowserController} and
* `pageOptions`: `object|undefined` - This only works if the underlying `BrowserController` supports new page options.
* So far, new page options are only supported by `PlaywrightController` in incognito contexts.
* If the page options are not supported by `BrowserController` the `pageOptions` argument is `undefined`.
*/
export type PrePageCreateHook<BC extends BrowserController, PO = Parameters<BC['newPage']>[0]> = (
pageId: string,
browserController: BC,
pageOptions?: PO,
) => void | Promise<void>;
/**
* Post-page-create hooks are called right after a new page is created
* and all internal actions of Browser Pool are completed. This is the
* place to make changes to a page that you would like to apply to all
* pages. Such as injecting a JavaScript library into all pages.
* The hooks are called with two arguments:
* `page`: `Page` and `browserController`: {@apilink BrowserController}
*/
export type PostPageCreateHook<BC extends BrowserController, Page = UnwrapPromise<ReturnType<BC['newPage']>>> = (
page: Page,
browserController: BC,
) => void | Promise<void>;
/**
* Pre-page-close hooks give you the opportunity to make last second changes
* in a page that's about to be closed, such as saving a snapshot or updating
* state.
* The hooks are called with two arguments:
* `page`: `Page` and `browserController`: {@apilink BrowserController}
*/
export type PrePageCloseHook<BC extends BrowserController, Page = UnwrapPromise<ReturnType<BC['newPage']>>> = (
page: Page,
browserController: BC,
) => void | Promise<void>;
/**
* Post-page-close hooks allow you to do page related clean up.
* The hooks are called with two arguments:
* `pageId`: `string` and `browserController`: {@apilink BrowserController}
*/
export type PostPageCloseHook<BC extends BrowserController> = (
pageId: string,
browserController: BC,
) => void | Promise<void>;
export interface BrowserPoolHooks<
BC extends BrowserController,
LC extends LaunchContext,
PR extends UnwrapPromise<ReturnType<BC['newPage']>> = UnwrapPromise<ReturnType<BC['newPage']>>,
> {
/**
* Pre-launch hooks are executed just before a browser is launched and provide
* a good opportunity to dynamically change the launch options.
* The hooks are called with two arguments:
* `pageId`: `string` and `launchContext`: {@apilink LaunchContext}
*/
preLaunchHooks?: PreLaunchHook<LC>[];
/**
* Post-launch hooks are executed as soon as a browser is launched.
* The hooks are called with two arguments:
* `pageId`: `string` and `browserController`: {@apilink BrowserController}
* To guarantee order of execution before other hooks in the same browser,
* the {@apilink BrowserController} methods cannot be used until the post-launch
* hooks complete. If you attempt to call `await browserController.close()` from
* a post-launch hook, it will deadlock the process. This API is subject to change.
*/
postLaunchHooks?: PostLaunchHook<BC>[];
/**
* Pre-page-create hooks are executed just before a new page is created. They
* are useful to make dynamic changes to the browser before opening a page.
* The hooks are called with three arguments:
* `pageId`: `string`, `browserController`: {@apilink BrowserController} and
* `pageOptions`: `object|undefined` - This only works if the underlying `BrowserController` supports new page options.
* So far, new page options are only supported by `PlaywrightController` in incognito contexts.
* If the page options are not supported by `BrowserController` the `pageOptions` argument is `undefined`.
*/
prePageCreateHooks?: PrePageCreateHook<BC>[];
/**
* Post-page-create hooks are called right after a new page is created
* and all internal actions of Browser Pool are completed. This is the
* place to make changes to a page that you would like to apply to all
* pages. Such as injecting a JavaScript library into all pages.
* The hooks are called with two arguments:
* `page`: `Page` and `browserController`: {@apilink BrowserController}
*/
postPageCreateHooks?: PostPageCreateHook<BC, PR>[];
/**
* Pre-page-close hooks give you the opportunity to make last second changes
* in a page that's about to be closed, such as saving a snapshot or updating
* state.
* The hooks are called with two arguments:
* `page`: `Page` and `browserController`: {@apilink BrowserController}
*/
prePageCloseHooks?: PrePageCloseHook<BC, PR>[];
/**
* Post-page-close hooks allow you to do page related clean up.
* The hooks are called with two arguments:
* `pageId`: `string` and `browserController`: {@apilink BrowserController}
*/
postPageCloseHooks?: PostPageCloseHook<BC>[];
}
/**
* The `BrowserPool` class is the most important class of the `browser-pool` module.
* It manages opening and closing of browsers and their pages and its constructor
* options allow easy configuration of the browsers' and pages' lifecycle.
*
* The most important and useful constructor options are the various lifecycle hooks.
* Those allow you to sequentially call a list of (asynchronous) functions at each
* stage of the browser / page lifecycle.
*
* **Example:**
* ```js
* import { BrowserPool, PlaywrightPlugin } from '@crawlee/browser-pool';
* import playwright from 'playwright';
*
* const browserPool = new BrowserPool({
* browserPlugins: [new PlaywrightPlugin(playwright.chromium)],
* preLaunchHooks: [(pageId, launchContext) => {
* // do something before a browser gets launched
* launchContext.launchOptions.headless = false;
* }],
* postLaunchHooks: [(pageId, browserController) => {
* // manipulate the browser right after launch
* console.dir(browserController.browser.contexts());
* }],
* prePageCreateHooks: [(pageId, browserController) => {
* if (pageId === 'my-page') {
* // make changes right before a specific page is created
* }
* }],
* postPageCreateHooks: [async (page, browserController) => {
* // update some or all new pages
* await page.evaluate(() => {
* // now all pages will have 'foo'
* window.foo = 'bar'
* })
* }],
* prePageCloseHooks: [async (page, browserController) => {
* // collect information just before a page closes
* await page.screenshot();
* }],
* postPageCloseHooks: [(pageId, browserController) => {
* // clean up or log after a job is done
* console.log('Page closed: ', pageId)
* }]
* });
* ```
*/
export class BrowserPool<
Options extends BrowserPoolOptions = BrowserPoolOptions,
BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>,
BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>,
LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>,
PageOptions = Parameters<BrowserControllerReturn['newPage']>[0],
PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<
ReturnType<BrowserControllerReturn['newPage']>
>,
>
extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>>
implements IBrowserPool<PageReturn>
{
browserPlugins: BrowserPlugins;
/** @internal */
fingerprintOptions: FingerprintOptions;
/** @internal */
fingerprintInjector?: FingerprintInjector;
fingerprintGenerator?: FingerprintGenerator;
/** @internal */
fingerprintCache?: QuickLRU<string, BrowserFingerprintWithHeaders>;
readonly #maxOpenPagesPerBrowser: number;
readonly #maxOpenBrowsers: number;
readonly #retireBrowserAfterPageCount: number;
readonly #operationTimeoutMillis: number;
readonly #closeInactiveBrowserAfterMillis: number;
readonly #preLaunchHooks: PreLaunchHook<LaunchContextReturn>[];
readonly #postLaunchHooks: PostLaunchHook<BrowserControllerReturn>[];
readonly #prePageCreateHooks: PrePageCreateHook<BrowserControllerReturn, PageOptions>[];
readonly #postPageCreateHooks: PostPageCreateHook<BrowserControllerReturn, PageReturn>[];
readonly #prePageCloseHooks: PrePageCloseHook<BrowserControllerReturn, PageReturn>[];
readonly #postPageCloseHooks: PostPageCloseHook<BrowserControllerReturn>[];
#pageCounter = 0;
// oxlint-disable-next-line crawlee/prefer-private-fields -- tests observe page tracking and controller retirement directly
private pages = new Map<string, PageReturn>();
#pageIds = new WeakMap<PageReturn, string>();
#startingBrowserControllers = new Set<BrowserControllerReturn>();
// oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
private activeBrowserControllers = new Set<BrowserControllerReturn>();
// oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
private retiredBrowserControllers = new Set<BrowserControllerReturn>();
#pageToBrowserController = new WeakMap<PageReturn, BrowserControllerReturn>();
// oxlint-disable-next-line crawlee/prefer-private-fields -- tests replace this interval through bracket access
private browserKillerInterval?: NodeJS.Timeout;
#browserRetireInterval?: NodeJS.Timeout;
#limiter = pLimit(1);
#log!: CrawleeLogger;
/** Shared by all `newPage` callers waiting for a free slot, so they don't each register their own listeners. */
#capacityChange?: Promise<void>;
constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>) {
super();
this.#log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
const {
browserPlugins,
maxOpenPagesPerBrowser,
maxOpenBrowsers,
retireBrowserAfterPageCount,
operationTimeoutSecs,
closeInactiveBrowserAfterSecs,
retireInactiveBrowserAfterSecs,
preLaunchHooks,
postLaunchHooks,
prePageCreateHooks,
postPageCreateHooks,
prePageCloseHooks,
postPageCloseHooks,
useFingerprints,
fingerprintOptions,
} = parseArgument(options, browserPoolOptionsSchema);
const firstPluginConstructor = browserPlugins[0].constructor as typeof BrowserPlugin;
for (let i = 1; i < browserPlugins.length; i++) {
const providedPlugin = browserPlugins[i];
if (!(providedPlugin instanceof firstPluginConstructor)) {
const firstPluginName = firstPluginConstructor.name;
const providedPluginName = (providedPlugin as BrowserPlugin).constructor.name;
throw new Error(
`Browser plugin at index ${i} (${providedPluginName}) is not an instance of the same plugin as the first plugin provided (${firstPluginName}).`,
);
}
}
this.browserPlugins = browserPlugins as unknown as BrowserPlugins;
this.#maxOpenBrowsers = maxOpenBrowsers;
this.fingerprintOptions = fingerprintOptions;
this.#maxOpenPagesPerBrowser = maxOpenPagesPerBrowser;
this.#retireBrowserAfterPageCount = retireBrowserAfterPageCount;
this.#operationTimeoutMillis = operationTimeoutSecs * 1000;
this.#closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
// Sweeping slower than the window it enforces would round any sub-10s
// `closeInactiveBrowserAfterSecs` up to the sweep period.
this.browserKillerInterval = setInterval(
async () => this.closeInactiveRetiredBrowsers(),
Math.min(BROWSER_KILLER_INTERVAL_MILLIS, this.#closeInactiveBrowserAfterMillis),
);
this.browserKillerInterval.unref();
this.#browserRetireInterval = setInterval(
async () =>
this.activeBrowserControllers.forEach((controller) => {
if (
controller.activePages === 0 &&
controller.lastPageOpenedAt < Date.now() - retireInactiveBrowserAfterSecs * 1000
) {
this.retireBrowserController(controller);
}
}),
retireInactiveBrowserAfterSecs * 1000,
);
this.#browserRetireInterval!.unref();
// hooks
this.#preLaunchHooks = preLaunchHooks;
this.#postLaunchHooks = postLaunchHooks;
this.#prePageCreateHooks = prePageCreateHooks;
this.#postPageCreateHooks = postPageCreateHooks;
this.#prePageCloseHooks = prePageCloseHooks;
this.#postPageCloseHooks = postPageCloseHooks;
// fingerprinting
if (useFingerprints) {
this.#initializeFingerprinting();
// The fingerprint pre-launch hook goes last because of the fingerprint cache.
// It is usual to generate proxy per browser and we want to know the proxyUrl for the caching.
this.#preLaunchHooks = [...this.#preLaunchHooks, createFingerprintPreLaunchHook(this)];
this.#prePageCreateHooks = [createPrePageCreateHook(), ...this.#prePageCreateHooks];
this.#postPageCreateHooks = [
createPostPageCreateHook(this.fingerprintInjector!),
...this.#postPageCreateHooks,
];
}
}
/**
* Opens a new page in one of the running browsers or launches
* a new browser and opens a page there, if no browsers are active,
* or their page limits have been exceeded.
*
* **Session injection (best-effort):** When a {@apilink NewPageOptions.session|session} is
* provided, this implementation uses it as a cache key for browser fingerprints (when
* fingerprinting is enabled) and reads
* {@apilink ProxyInfo.url|session.proxyInfo.url} /
* {@apilink ProxyInfo.ignoreTlsErrors|session.proxyInfo.ignoreTlsErrors} as defaults
* for `proxyUrl` and `ignoreTlsErrors` respectively. Explicit `proxyUrl` /
* `ignoreTlsErrors` values in the options take precedence.
*
* Beyond fingerprint caching and proxy configuration, no other session
* properties are consumed — cookie and header injection remain the
* crawler's responsibility.
*/
async newPage(options: BrowserPoolNewPageOptions<PageOptions, BrowserPlugins[number]> = {}): Promise<PageReturn> {
const {
id = nanoid(),
pageOptions,
browserPlugin = this.#pickBrowserPlugin(),
session,
proxyUrl = session?.proxyInfo?.url,
ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors,
} = options;
if (this.pages.has(id)) {
throw new Error(`Page with ID: ${id} already exists.`);
}
if (browserPlugin && !this.browserPlugins.includes(browserPlugin)) {
throw new Error('Provided browserPlugin is not one of the plugins used by BrowserPool.');
}
// Limiter is necessary - https://github.com/apify/crawlee/issues/1126
return this.#limiter(async () => {
let browserController = this.#pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
// Checked under the limiter so concurrent waiters can't all pass on the same freed slot.
while (!browserController && !this.hasFreeBrowserSlot()) {
await this.#nextCapacityChange();
browserController = this.#pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
}
browserController ??= await this.#launchBrowser(id, { browserPlugin, proxyUrl, ignoreTlsErrors });
tryCancel();
return await this.#createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
});
}
/**
* Unlike {@apilink newPage}, `newPageInNewBrowser` always launches a new
* browser to open the page in. Use the `launchOptions` option to
* configure the new browser.
*/
async newPageInNewBrowser(
options: BrowserPoolNewPageInNewBrowserOptions<PageOptions, BrowserPlugins[number]> = {},
): Promise<PageReturn> {
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.#pickBrowserPlugin() } = options;
if (this.pages.has(id)) {
throw new Error(`Page with ID: ${id} already exists.`);
}
const browserController = await this.#launchBrowser(id, { launchOptions, browserPlugin });
tryCancel();
return await this.#createPageForBrowser(id, browserController, pageOptions);
}
/**
* Opens new pages with all available plugins and returns an array
* of pages in the same order as the plugins were provided to `BrowserPool`.
* This is useful when you want to run a script in multiple environments
* at the same time, typically in testing or website analysis.
*
* **Example:**
* ```js
* const browserPool = new BrowserPool({
* browserPlugins: [
* new PlaywrightPlugin(playwright.chromium),
* new PlaywrightPlugin(playwright.firefox),
* new PlaywrightPlugin(playwright.webkit),
* ]
* });
*
* const pages = await browserPool.newPageWithEachPlugin();
* const [chromiumPage, firefoxPage, webkitPage] = pages;
* ```
*/
async newPageWithEachPlugin(
optionsList: Omit<BrowserPoolNewPageOptions<PageOptions, BrowserPlugins[number]>, 'browserPlugin'>[] = [],
): Promise<PageReturn[]> {
const pagePromises = this.browserPlugins.map(async (browserPlugin, idx) => {
const userOptions = optionsList[idx] || {};
return this.newPage({
...userOptions,
browserPlugin,
});
});
return Promise.all(pagePromises);
}
/**
* Retrieves a {@apilink BrowserController} for a given page. This is useful
* when you're working only with pages and need to access the browser
* manipulation functionality.
*
* You could access the browser directly from the page,
* but that would circumvent `BrowserPool` and most likely
* cause weird things to happen, so please always use `BrowserController`
* to control your browsers. The function returns `undefined` if the
* browser is closed.
*
* @param page - Browser plugin page
*/
getBrowserControllerByPage(page: PageReturn): BrowserControllerReturn | undefined {
return this.#pageToBrowserController.get(page);
}
/**
* If you provided a custom ID to one of your pages or saved the
* randomly generated one, you can use this function to retrieve
* the page. If the page is no longer open, the function will
* return `undefined`.
*/
getPage(id: string): PageReturn | undefined {
return this.pages.get(id);
}
/**
* Page IDs are used throughout `BrowserPool` as a method of linking
* events. You can use a page ID to track the full lifecycle of the page.
* It is created even before a browser is launched and stays with the page
* until it's closed.
*/
getPageId(page: PageReturn): string | undefined {
return this.#pageIds.get(page);
}
async #createPageForBrowser(
pageId: string,
browserController: BrowserControllerReturn,
pageOptions: PageOptions = {} as PageOptions,
proxyUrl?: string,
ignoreTlsErrors?: boolean,
) {
// This is needed for concurrent newPage calls to wait for the browser launch.
// It's not ideal though, we need to come up with a better API.
await browserController.waitForActive();
tryCancel();
const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;
if (finalPageOptions) {
Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions));
if (ignoreTlsErrors) {
Object.assign(finalPageOptions, {
ignoreHTTPSErrors: true,
acceptInsecureCerts: true,
});
}
}
await this.#executeHooks(this.#prePageCreateHooks, pageId, browserController, finalPageOptions);
tryCancel();
let page: PageReturn;
try {
page = (await addTimeoutToPromise(
async () => browserController.newPage(finalPageOptions),
this.#operationTimeoutMillis,
'browserController.newPage() timed out.',
)) as PageReturn;
tryCancel();
this.pages.set(pageId, page);
this.#pageIds.set(page, pageId);
this.#pageToBrowserController.set(page, browserController);
// if you synchronously trigger a lot of page launches, browser will not get retired soon enough. Not sure if it's a problem, let's monitor it.
if (browserController.totalPages >= this.#retireBrowserAfterPageCount) {
this.retireBrowserController(browserController);
}
this.#overridePageClose(page);
} catch (err) {
this.retireBrowserController(browserController);
throw new Error(
`browserController.newPage() failed: ${browserController.id}\nCause:${(err as Error).message}.`,
);
}
await this.#executeHooks(this.#postPageCreateHooks, page, browserController);
tryCancel();
this.emit(BROWSER_POOL_EVENTS.PAGE_CREATED, page);
return page;
}
/**
* Removes a browser controller from the pool. The underlying
* browser will be closed after all its pages are closed.
*
*/
retireBrowserController(browserController: BrowserControllerReturn): void {
const isStarting = this.#startingBrowserControllers.has(browserController);
const isActive = this.activeBrowserControllers.has(browserController);
const hasBeenRetiredOrKilled = !isStarting && !isActive;
if (hasBeenRetiredOrKilled) return;
this.retiredBrowserControllers.add(browserController);
this.emit(BROWSER_POOL_EVENTS.BROWSER_RETIRED, browserController);
this.#startingBrowserControllers.delete(browserController);
this.activeBrowserControllers.delete(browserController);
}
/**
* Removes a browser from the pool. It will be
* closed after all its pages are closed.
*/
retireBrowserByPage(page: PageReturn): void {
const browserController = this.getBrowserControllerByPage(page);
if (browserController) this.retireBrowserController(browserController);
}
/**
* Releases a page back to the pool. The page is closed and, if the
* optional `error` is a {@apilink SessionError} (including
* {@apilink SessionRetiredError}), the browser controller that served
* the page is retired so that the finished session's state (cookies,
* storage, etc.) cannot leak into future sessions.
*
* This is the primary way the crawler should return pages to the pool.
*
* @param page The page to release.
* @param options.error The error that caused the page to be released, if any.
*/
async closePage(page: PageReturn, options?: { error?: Error }): Promise<void> {
if (options?.error instanceof SessionError) {
this.retireBrowserByPage(page);
}
// The bound on this lives in `overridePageClose`, where it covers the pool's own
// bookkeeping as well. Wrapping it here instead would time out the override and abandon
// that bookkeeping, leaving the browser holding a slot it can never get back.
await page.close();
}
/**
* Extracts the relevant state (currently just cookies) from a page via its
* owning {@apilink BrowserController}. Returns empty state when the page is
* no longer associated with a controller.
*
* As with {@apilink BrowserPool.injectPageState}, cookies are isolated per
* page only when the pool is configured with `useIncognitoPages: true`.
* With the default `useIncognitoPages: false`, the extracted cookies
* include those set by any sibling page sharing the same browser.
*/
async extractPageState(page: PageReturn): Promise<PageState> {
const controller = this.getBrowserControllerByPage(page);
if (!controller) {
return { cookies: [] };
}
return { cookies: await controller.getCookies(page) };
}
/**
* Injects state into a page via its owning {@apilink BrowserController}.
*
* No-op when the page is no longer associated with a controller.
*
* Note that cookies are isolated per page only when the pool is configured
* with `useIncognitoPages: true` — each page then gets its own browser
* context. With the default `useIncognitoPages: false`, all pages in a
* browser share a single context, so injected cookies are visible to every
* page served by that browser.
*/
async injectPageState(page: PageReturn, state: PageState): Promise<void> {
const controller = this.getBrowserControllerByPage(page);
if (!controller) {
return;
}
await controller.setCookies(page, state.cookies);
}
/**
* Removes all active browsers from the pool. The browsers will be
* closed after all their pages are closed.
*/
retireAllBrowsers(): void {
[...this.#startingBrowserControllers, ...this.activeBrowserControllers].forEach((controller) => {
this.retireBrowserController(controller);
});
}
/**
* Closes all managed browsers without waiting for pages to close.
* @return {Promise<void>}
*/
async closeAllBrowsers(): Promise<void> {
const controllers = this.#getAllBrowserControllers();
const promises = [...controllers]
.filter((controller) => controller.isActive)
.map(async (controller) => controller.close());
await Promise.all(promises);
}
async [Symbol.asyncDispose](): Promise<void> {
await this.destroy();
}
/**
* Closes every managed browser and empties the pool, which stays usable afterwards — a crawler releases its
* browsers when a run ends and may start another run on the same pool.
*/
async releaseAllBrowsers(): Promise<void> {
await this.closeAllBrowsers();
this.#startingBrowserControllers.clear();
this.activeBrowserControllers.clear();
this.retiredBrowserControllers.clear();
}
/**
* Closes all managed browsers and tears the pool down for good: its intervals are cleared without being
* re-armed and its listeners are dropped, so it cannot be used again.
*/
async destroy(): Promise<void> {
clearInterval(this.browserKillerInterval!);
clearInterval(this.#browserRetireInterval!);
this.browserKillerInterval = undefined;
this.#browserRetireInterval = undefined;
await this.releaseAllBrowsers();
this.removeAllListeners();
}
#getAllBrowserControllers() {
return new Set([
...this.#startingBrowserControllers,
...this.activeBrowserControllers,
...this.retiredBrowserControllers,
]);
}
async #launchBrowser(pageId: string, options: InternalLaunchBrowserOptions<BrowserPlugins[number]>) {
const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;
const browserController = browserPlugin.createController() as BrowserControllerReturn;
this.#startingBrowserControllers.add(browserController);
const launchContext = browserPlugin.createLaunchContext({
id: pageId,
launchOptions,
proxyUrl,
});
// Disable SSL verification for MITM proxies
if (ignoreTlsErrors) {
/**
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
*/
(launchContext.launchOptions as Record<string, unknown>).ignoreHTTPSErrors = true;
(launchContext.launchOptions as Record<string, unknown>).acceptInsecureCerts = true;
}
try {
// If the hooks or the launch fails, we need to delete the controller,
// because otherwise it would be stuck in limbo without a browser.
await this.#executeHooks(this.#preLaunchHooks, pageId, launchContext);
tryCancel();
const browser = await browserPlugin.launch(launchContext);
tryCancel();
browserController.assignBrowser(browser, launchContext);
} catch (err) {
this.#startingBrowserControllers.delete(browserController);
throw err;
}
this.#log.debug('Launched new browser.', { id: browserController.id });
browserController.proxyUrl = proxyUrl;
try {
// If the launch fails on the post-launch hooks, we need to clean up
// both the controller and the browser before throwing.
await this.#executeHooks(this.#postLaunchHooks, pageId, browserController);
} catch (err) {
this.#startingBrowserControllers.delete(browserController);
browserController.close().catch((closeErr) => {
this.#log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
id: browserController.id,
});
});
throw err;
}
tryCancel();
browserController.activate();
this.#startingBrowserControllers.delete(browserController);
this.activeBrowserControllers.add(browserController);
this.emit(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, browserController);
return browserController;
}
/**
* Picks plugins round robin.
*/
#pickBrowserPlugin() {
const pluginIndex = this.#pageCounter % this.browserPlugins.length;
this.#pageCounter++;
return this.browserPlugins[pluginIndex];
}
#pickBrowserWithFreeCapacity(browserPlugin: BrowserPlugin, options?: { proxyUrl?: string }) {
return [...this.activeBrowserControllers].find((controller) => {
const hasCapacity = controller.activePages < this.#maxOpenPagesPerBrowser;
const isCorrectPlugin = controller.browserPlugin === browserPlugin;
const isSameProxyUrl = controller.proxyUrl === options?.proxyUrl;
return (
isCorrectPlugin &&
hasCapacity &&
(!controller.launchContext.browserPerProxy ||
(options?.proxyUrl && isSameProxyUrl) ||
(!options?.proxyUrl && !controller.proxyUrl))
);
});
}
// oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
private async closeInactiveRetiredBrowsers() {
const closedBrowserIds: string[] = [];
for (const controller of this.retiredBrowserControllers) {
const millisSinceLastPageOpened = Date.now() - controller.lastPageOpenedAt;
const isBrowserIdle = millisSinceLastPageOpened >= this.#closeInactiveBrowserAfterMillis;
const isBrowserEmpty = controller.activePages === 0;
if (isBrowserIdle || isBrowserEmpty) {
const { id } = controller;
this.#log.debug('Closing retired browser.', { id });
await controller.close();
this.retiredBrowserControllers.delete(controller);
closedBrowserIds.push(id);
}
}
if (closedBrowserIds.length) {
this.#log.debug('Closed retired browsers.', {
count: closedBrowserIds.length,
closedBrowserIds,
});
}
}
#overridePageClose(page: PageReturn) {
const originalPageClose = page.close;
const browserController = this.#pageToBrowserController.get(page)!;
const pageId = this.getPageId(page)!;
page.close = async (...args: unknown[]) => {
// A browser can acknowledge the close and then never destroy the target, in which case
// this never settles and the page's own `close` event never fires either (Chromium
// 536385539). Bound the whole sequence so nothing here can hang the caller, then
// reconcile the pool regardless of the outcome, so a page that refuses to close cannot
// leave the pool believing it is still open. `Promise.race` rather than
// `addTimeoutToPromise`: the latter inherits its AbortController from the calling frame,
// so a timeout here would cancel the task of whoever awaited `page.close()`.
let pageClosed = false;
const closing = (async () => {
await this.#executeHooks(this.#prePageCloseHooks, page, browserController);
await originalPageClose.apply(page, args).catch((err: Error) => {
this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
});
// Whether the page itself is gone. Tracked separately from the sequence finishing,
// so that a slow hook does not get the browser retired.
pageClosed = true;
await this.#executeHooks(this.#postPageCloseHooks, pageId, browserController);
})();
let timeout: NodeJS.Timeout | undefined;
const finished = await Promise.race([
closing.then(
() => true,
(err: Error) => {
this.#log.warning(
`Closing a page failed, releasing it from the pool anyway.\nCause:${err.message}`,
{ id: browserController.id, pageId },
);
return true;
},
),
new Promise<boolean>((resolve) => {
timeout = setTimeout(() => resolve(false), PAGE_CLOSE_TIMEOUT_MILLIS);
}),
]);
clearTimeout(timeout);
if (!finished) {
this.#log.warning(
`Closing a page did not finish within ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds, ` +
'releasing it from the pool anyway.',
{ id: browserController.id, pageId },
);
}
if (!pageClosed) {
// The page is still attached, so this browser cannot be trusted with more work.
// Retiring it lets the reclamation below close the process once its pages drain.
this.retireBrowserController(browserController);
}
browserController.registerPageClosed(page);
this.pages.delete(pageId);
this.#closeRetiredBrowserWithNoPages(browserController);
this.emit(BROWSER_POOL_EVENTS.PAGE_CLOSED, page);
};
}
async #executeHooks(hooks: ((...args: any[]) => unknown)[], ...args: unknown[]) {
for (const hook of hooks) {
await hook(...args);
}
}
#closeRetiredBrowserWithNoPages(browserController: BrowserControllerReturn) {
if (browserController.activePages === 0 && this.retiredBrowserControllers.has(browserController)) {
// Run this with a delay, otherwise page.close()
// might fail with "Protocol error (Target.closeTarget): Target closed."
setTimeout(() => {
this.#log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
void browserController.close().finally(() => {
this.retiredBrowserControllers.delete(browserController);
});
}, PAGE_CLOSE_KILL_TIMEOUT_MILLIS);
}
}
/**
* Returns `true` if the pool can accept a new browser launch without exceeding `maxOpenBrowsers`.
* Counts starting, active, and retired browsers.
*
* @internal
*/
hasFreeBrowserSlot(): boolean {
const total =
this.#startingBrowserControllers.size +
this.activeBrowserControllers.size +
this.retiredBrowserControllers.size;
return total < this.#maxOpenBrowsers;
}
/**
* Resolves on the next browser-retired / page-closed event, or after a short fallback interval. All
* concurrently-waiting `newPage` calls share a single promise (and a single pair of event listeners) per tick.
*/
#nextCapacityChange(): Promise<void> {
this.#capacityChange ??= new Promise<void>((resolve) => {
const done = () => {
clearTimeout(timer);
this.off(BROWSER_POOL_EVENTS.BROWSER_RETIRED, done);
this.off(BROWSER_POOL_EVENTS.PAGE_CLOSED, done);
this.#capacityChange = undefined;
resolve();
};
const timer = setTimeout(done, SLOT_POLL_INTERVAL_MILLIS);
timer.unref?.();
this.once(BROWSER_POOL_EVENTS.BROWSER_RETIRED, done);
this.once(BROWSER_POOL_EVENTS.PAGE_CLOSED, done);
});
return this.#capacityChange;
}
#initializeFingerprinting(): void {
const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;
this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);
this.fingerprintInjector = new FingerprintInjector();
if (useFingerprintCache) {
this.fingerprintCache = new QuickLRU({ maxSize: fingerprintCacheSize });
}
}
}
export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> extends NewPageOptions {
/**
* The proxy URL the pool uses internally to route the page: it keys browser
* reuse (with `browserPerProxy`, only a browser already on this proxy is
* reused), configures the launched browser, and is applied to incognito
* pages. When omitted, it is derived from the
* {@apilink NewPageOptions.session|session}'s `proxyInfo`; an explicit value
* here takes precedence.
*
* This is an implementation detail of the built-in `BrowserPool`'s proxy
* handling and is intentionally not part of the {@apilink IBrowserPool}
* contract — through that interface the proxy is supplied via the session.
*/
proxyUrl?: string;
/**
* Disable TLS certificate verification for MITM proxies. Applied both when
* launching a new browser and when creating a page in an existing one. When
* omitted, it is derived from the
* {@apilink NewPageOptions.session|session}'s `proxyInfo`; an explicit value
* here takes precedence.
*
* This is an implementation detail of the built-in `BrowserPool` and is
* intentionally not part of the {@apilink IBrowserPool} contract — through
* that interface, configure it via the session's `proxyInfo` or through the
* browser's `launchOptions`.
*/
ignoreTlsErrors?: boolean;
/**
* Some libraries (Playwright) allow you to open new pages with specific
* options. Use this property to set those options.
*/
pageOptions?: PageOptions;
/**
* Choose a plugin to open the page with. If none is provided,
* one of the pool's available plugins will be used.
*
* It must be one of the plugins browser pool was created with.
* If you wish to start a browser with a different configuration,
* see the `newPageInNewBrowser` function.
*/
browserPlugin?: BP;
}
export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends BrowserPlugin> {
/**
* Assign a custom ID to the page. If you don't a random string ID
* will be generated.
*/
id?: string;
/**
* Some libraries (Playwright) allow you to open new pages with specific
* options. Use this property to set those options.
*/
pageOptions?: PageOptions;
/**
* Provide a plugin to launch the browser. If none is provided,
* one of the pool's available plugins will be used.
*
* If you configured `BrowserPool` to rotate multiple libraries,
* such as both Puppeteer and Playwright, you should always set
* the `browserPlugin` when using the `launchOptions` option.
*
* The plugin will not be added to the list of plugins used by
* the pool. You can either use one of those, to launch a specific
* browser, or provide a completely new configuration.
*/
browserPlugin?: BP;
/**
* Options that will be used to launch the new browser.
*/
launchOptions?: BP['launchOptions'];
}
interface InternalLaunchBrowserOptions<BP extends BrowserPlugin> {
browserPlugin: BP;
launchOptions?: BP['launchOptions'];
proxyUrl?: string;
ignoreTlsErrors?: boolean;
}