Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor!: replace `browserPoolOptions` with browser pool factories (#4026)
Jan Buchar39148e6f1ab1
diff --git a/docs/guides/avoid_blocking.mdx b/docs/guides/avoid_blocking.mdxindex 65cd2fc95..f2acff31c 100644--- a/docs/guides/avoid_blocking.mdx+++ b/docs/guides/avoid_blocking.mdx@@ -45,7 +45,7 @@ In certain cases we want to narrow down the fingerprints used - e.g. specify a c## Disabling browser fingerprints-On the contrary, sometimes we want to entirely disable the usage of browser fingerprints. This is easy to do with Crawlee too. All we have to do is set the `useFingerprints` option of the `browserPoolOptions` to `false`:+On the contrary, sometimes we want to entirely disable the usage of browser fingerprints. This is easy to do with Crawlee too. All we have to do is build the crawler's browser pool with `useFingerprints` set to `false`:<Tabs groupId="avoid_blocking"><TabItem value="playwright" label="PlaywrightCrawler" default>diff --git a/docs/guides/avoid_blocking_camoufox.ts b/docs/guides/avoid_blocking_camoufox.tsindex 01b8f8ca4..65e3ffb6f 100644--- a/docs/guides/avoid_blocking_camoufox.ts+++ b/docs/guides/avoid_blocking_camoufox.ts@@ -1,18 +1,18 @@-import { PlaywrightCrawler, handleCloudflareChallengeHook } from 'crawlee';+import { PlaywrightCrawler, handleCloudflareChallengeHook, playwrightBrowserPool } from 'crawlee';import { launchOptions } from 'camoufox-js';import { firefox } from 'playwright';const crawler = new PlaywrightCrawler({postNavigationHooks: [handleCloudflareChallengeHook()],- browserPoolOptions: {+ browserPool: playwrightBrowserPool({// Disable the default fingerprint spoofing to avoid conflicts with Camoufox.useFingerprints: false,- },- launchContext: {- launcher: firefox,- launchOptions: await launchOptions({- headless: true,- }),- },+ launchContext: {+ launcher: firefox,+ launchOptions: await launchOptions({+ headless: true,+ }),+ },+ }),// ...});diff --git a/docs/guides/avoid_blocking_playwright.ts b/docs/guides/avoid_blocking_playwright.tsindex 1e5e23aea..ec88f284c 100644--- a/docs/guides/avoid_blocking_playwright.ts+++ b/docs/guides/avoid_blocking_playwright.ts@@ -1,8 +1,8 @@-import { PlaywrightCrawler } from 'crawlee';+import { PlaywrightCrawler, playwrightBrowserPool } from 'crawlee';import { BrowserName, DeviceCategory, OperatingSystemsName } from '@crawlee/browser-pool';const crawler = new PlaywrightCrawler({- browserPoolOptions: {+ browserPool: playwrightBrowserPool({useFingerprints: true, // this is the defaultfingerprintOptions: {fingerprintGeneratorOptions: {@@ -16,6 +16,6 @@ const crawler = new PlaywrightCrawler({operatingSystems: [OperatingSystemsName.windows],},},- },+ }),// ...});diff --git a/docs/guides/avoid_blocking_playwright_fingerprints_off.ts b/docs/guides/avoid_blocking_playwright_fingerprints_off.tsindex c36e52fc3..18a4ee900 100644--- a/docs/guides/avoid_blocking_playwright_fingerprints_off.ts+++ b/docs/guides/avoid_blocking_playwright_fingerprints_off.ts@@ -1,8 +1,8 @@-import { PlaywrightCrawler } from 'crawlee';+import { PlaywrightCrawler, playwrightBrowserPool } from 'crawlee';const crawler = new PlaywrightCrawler({- browserPoolOptions: {+ browserPool: playwrightBrowserPool({useFingerprints: false,- },+ }),// ...});diff --git a/docs/guides/avoid_blocking_puppeteer.ts b/docs/guides/avoid_blocking_puppeteer.tsindex 5af3ca7fa..5ca21c82c 100644--- a/docs/guides/avoid_blocking_puppeteer.ts+++ b/docs/guides/avoid_blocking_puppeteer.ts@@ -1,8 +1,8 @@-import { PuppeteerCrawler } from 'crawlee';+import { PuppeteerCrawler, puppeteerBrowserPool } from 'crawlee';import { BrowserName, DeviceCategory } from '@crawlee/browser-pool';const crawler = new PuppeteerCrawler({- browserPoolOptions: {+ browserPool: puppeteerBrowserPool({useFingerprints: true, // this is the defaultfingerprintOptions: {fingerprintGeneratorOptions: {@@ -11,6 +11,6 @@ const crawler = new PuppeteerCrawler({locales: ['en-US'],},},- },+ }),// ...});diff --git a/docs/guides/avoid_blocking_puppeteer_fingerprints_off.ts b/docs/guides/avoid_blocking_puppeteer_fingerprints_off.tsindex 425b4d3d5..cf97daaab 100644--- a/docs/guides/avoid_blocking_puppeteer_fingerprints_off.ts+++ b/docs/guides/avoid_blocking_puppeteer_fingerprints_off.ts@@ -1,8 +1,8 @@-import { PuppeteerCrawler } from 'crawlee';+import { PuppeteerCrawler, puppeteerBrowserPool } from 'crawlee';const crawler = new PuppeteerCrawler({- browserPoolOptions: {+ browserPool: puppeteerBrowserPool({useFingerprints: false,- },+ }),// ...});diff --git a/docs/public-api/crawlee-browser.api.md b/docs/public-api/crawlee-browser.api.mdindex a4b3e6cf0..da5aff65b 100644--- a/docs/public-api/crawlee-browser.api.md+++ b/docs/public-api/crawlee-browser.api.md@@ -9,8 +9,6 @@ import type { Awaitable } from '@crawlee/types';import { BaseHttpClient } from '@crawlee/http-client';import { BasicCrawler } from '@crawlee/basic';import { BasicCrawlerOptions } from '@crawlee/basic';-import type { BrowserController } from '@crawlee/browser-pool';-import type { BrowserPlugin } from '@crawlee/browser-pool';import type { BrowserPluginOptions } from '@crawlee/browser-pool';import type { BrowserPoolHooks } from '@crawlee/browser-pool';import type { BrowserPoolOptions } from '@crawlee/browser-pool';@@ -26,14 +24,16 @@ import { EventManager } from '@crawlee/basic';import type { ExtractLinksOptions } from '@crawlee/basic';import type { GetUserDataFromRequest } from '@crawlee/basic';import type { IBrowserPool } from '@crawlee/types';-import type { InferBrowserPluginArray } from '@crawlee/browser-pool';-import type { LaunchContext } from '@crawlee/browser-pool';import type { LoadedRequest } from '@crawlee/basic';+import type { RemoteBrowserPoolOptions } from '@crawlee/browser-pool';import { Request as Request_2 } from '@crawlee/basic';import type { RequestHandler } from '@crawlee/basic';import type { RouterHandler } from '@crawlee/basic';import { z } from 'zod';+// @public+export function assertBrowserPoolNotConfigured(crawlerName: string, ignoredOptions: Dictionary): void;+// Not exported by the entry point; reachable only as a referenced type.// @public (undocumented)interface BaseResponse {@@ -43,9 +43,10 @@ interface BaseResponse {}// @public-export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {- protected constructor(options: BrowserCrawlerOptions<Page, Response, Context, ContextExtension, ExtendedContext, BrowserPoolOptions, Routes, StatisticStateExtension> & {+export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+ protected constructor(options: BrowserCrawlerOptions<Page, Response, Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & {contextPipelineBuilder: () => ContextPipeline<CrawlingContext, Context>;+ browserPoolBuilder: (remoteBrowser?: CrawlerRemoteBrowserOptions) => OwnedBrowserPool<Page>;});get browserPool(): IBrowserPool<Page>;// (undocumented)@@ -66,10 +67,9 @@ export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, RespopreNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;saveResponseCookies: z.ZodDefault<z.ZodBoolean>;proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;ignoreIframes: z.ZodDefault<z.ZodBoolean>;@@ -122,10 +122,9 @@ export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, RespopreNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;saveResponseCookies: z.ZodDefault<z.ZodBoolean>;proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;ignoreIframes: z.ZodDefault<z.ZodBoolean>;@@ -176,12 +175,10 @@ export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Respo}// @public (undocumented)-export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {+export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {browserPool?: IBrowserPool<Page>;- browserPoolOptions?: Partial<BrowserPoolOptions> & Partial<BrowserPoolHooks<__BrowserControllerReturn, __LaunchContextReturn>>;errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;- headless?: boolean | 'new' | 'old';ignoreIframes?: boolean;ignoreShadowRoots?: boolean;// (undocumented)@@ -220,6 +217,19 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginOuserDataDir?: string;}+// @public+export type LauncherBrowserPoolOptions = Omit<BrowserPoolOptions, 'browserPlugins'> & {+ [Hook in keyof BrowserPoolHooks<any, any, any>]?: readonly ((...args: any[]) => unknown)[];+};++// @public+export type LauncherRemoteBrowserPoolOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins'>;++// @public+export type OwnedBrowserPool<Page> = IBrowserPool<Page> & {+ destroy: () => Promise<void>;+};+export * from "@crawlee/basic";diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.mdindex d64461ed4..ea3f16af0 100644--- a/docs/public-api/crawlee-playwright.api.md+++ b/docs/public-api/crawlee-playwright.api.md@@ -12,10 +12,13 @@ import type { BasicCrawlerOptions } from '@crawlee/basic';import { BatchAddRequestsResult } from '@crawlee/types';import type { Browser } from 'playwright';import { BrowserCrawler } from '@crawlee/browser';-import type { BrowserCrawlerOptions } from '@crawlee/browser';+import { BrowserCrawlerOptions } from '@crawlee/browser';import type { BrowserCrawlingContext } from '@crawlee/browser';import type { BrowserHook } from '@crawlee/browser';import type { BrowserLaunchContext } from '@crawlee/browser';+import type { BrowserPool } from '@crawlee/browser-pool';+import type { BrowserPoolHooks } from '@crawlee/browser-pool';+import type { BrowserPoolOptions } from '@crawlee/browser-pool';import type { BrowserType } from 'playwright';import { Cheerio } from 'cheerio';import { CheerioAPI } from '@crawlee/browser';@@ -37,6 +40,8 @@ import type { LoadedRequest } from '@crawlee/browser';import type { Page } from 'playwright';import { PlaywrightPlugin } from '@crawlee/browser-pool';import type { RecoverableStatePersistenceOptions } from '@crawlee/core';+import type { RemoteBrowserPool } from '@crawlee/browser-pool';+import type { RemoteBrowserPoolOptions } from '@crawlee/browser-pool';import type { Request as Request_2 } from '@crawlee/core';import { Request as Request_3 } from '@crawlee/browser';import type { RequestHandler } from '@crawlee/browser';@@ -276,6 +281,21 @@ export function launchPlaywright(launchContext?: PlaywrightLaunchContext, config// @publicfunction parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;+// @public+export type PlaywrightBrowserPool = BrowserPool<{+ browserPlugins: [PlaywrightPlugin];+}, [PlaywrightPlugin]>;++// @public+export function playwrightBrowserPool(options?: PlaywrightBrowserPoolOptions): PlaywrightBrowserPool;++// @public (undocumented)+export interface PlaywrightBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PlaywrightPlugin['createController']>, ReturnType<PlaywrightPlugin['createLaunchContext']>, Page> {+ configuration?: Configuration;+ headless?: boolean;+ launchContext?: PlaywrightLaunchContext;+}+declare namespace playwrightClickElements {export {enqueueLinksByClickingElements,@@ -302,9 +322,7 @@ interface PlaywrightContextUtils {}// @public-export class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, Response_2, {- browserPlugins: [PlaywrightPlugin];-}, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+export class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, Response_2, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);// (undocumented)protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, Response_2, Dictionary, Dictionary> & {@@ -325,14 +343,14 @@ export class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedConprotected navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: PlaywrightDirectNavigationOptions): Promise<Response_2 | null>;// (undocumented)protected static optionsSchema: z.ZodObject<{- browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;+ headless: z.ZodOptional<z.ZodBoolean>;launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;saveResponseCookies: z.ZodDefault<z.ZodBoolean>;proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;@@ -382,14 +400,14 @@ export class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedCon}, z.core.$strict>;// (undocumented)protected static optionsShape: {- browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;+ headless: z.ZodOptional<z.ZodBoolean>;launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;saveResponseCookies: z.ZodDefault<z.ZodBoolean>;proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;@@ -440,9 +458,8 @@ export class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedCon}// @public (undocumented)-export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, Response_2, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {- browserPlugins: [PlaywrightPlugin];-}, Routes, StatisticStateExtension> {+export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, Response_2, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+ headless?: boolean;launchContext?: PlaywrightLaunchContext;postNavigationHooks?: BrowserHook<PlaywrightCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];preNavigationHooks?: BrowserHook<PlaywrightCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];@@ -501,6 +518,13 @@ declare namespace playwrightUtils {}}+// @public+export function remotePlaywrightBrowserPool(options: RemotePlaywrightBrowserPoolOptions): RemoteBrowserPool<Page>;++// @public (undocumented)+export interface RemotePlaywrightBrowserPoolOptions extends Pick<PlaywrightBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {+}+// @public (undocumented)export type RenderingType = 'clientOnly' | 'static';diff --git a/docs/public-api/crawlee-puppeteer.api.md b/docs/public-api/crawlee-puppeteer.api.mdindex 92cbf390b..a545068b3 100644--- a/docs/public-api/crawlee-puppeteer.api.md+++ b/docs/public-api/crawlee-puppeteer.api.md@@ -8,10 +8,13 @@ import { BaseHttpClient } from '@crawlee/http-client';import { BatchAddRequestsResult } from '@crawlee/types';import type { Browser } from 'puppeteer';import { BrowserCrawler } from '@crawlee/browser';-import type { BrowserCrawlerOptions } from '@crawlee/browser';+import { BrowserCrawlerOptions } from '@crawlee/browser';import type { BrowserCrawlingContext } from '@crawlee/browser';import type { BrowserHook } from '@crawlee/browser';import type { BrowserLaunchContext } from '@crawlee/browser';+import type { BrowserPool } from '@crawlee/browser-pool';+import type { BrowserPoolHooks } from '@crawlee/browser-pool';+import type { BrowserPoolOptions } from '@crawlee/browser-pool';import { CheerioAPI } from '@crawlee/browser';import { CheerioRoot } from '@crawlee/utils/internal';import type { ClickOptions } from 'puppeteer';@@ -27,6 +30,8 @@ import { IRequestManager } from '@crawlee/browser';import type { LaunchOptions } from 'puppeteer';import type { Page } from 'puppeteer';import { PuppeteerPlugin } from '@crawlee/browser-pool';+import type { RemoteBrowserPool } from '@crawlee/browser-pool';+import type { RemoteBrowserPoolOptions } from '@crawlee/browser-pool';import { Request as Request_2 } from '@crawlee/browser';import type { RequestTransform } from '@crawlee/browser';import type { ResponseForRequest } from 'puppeteer';@@ -145,6 +150,21 @@ export function launchPuppeteer(launchContext?: PuppeteerLaunchContext, configur// @publicfunction parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;+// @public+export type PuppeteerBrowserPool = BrowserPool<{+ browserPlugins: [PuppeteerPlugin];+}, [PuppeteerPlugin]>;++// @public+export function puppeteerBrowserPool(options?: PuppeteerBrowserPoolOptions): PuppeteerBrowserPool;++// @public (undocumented)+export interface PuppeteerBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PuppeteerPlugin['createController']>, ReturnType<PuppeteerPlugin['createLaunchContext']>, Page> {+ configuration?: Configuration;+ headless?: boolean | 'new' | 'old';+ launchContext?: PuppeteerLaunchContext;+}+declare namespace puppeteerClickElements {export {enqueueLinksByClickingElements,@@ -172,9 +192,7 @@ interface PuppeteerContextUtils {}// @public-export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, HTTPResponse, {- browserPlugins: [PuppeteerPlugin];-}, LaunchOptions, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, HTTPResponse, LaunchOptions, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);// (undocumented)protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, HTTPResponse, Dictionary, Dictionary> & {@@ -195,13 +213,13 @@ export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContprotected navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: PuppeteerDirectNavigationOptions): Promise<HTTPResponse | null>;// (undocumented)protected static optionsSchema: z.ZodObject<{- browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;+ headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;saveResponseCookies: z.ZodDefault<z.ZodBoolean>;proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;@@ -251,13 +269,13 @@ export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedCont}, z.core.$strict>;// (undocumented)protected static optionsShape: {- browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;+ headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;saveResponseCookies: z.ZodDefault<z.ZodBoolean>;proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;@@ -308,9 +326,8 @@ export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedCont}// @public (undocumented)-export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, {- browserPlugins: [PuppeteerPlugin];-}, Routes, StatisticStateExtension> {+export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+ headless?: boolean | 'new' | 'old';launchContext?: PuppeteerLaunchContext;postNavigationHooks?: BrowserHook<PuppeteerCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];preNavigationHooks?: BrowserHook<PuppeteerCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];@@ -379,6 +396,13 @@ declare namespace puppeteerUtils {}}+// @public+export function remotePuppeteerBrowserPool(options: RemotePuppeteerBrowserPoolOptions): RemoteBrowserPool<Page>;++// @public (undocumented)+export interface RemotePuppeteerBrowserPoolOptions extends Pick<PuppeteerBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {+}+// @publicfunction removeInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;diff --git a/docs/public-api/crawlee-stagehand.api.md b/docs/public-api/crawlee-stagehand.api.mdindex 02c0224b3..e50a2ef0b 100644--- a/docs/public-api/crawlee-stagehand.api.md+++ b/docs/public-api/crawlee-stagehand.api.md@@ -13,12 +13,15 @@ import { BaseHttpClient } from '@crawlee/http-client';import type { Browser } from 'playwright';import type { BrowserController } from '@crawlee/browser-pool';import { BrowserCrawler } from '@crawlee/browser';-import type { BrowserCrawlerOptions } from '@crawlee/browser';+import { BrowserCrawlerOptions } from '@crawlee/browser';import type { BrowserCrawlingContext } from '@crawlee/browser';import type { BrowserHook } from '@crawlee/browser';import type { BrowserLaunchContext } from '@crawlee/browser';import { BrowserPlugin } from '@crawlee/browser-pool';import type { BrowserPluginOptions } from '@crawlee/browser-pool';+import type { BrowserPool } from '@crawlee/browser-pool';+import type { BrowserPoolHooks } from '@crawlee/browser-pool';+import type { BrowserPoolOptions } from '@crawlee/browser-pool';import type { BrowserType } from 'playwright';import { Configuration } from '@crawlee/browser';import type { ContextPipeline } from '@crawlee/browser';@@ -35,6 +38,8 @@ import { ModelConfiguration } from '@browserbasehq/stagehand';import type { NonStreamingAgentInstance } from '@browserbasehq/stagehand';import { ObserveOptions } from '@browserbasehq/stagehand';import type { Page } from 'playwright';+import type { RemoteBrowserPool } from '@crawlee/browser-pool';+import type { RemoteBrowserPoolOptions } from '@crawlee/browser-pool';import type { RequestHandler } from '@crawlee/browser';import type { Response as Response_2 } from 'playwright';import type { RouterHandler } from '@crawlee/browser';@@ -70,12 +75,33 @@ export { ModelConfiguration }export { ObserveOptions }+// @public+export function remoteStagehandBrowserPool(options: RemoteStagehandBrowserPoolOptions): RemoteBrowserPool<Page>;++// @public (undocumented)+export interface RemoteStagehandBrowserPoolOptions extends Pick<StagehandBrowserPoolOptions, 'launchContext' | 'stagehandOptions' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {+}+export { Stagehand }// @public-export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<StagehandPage, Response_2, {+export type StagehandBrowserPool = BrowserPool<{browserPlugins: [StagehandPlugin];-}, LaunchOptions, StagehandCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+}, [StagehandPlugin]>;++// @public+export function stagehandBrowserPool(options?: StagehandBrowserPoolOptions): StagehandBrowserPool;++// @public (undocumented)+export interface StagehandBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<StagehandPlugin['createController']>, ReturnType<StagehandPlugin['createLaunchContext']>, Page> {+ configuration?: Configuration;+ headless?: boolean;+ launchContext?: StagehandLaunchContext;+ stagehandOptions?: StagehandOptions;+}++// @public+export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<StagehandPage, Response_2, LaunchOptions, StagehandCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {constructor(options?: StagehandCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);// (undocumented)protected buildContextPipeline(): ContextPipeline<CrawlingContext, StagehandCrawlingContext>;@@ -83,13 +109,13 @@ export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedCont// (undocumented)protected static optionsSchema: z.ZodObject<{stagehandOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;+ headless: z.ZodOptional<z.ZodBoolean>;navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;saveResponseCookies: z.ZodDefault<z.ZodBoolean>;proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;@@ -140,13 +166,13 @@ export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedCont// (undocumented)protected static optionsShape: {stagehandOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;+ headless: z.ZodOptional<z.ZodBoolean>;navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;saveResponseCookies: z.ZodDefault<z.ZodBoolean>;proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;@@ -197,9 +223,8 @@ export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedCont}// @public-export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<StagehandPage, Response_2, StagehandCrawlingContext, ContextExtension, ExtendedContext, {- browserPlugins: [StagehandPlugin];-}, Routes, StatisticStateExtension> {+export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<StagehandPage, Response_2, StagehandCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+ headless?: boolean;launchContext?: StagehandLaunchContext;postNavigationHooks?: BrowserHook<StagehandCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];preNavigationHooks?: BrowserHook<StagehandCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex 291bdc828..c839f0045 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -53,6 +53,7 @@ The purely mechanical renames, collected in one place. Where a row links to a se| `enqueueLinks({ globs, regexps, pseudoUrls })` | `enqueueLinks({ include })` ([details](#globs-regexps-and-pseudourls-replaced-by-include)) || `(await enqueueLinks()).processedRequests` | `(await enqueueLinks()).addedRequests` ([details](#enqueuelinks-return-value-reshaped-addrequestsbatchedresult-instead-of-batchaddrequestsresult)) || `autoscaledPoolOptions` | `taskLoopOptions` ([narrowed](#autoscaledpooloptions-is-now-taskloopoptions-and-no-longer-carries-concurrency-config)) |+| `browserPoolOptions` | `browserPool` + a `*BrowserPool()` factory ([details](#browserpooloptions-is-removed)) || `gotScraping` (from `@crawlee/utils`) | `GotScrapingHttpClient` (`@crawlee/got-scraping-client`) || `SDK_`-prefixed internal KVS keys | `CRAWLEE_`-prefixed ([details](#internal-kvs-keys-renamed)) || `ArgumentError` (from `ow`) | `ArgumentValidationError` ([details](#argument-validation-errors-use-zod)) |@@ -443,6 +444,42 @@ Previously, `BrowserCrawler` with `saveResponseCookies` (formerly `persistCookieIn v4, when `saveResponseCookies` is enabled (the default), browser cookies are also re-read and stored in the session cookie jar **after** `requestHandler` completes. If you relied on handler-set cookies staying page-local and not affecting later requests on the same session, set `saveResponseCookies: false` or clear/overwrite cookies on the session explicitly.+### `browserPoolOptions` is removed++`browserPoolOptions` is gone from every browser crawler. It was a second way of configuring the very pool that the `browserPool` option accepts, and the two could not be combined — passing a pool made the options silently disappear.++Build the pool with the factory that matches your crawler instead. Each factory takes every `BrowserPool` option plus the crawler's own `launchContext` and `headless`, and derives the browser plugin from them, so the pool can never mismatch the crawler it is passed to:++| crawler | factory | remote counterpart |+| --- | --- | --- |+| `PlaywrightCrawler` | `playwrightBrowserPool()` | `remotePlaywrightBrowserPool()` |+| `PuppeteerCrawler` | `puppeteerBrowserPool()` | `remotePuppeteerBrowserPool()` |+| `StagehandCrawler` | `stagehandBrowserPool()` | `remoteStagehandBrowserPool()` |++**Before:**+```typescript+const crawler = new PlaywrightCrawler({+ browserPoolOptions: { useFingerprints: false },+ launchContext: { launcher: firefox },+});+```++**After:**+```typescript+const crawler = new PlaywrightCrawler({+ browserPool: playwrightBrowserPool({+ useFingerprints: false,+ launchContext: { launcher: firefox },+ }),+});+```++Building the pool outside the crawler has one consequence worth knowing: a pool passed as `browserPool` is borrowed, so the crawler never destroys it, and the options that would have configured a pool of the crawler's own — `launchContext`, `headless` and `remoteBrowser` — are now **rejected** instead of silently ignored. Move them into the factory call.++`remoteBrowser` keeps working on its own for the terse case; reach for a `remote*BrowserPool()` factory when you also want to tune the pool wrapping the remote connection, or to share one remote pool between crawlers.++`headless` is now declared on each concrete crawler rather than on `BrowserCrawler`, so the puppeteer-only `'new'` and `'old'` values are only accepted by `PuppeteerCrawler`.+### `ignoreSslErrors` is renamed to `ignoreTlsErrors`The crawler option is renamed to `ignoreTlsErrors`, matching the naming used everywhere else in v4 (`session.proxyInfo.ignoreTlsErrors`, the browser pool, the impit client). The old `ignoreSslErrors` name is no longer accepted — rename it in your crawler options. Behavior is unchanged from v3: the option defaults to `true` and HTTP crawlers accept invalid TLS certificates by default.diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 42558e4a7..fa8d46042 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -32,17 +32,7 @@ import {tryAbsoluteURL,validators,} from '@crawlee/basic';-import type {- BrowserController,- BrowserPlugin,- BrowserPoolHooks,- BrowserPoolOptions,- CommonPage,- CrawlerRemoteBrowserOptions,- InferBrowserPluginArray,- LaunchContext,-} from '@crawlee/browser-pool';-import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';+import type { CommonPage, CrawlerRemoteBrowserOptions } from '@crawlee/browser-pool';import type { Awaitable, Cookie as CookieObject, Dictionary, IBrowserPool, ISession } from '@crawlee/types';import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';import { sleep } from '@crawlee/utils';@@ -63,7 +53,26 @@ interface BaseResponse {* additionally exposes `destroy()` — the crawler only ever tears down pools it created, which is why {@apilink IBrowserPool}* itself intentionally omits `destroy`.*/-type OwnedBrowserPool<Page> = IBrowserPool<Page> & { destroy: () => Promise<void> };+export type OwnedBrowserPool<Page> = IBrowserPool<Page> & { destroy: () => Promise<void> };++/**+ * Rejects options that exist only to configure the browser pool the crawler would have built for itself.+ * Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew+ * into a second, half-working way of configuring the same pool.+ */+export function assertBrowserPoolNotConfigured(crawlerName: string, ignoredOptions: Dictionary): void {+ const names = Object.keys(ignoredOptions).filter((name) => ignoredOptions[name] !== undefined);++ if (names.length === 0) {+ return;+ }++ throw new Error(+ `${crawlerName}: ${names.map((name) => `\`${name}\``).join(', ')} cannot be combined with \`browserPool\`, ` ++ `${names.length > 1 ? 'they configure' : 'it configures'} the browser pool the crawler would build for ` ++ 'itself. Configure the pool you pass in instead.',+ );+}type ContextDifference<T, U> = Omit<U, keyof T> & Partial<U>;@@ -119,12 +128,8 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>,ContextExtension = Dictionary<never>,ExtendedContext extends Context = Context & ContextExtension,- InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions,Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,StatisticStateExtension extends object = {},- __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>,- __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>,- __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>,> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>,// Overridden with browser context@@ -133,9 +138,17 @@ export interface BrowserCrawlerOptions<launchContext?: BrowserLaunchContext<any, any>;/**- * An existing browser pool instance to use. When provided, the crawler will use this pool directly instead of- * constructing a new one from `browserPoolOptions`, enabling browser sharing across multiple crawlers. The crawler- * will not tear down a shared pool — the caller is responsible for its lifecycle.+ * The browser pool the crawler should serve its pages from. This is the single way to run a pool with+ * non-default options: build one with the factory that matches your crawler+ * ({@apilink playwrightBrowserPool}, {@apilink puppeteerBrowserPool}, {@apilink stagehandBrowserPool}) - it+ * accepts every {@apilink BrowserPoolOptions|`BrowserPool` option} and supplies the correct browser plugin+ * itself, so the pool can never mismatch the crawler.+ *+ * A pool passed in this way is borrowed, not owned: the crawler will not tear it down, which is what makes it+ * shareable across crawlers. Since the crawler then builds nothing itself, the options that configure its own+ * pool (`launchContext`, `headless`, `remoteBrowser`) are rejected rather than silently ignored.+ *+ * When omitted, the crawler builds - and tears down - a default pool for its own browser.*/browserPool?: IBrowserPool<Page>;@@ -147,8 +160,9 @@ export interface BrowserCrawlerOptions<* crawler. Supply the connection details only: a static `endpoint` URL, a function returning one per launch,* or a {@apilink RemoteBrowserProvider}.*- * Ignored when `browserPool` is set. For sharing a remote pool across crawlers, construct a- * {@apilink RemoteBrowserPool} yourself and pass it as `browserPool` instead.+ * Cannot be combined with `browserPool`. To tune the pool wrapping the remote connection, or to share it+ * across crawlers, build it with the remote factory for your crawler ({@apilink remotePlaywrightBrowserPool},+ * {@apilink remotePuppeteerBrowserPool}, {@apilink remoteStagehandBrowserPool}) and pass it as `browserPool`.*/remoteBrowser?: CrawlerRemoteBrowserOptions;@@ -203,13 +217,6 @@ export interface BrowserCrawlerOptions<*/failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;- /**- * Custom options passed to the underlying {@apilink BrowserPool} constructor.- * We can tweak those to fine-tune browser management.- */- browserPoolOptions?: Partial<BrowserPoolOptions> &- Partial<BrowserPoolHooks<__BrowserControllerReturn, __LaunchContextReturn>>;-/*** Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies* or browser properties before navigation. The function receives the `crawlingContext`; the options object@@ -278,12 +285,6 @@ export interface BrowserCrawlerOptions<*/saveResponseCookies?: boolean;- /**- * Whether to run browser in headless mode. Defaults to `true`.- * Can be also set via {@apilink Configuration}.- */- headless?: boolean | 'new' | 'old'; // `new`/`old` are for puppeteer only-/*** Whether to ignore custom elements (and their #shadow-roots) when processing the page content via `parseWithCheerio` helper.* By default, they are expanded automatically. Use this option to disable this behavior.@@ -354,7 +355,6 @@ function isNavigationTimeoutError(error: Error): boolean {export abstract class BrowserCrawler<Page extends CommonPage = CommonPage,Response extends BaseResponse = BaseResponse,- InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions,LaunchOptions extends Dictionary | undefined = Dictionary,Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>,ContextExtension = Dictionary<never>,@@ -394,10 +394,9 @@ export abstract class BrowserCrawler<postNavigationHooks: schemas.anyArray.default(() => []),launchContext: schemas.anyObject.default(() => ({})),- headless: z.union([z.boolean(), z.string()]).optional(),browserPool: validators.browserPool.optional(),+ browserPoolBuilder: schemas.anyFunction.optional(),remoteBrowser: schemas.anyObject.optional(),- browserPoolOptions: schemas.anyObject.optional(),saveResponseCookies: z.boolean().default(true),proxyConfiguration: validators.proxyConfiguration.optional(),ignoreIframes: z.boolean().default(false),@@ -416,13 +415,16 @@ export abstract class BrowserCrawler<Context,ContextExtension,ExtendedContext,- // Left at the default rather than forwarding `InternalBrowserPoolOptions` - narrowing the pool options- // here would reject subclasses that declare their own options without pinning the pool type.- BrowserPoolOptions,Routes,StatisticStateExtension> & {contextPipelineBuilder: () => ContextPipeline<CrawlingContext, Context>;+ /**+ * Builds the pool the crawler owns, used only when the user injected no `browserPool`. Supplied by the+ * concrete crawler, which is the only place that knows which browser plugin to run - that is also why+ * `remoteBrowser` is handed to it rather than acted upon here.+ */+ browserPoolBuilder: (remoteBrowser?: CrawlerRemoteBrowserOptions) => OwnedBrowserPool<Page>;},) {const {@@ -431,17 +433,20 @@ export abstract class BrowserCrawler<launchContext,browserPool,remoteBrowser,- browserPoolOptions,preNavigationHooks,postNavigationHooks,- headless,ignoreIframes,ignoreShadowRoots,contextPipelineBuilder,+ browserPoolBuilder,extendContext,...basicCrawlerOptions} = parseArgument(options, BrowserCrawler.optionsSchema, 'BrowserCrawlerOptions');+ if (browserPool) {+ assertBrowserPoolNotConfigured(new.target.name, { remoteBrowser });+ }+const skipGuard = <Ctx extends Context>(action: (ctx: Ctx) => Awaitable<void | Partial<Ctx>>,): ContextMiddleware<Ctx, Partial<Ctx>> => ({@@ -502,44 +507,9 @@ export abstract class BrowserCrawler<this.ignoreIframes = ignoreIframes;this.ignoreShadowRoots = ignoreShadowRoots;- if (headless != null) {- this.launchContext.launchOptions ??= {} as LaunchOptions;- (this.launchContext.launchOptions as Dictionary).headless = headless;- }-this.#saveResponseCookies = saveResponseCookies;- // `browserPool` wins over `remoteBrowser` — a passed-in pool is used as-is (borrowed), the sugar is ignored.- // The default is only built when no pool was injected, so all the option/launchContext fiddling below stays- // inside the factory.- this.#browserPoolDep = OwnedOrInjected.resolve(browserPool, () => {- const resolvedBrowserPoolOptions = browserPoolOptions ?? ({} as Partial<BrowserPoolOptions>);-- if (launchContext?.userAgent) {- if (resolvedBrowserPoolOptions.useFingerprints)- this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');- resolvedBrowserPoolOptions.useFingerprints = false;- }-- if (remoteBrowser) {- // The crawler already built the right plugin for its browser — hand it to a RemoteBrowserPool so the- // remote connection is always for the matching browser (no plugin to construct, no way to mismatch).- const { browserPlugins, ...remoteBrowserPoolOptions } = resolvedBrowserPoolOptions;- return new RemoteBrowserPool({- browserPlugins: browserPlugins as BrowserPlugin[],- ...remoteBrowser,- browserPoolOptions: remoteBrowserPoolOptions as any,- }) as OwnedBrowserPool<Page>;- }-- // Double cast: `BrowserPool` implements `IBrowserPool<PageReturn>`, where `PageReturn` is derived from the- // plugin/controller generics and doesn't overlap with the crawler's free `Page` type param, so TS won't- // narrow it directly. The concrete pool does satisfy the `Page`/`destroy` contract at runtime — this is the- // long-standing `Page` variance gap, not a `destroy`-related hole.- return new BrowserPool<InternalBrowserPoolOptions>({- ...(resolvedBrowserPoolOptions as any),- }) as unknown as OwnedBrowserPool<Page>;- });+ this.#browserPoolDep = OwnedOrInjected.resolve(browserPool, () => browserPoolBuilder(remoteBrowser));}protected override getNavigationTimeoutMillis(): number {diff --git a/packages/browser-crawler/src/internals/browser-launcher.ts b/packages/browser-crawler/src/internals/browser-launcher.tsindex 5d7ff98f5..1e52aba5d 100644--- a/packages/browser-crawler/src/internals/browser-launcher.ts+++ b/packages/browser-crawler/src/internals/browser-launcher.ts@@ -2,8 +2,15 @@ import fs from 'node:fs';import { createRequire } from 'node:module';import os from 'node:os';-import { Configuration, schemas } from '@crawlee/basic';-import type { BrowserPlugin, BrowserPluginOptions } from '@crawlee/browser-pool';+import { Configuration, schemas, serviceLocator } from '@crawlee/basic';+import type {+ BrowserPlugin,+ BrowserPluginOptions,+ BrowserPoolHooks,+ BrowserPoolOptions,+ RemoteBrowserPoolOptions,+} from '@crawlee/browser-pool';+import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';import type { Constructor, Dictionary } from '@crawlee/types';import { z } from 'zod';@@ -84,6 +91,21 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginOlauncher?: Launcher;}+/**+ * The {@apilink BrowserPool} options a launcher-built pool accepts: everything the pool itself takes except+ * `browserPlugins`, which the launcher derives from its launch context. The hooks are deliberately unconstrained -+ * the browser they run against is only known to the concrete `*BrowserPool()` factory, which is where the+ * caller-facing types are pinned down.+ */+export type LauncherBrowserPoolOptions = Omit<BrowserPoolOptions, 'browserPlugins'> & {+ [Hook in keyof BrowserPoolHooks<any, any, any>]?: readonly ((...args: any[]) => unknown)[];+};++/**+ * The {@apilink RemoteBrowserPool} counterpart of {@apilink LauncherBrowserPoolOptions}.+ */+export type LauncherRemoteBrowserPoolOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins'>;+/*** Abstract class for creating browser launchers, such as `PlaywrightLauncher` and `PuppeteerLauncher`.* @ignore@@ -173,6 +195,54 @@ export abstract class BrowserLauncher<});}+ /**+ * Builds a {@apilink BrowserPool} running a single plugin for this launcher's browser. Shared body of the+ * per-library `*BrowserPool()` factories, which exist so that configuring a pool never requires assembling+ * a plugin by hand — and therefore never lets the plugin drift away from the crawler it is used with.+ * @internal+ */+ createBrowserPool(options: LauncherBrowserPoolOptions = {}): BrowserPool<{ browserPlugins: [Plugin] }, [Plugin]> {+ // The hook types `BrowserPool` derives from `Plugin` are unresolvable while `Plugin` is still a free type+ // parameter, so the argument cannot be checked here. The concrete `*BrowserPool()` factories are where the+ // caller-facing hook types get pinned down.+ return new BrowserPool<{ browserPlugins: [Plugin] }, [Plugin]>({+ ...this.resolveFingerprinting(options),+ browserPlugins: [this.createBrowserPlugin()],+ } as any);+ }++ /**+ * The {@apilink RemoteBrowserPool} counterpart of {@apilink BrowserLauncher.createBrowserPool}: the launcher+ * supplies the plugin, the caller supplies the remote connection details.+ * @internal+ */+ createRemoteBrowserPool<Page>(options: LauncherRemoteBrowserPoolOptions): RemoteBrowserPool<Page> {+ return new RemoteBrowserPool<Page>({+ ...options,+ browserPlugins: [this.createBrowserPlugin()],+ browserPoolOptions: this.resolveFingerprinting(options.browserPoolOptions ?? {}),+ });+ }++ /**+ * A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an+ * explicitly requested user agent wins.+ */+ private resolveFingerprinting<T extends { useFingerprints?: boolean }>(options: T): T {+ if (!this.userAgent) {+ return options;+ }++ if (options.useFingerprints) {+ serviceLocator+ .getLogger()+ .child({ prefix: 'BrowserLauncher' })+ .info('Custom user agent provided, disabling automatic browser fingerprint injection!');+ }++ return { ...options, useFingerprints: false };+ }+/*** Launches a browser instance based on the plugin.* @returns Browser instance.diff --git a/packages/browser-pool/src/remote-browser-pool.ts b/packages/browser-pool/src/remote-browser-pool.tsindex d07478dac..ca743c8cc 100644--- a/packages/browser-pool/src/remote-browser-pool.ts+++ b/packages/browser-pool/src/remote-browser-pool.ts@@ -159,10 +159,10 @@ export interface RemoteBrowserPoolOptions {}/**- * The remote-connection configuration a browser crawler accepts on its `remoteBrowser` option. It is the- * {@apilink RemoteBrowserPoolOptions} a user supplies *minus* the parts the crawler provides itself — the- * `browserPlugins` (the crawler builds the correct one for its browser) and `browserPoolOptions` (taken from- * the crawler's own `browserPoolOptions`). This is what makes the crawler path both terse and mismatch-proof.+ * The remote-connection configuration a browser crawler accepts on its `remoteBrowser` option: the+ * {@apilink RemoteBrowserPoolOptions} minus the `browserPlugins` (the crawler builds the correct one for its+ * browser, which is what makes this path mismatch-proof) and minus `browserPoolOptions` — tuning the wrapping+ * pool means building the pool yourself, through the `remote*BrowserPool()` factory for your crawler.*/export type CrawlerRemoteBrowserOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins' | 'browserPoolOptions'>;diff --git a/packages/playwright-crawler/src/index.ts b/packages/playwright-crawler/src/index.tsindex 9607b713b..d2643a3bf 100644--- a/packages/playwright-crawler/src/index.ts+++ b/packages/playwright-crawler/src/index.ts@@ -1,4 +1,5 @@export * from '@crawlee/browser';+export * from './internals/playwright-browser-pool.js';export * from './internals/playwright-crawler.js';export * from './internals/playwright-launcher.js';export * from './internals/adaptive-playwright-crawler.js';diff --git a/packages/playwright-crawler/src/internals/playwright-browser-pool.ts b/packages/playwright-crawler/src/internals/playwright-browser-pool.tsnew file mode 100644index 000000000..8c6a781e3--- /dev/null+++ b/packages/playwright-crawler/src/internals/playwright-browser-pool.ts@@ -0,0 +1,114 @@+import type { Configuration } from '@crawlee/browser';+import type {+ BrowserPool,+ BrowserPoolHooks,+ BrowserPoolOptions,+ PlaywrightPlugin,+ RemoteBrowserPool,+ RemoteBrowserPoolOptions,+} from '@crawlee/browser-pool';+import type { Page } from 'playwright';++import type { PlaywrightLaunchContext } from './playwright-launcher.js';+import { PlaywrightLauncher } from './playwright-launcher.js';++/** A {@apilink BrowserPool} of Playwright browsers, as built by {@apilink playwrightBrowserPool}. */+export type PlaywrightBrowserPool = BrowserPool<{ browserPlugins: [PlaywrightPlugin] }, [PlaywrightPlugin]>;++export interface PlaywrightBrowserPoolOptions+ extends+ Omit<BrowserPoolOptions, 'browserPlugins'>,+ BrowserPoolHooks<+ ReturnType<PlaywrightPlugin['createController']>,+ ReturnType<PlaywrightPlugin['createLaunchContext']>,+ Page+ > {+ /** How to launch the browser: which Playwright browser type, proxy, user data dir, ... */+ launchContext?: PlaywrightLaunchContext;++ /**+ * Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.+ * Defaults to `true`, and can also be set via {@apilink Configuration}.+ */+ headless?: boolean;++ /** Configuration to read the browser defaults from. Defaults to the global configuration. */+ configuration?: Configuration;+}++export interface RemotePlaywrightBrowserPoolOptions+ extends+ Pick<PlaywrightBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>,+ Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {}++/**+ * Builds a {@apilink BrowserPool} of Playwright browsers to pass to a {@apilink PlaywrightCrawler} as its+ * {@apilink BrowserCrawlerOptions.browserPool|`browserPool`}.+ *+ * It accepts every {@apilink BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is+ * given to, and configuring one never means assembling a {@apilink PlaywrightPlugin} by hand.+ *+ * **Example usage:**+ *+ * ```javascript+ * const crawler = new PlaywrightCrawler({+ * browserPool: playwrightBrowserPool({+ * maxOpenPagesPerBrowser: 1,+ * launchContext: { launcher: firefox },+ * }),+ * requestHandler: async ({ page }) => { ... },+ * });+ * ```+ *+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.+ *+ * @category Browser management+ */+export function playwrightBrowserPool(options: PlaywrightBrowserPoolOptions = {}): PlaywrightBrowserPool {+ const { launchContext, headless, configuration, ...poolOptions } = options;++ return playwrightLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);+}++/**+ * The {@apilink RemoteBrowserPool} counterpart of {@apilink playwrightBrowserPool}: connects to a remote browser+ * service (Browserbase, Browserless, Steel, ...) with a Playwright plugin derived from `launchContext`.+ *+ * A {@apilink PlaywrightCrawler} accepts the same connection details directly via+ * {@apilink BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to+ * tune the wrapping pool, or to share one remote pool between crawlers.+ *+ * **Example usage:**+ *+ * ```javascript+ * const crawler = new PlaywrightCrawler({+ * browserPool: remotePlaywrightBrowserPool({+ * endpoint: 'wss://production-sfo.browserless.io?token=xxx',+ * maxOpenBrowsers: 2,+ * browserPoolOptions: { useFingerprints: false },+ * }),+ * requestHandler: async ({ page }) => { ... },+ * });+ * ```+ *+ * @category Browser management+ */+export function remotePlaywrightBrowserPool(options: RemotePlaywrightBrowserPoolOptions): RemoteBrowserPool<Page> {+ const { launchContext, headless, configuration, ...remoteOptions } = options;++ return playwrightLauncher(launchContext, headless, configuration).createRemoteBrowserPool<Page>(remoteOptions);+}++function playwrightLauncher(+ launchContext: PlaywrightLaunchContext = {},+ headless?: boolean,+ configuration?: Configuration,+): PlaywrightLauncher {+ return new PlaywrightLauncher(+ headless == null+ ? launchContext+ : { ...launchContext, launchOptions: { ...launchContext.launchOptions, headless } },+ configuration,+ );+}diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex eaf11d867..e38493ed3 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -9,15 +9,22 @@ import type {RouteSchemas,RoutesFromSchemas,} from '@crawlee/browser';-import { BrowserCrawler, parseArgument, RequestState, Router, schemas, serviceLocator } from '@crawlee/browser';-import type { BrowserPoolOptions, PlaywrightPlugin } from '@crawlee/browser-pool';+import {+ assertBrowserPoolNotConfigured,+ BrowserCrawler,+ parseArgument,+ RequestState,+ Router,+ schemas,+ serviceLocator,+} from '@crawlee/browser';import type { Dictionary } from '@crawlee/types';import type { Download, LaunchOptions, Page, Response } from 'playwright';import { z } from 'zod';import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js';+import { playwrightBrowserPool, remotePlaywrightBrowserPool } from './playwright-browser-pool.js';import type { PlaywrightLaunchContext } from './playwright-launcher.js';-import { PlaywrightLauncher } from './playwright-launcher.js';import type {BlockRequestsOptions,DirectNavigationOptions,@@ -53,7 +60,6 @@ export interface PlaywrightCrawlerOptions<PlaywrightCrawlingContext,ContextExtension,ExtendedContext,- { browserPlugins: [PlaywrightPlugin] },Routes,StatisticStateExtension> {@@ -62,6 +68,12 @@ export interface PlaywrightCrawlerOptions<*/launchContext?: PlaywrightLaunchContext;+ /**+ * Whether to run browser in headless mode. Defaults to `true`.+ * Can be also set via {@apilink Configuration}.+ */+ headless?: boolean;+/*** Function that is called to process each request.*@@ -205,7 +217,6 @@ export class PlaywrightCrawler<> extends BrowserCrawler<Page,Response,- { browserPlugins: [PlaywrightPlugin] },LaunchOptions,PlaywrightCrawlingContext,ContextExtension,@@ -215,7 +226,7 @@ export class PlaywrightCrawler<> {protected static override optionsShape = {...BrowserCrawler.optionsShape,- browserPoolOptions: schemas.anyObject.optional(),+ headless: z.boolean().optional(),launcher: schemas.anyObject.optional(),};@@ -229,11 +240,8 @@ export class PlaywrightCrawler<) {const parsedOptions = parseArgument(options, PlaywrightCrawler.optionsSchema, 'PlaywrightCrawlerOptions');- const { launchContext, headless, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;-- const browserPoolOptions = {- ...parsedOptions.browserPoolOptions,- } as BrowserPoolOptions;+ const { launchContext, headless, configuration, contextPipelineBuilder, ...browserCrawlerOptions } =+ parsedOptions;if (launchContext.proxyUrl) {throw new Error(@@ -242,21 +250,14 @@ export class PlaywrightCrawler<);}- // `browserPlugins` is working when it's not overridden by `launchContext`,- // which for crawlers it is always overridden. Hence the error to use the other option.- if (browserPoolOptions.browserPlugins) {- throw new Error('browserPoolOptions.browserPlugins is disallowed. Use launchContext.launcher instead.');+ if (options.browserPool) {+ // The raw options, not the parsed ones: `launchContext` has a default, so by now it is always set.+ assertBrowserPoolNotConfigured(new.target.name, {+ launchContext: options.launchContext,+ headless: options.headless,+ });}- if (headless != null) {- launchContext.launchOptions ??= {} as LaunchOptions;- launchContext.launchOptions.headless = headless as boolean;- }-- const playwrightLauncher = new PlaywrightLauncher(launchContext, parsedOptions.configuration);-- browserPoolOptions.browserPlugins = [playwrightLauncher.createBrowserPlugin()];-super({...(browserCrawlerOptions as unknown as PlaywrightCrawlerOptions<ContextExtension,@@ -265,7 +266,11 @@ export class PlaywrightCrawler<StatisticStateExtension>),launchContext,- browserPoolOptions,+ configuration,+ browserPoolBuilder: (remoteBrowser) =>+ remoteBrowser+ ? remotePlaywrightBrowserPool({ ...remoteBrowser, launchContext, headless, configuration })+ : playwrightBrowserPool({ launchContext, headless, configuration }),contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),});}diff --git a/packages/puppeteer-crawler/src/index.ts b/packages/puppeteer-crawler/src/index.tsindex 4d84972ba..99c5a18ff 100644--- a/packages/puppeteer-crawler/src/index.ts+++ b/packages/puppeteer-crawler/src/index.ts@@ -1,4 +1,5 @@export * from '@crawlee/browser';+export * from './internals/puppeteer-browser-pool.js';export * from './internals/puppeteer-crawler.js';export * from './internals/puppeteer-launcher.js';diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-browser-pool.ts b/packages/puppeteer-crawler/src/internals/puppeteer-browser-pool.tsnew file mode 100644index 000000000..58efd0ae8--- /dev/null+++ b/packages/puppeteer-crawler/src/internals/puppeteer-browser-pool.ts@@ -0,0 +1,102 @@+import type { Configuration } from '@crawlee/browser';+import type {+ BrowserPool,+ BrowserPoolHooks,+ BrowserPoolOptions,+ PuppeteerPlugin,+ RemoteBrowserPool,+ RemoteBrowserPoolOptions,+} from '@crawlee/browser-pool';+// @ts-ignore This only throws when compiled against puppeteer 25+ (ESM only), we only import types, so its alllll gooooood+import type { Page } from 'puppeteer';++import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';+import { PuppeteerLauncher } from './puppeteer-launcher.js';++/** A {@apilink BrowserPool} of Puppeteer browsers, as built by {@apilink puppeteerBrowserPool}. */+export type PuppeteerBrowserPool = BrowserPool<{ browserPlugins: [PuppeteerPlugin] }, [PuppeteerPlugin]>;++export interface PuppeteerBrowserPoolOptions+ extends+ Omit<BrowserPoolOptions, 'browserPlugins'>,+ BrowserPoolHooks<+ ReturnType<PuppeteerPlugin['createController']>,+ ReturnType<PuppeteerPlugin['createLaunchContext']>,+ Page+ > {+ /** How to launch the browser: proxy, user data dir, whether to use full Chrome, ... */+ launchContext?: PuppeteerLaunchContext;++ /**+ * Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.+ * Defaults to `true`, and can also be set via {@apilink Configuration}.+ */+ headless?: boolean | 'new' | 'old';++ /** Configuration to read the browser defaults from. Defaults to the global configuration. */+ configuration?: Configuration;+}++export interface RemotePuppeteerBrowserPoolOptions+ extends+ Pick<PuppeteerBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>,+ Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {}++/**+ * Builds a {@apilink BrowserPool} of Puppeteer browsers to pass to a {@apilink PuppeteerCrawler} as its+ * {@apilink BrowserCrawlerOptions.browserPool|`browserPool`}.+ *+ * It accepts every {@apilink BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is+ * given to, and configuring one never means assembling a {@apilink PuppeteerPlugin} by hand.+ *+ * **Example usage:**+ *+ * ```javascript+ * const crawler = new PuppeteerCrawler({+ * browserPool: puppeteerBrowserPool({ maxOpenPagesPerBrowser: 1 }),+ * requestHandler: async ({ page }) => { ... },+ * });+ * ```+ *+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.+ *+ * @category Browser management+ */+export function puppeteerBrowserPool(options: PuppeteerBrowserPoolOptions = {}): PuppeteerBrowserPool {+ const { launchContext, headless, configuration, ...poolOptions } = options;++ return puppeteerLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);+}++/**+ * The {@apilink RemoteBrowserPool} counterpart of {@apilink puppeteerBrowserPool}: connects to a remote browser+ * service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.+ *+ * A {@apilink PuppeteerCrawler} accepts the same connection details directly via+ * {@apilink BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to+ * tune the wrapping pool, or to share one remote pool between crawlers.+ *+ * @category Browser management+ */+export function remotePuppeteerBrowserPool(options: RemotePuppeteerBrowserPoolOptions): RemoteBrowserPool<Page> {+ const { launchContext, headless, configuration, ...remoteOptions } = options;++ return puppeteerLauncher(launchContext, headless, configuration).createRemoteBrowserPool<Page>(remoteOptions);+}++function puppeteerLauncher(+ launchContext: PuppeteerLaunchContext = {},+ headless?: boolean | 'new' | 'old',+ configuration?: Configuration,+): PuppeteerLauncher {+ return new PuppeteerLauncher(+ headless == null+ ? launchContext+ : {+ ...launchContext,+ launchOptions: { ...launchContext.launchOptions, headless: headless as boolean },+ },+ configuration,+ );+}diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex 16fb43bdd..9ac31a105 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -8,17 +8,16 @@ import type {RouteSchemas,RoutesFromSchemas,} from '@crawlee/browser';-import { BrowserCrawler, RequestState, Router } from '@crawlee/browser';-import type { BrowserPoolOptions, PuppeteerPlugin } from '@crawlee/browser-pool';-import { parseArgument, schemas, serviceLocator } from '@crawlee/core';+import { assertBrowserPoolNotConfigured, BrowserCrawler, RequestState, Router } from '@crawlee/browser';+import { parseArgument, serviceLocator } from '@crawlee/core';import type { Dictionary } from '@crawlee/types';// @ts-ignore This only throws when compiled against puppeteer 25+ (ESM only), we only import types, so its alllll goooooodimport type { HTTPResponse, LaunchOptions, Page } from 'puppeteer';import { z } from 'zod';import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js';+import { puppeteerBrowserPool, remotePuppeteerBrowserPool } from './puppeteer-browser-pool.js';import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';-import { PuppeteerLauncher } from './puppeteer-launcher.js';import type { InterceptHandler } from './utils/puppeteer_request_interception.js';import type {BlockRequestsOptions,@@ -54,7 +53,6 @@ export interface PuppeteerCrawlerOptions<PuppeteerCrawlingContext,ContextExtension,ExtendedContext,- { browserPlugins: [PuppeteerPlugin] },Routes,StatisticStateExtension> {@@ -63,6 +61,12 @@ export interface PuppeteerCrawlerOptions<*/launchContext?: PuppeteerLaunchContext;+ /**+ * Whether to run browser in headless mode. Defaults to `true`.+ * Can be also set via {@apilink Configuration}.+ */+ headless?: boolean | 'new' | 'old';+/*** Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies* or browser properties before navigation. The function receives the `crawlingContext`; the options object@@ -182,7 +186,6 @@ export class PuppeteerCrawler<> extends BrowserCrawler<Page,HTTPResponse,- { browserPlugins: [PuppeteerPlugin] },LaunchOptions,PuppeteerCrawlingContext,ContextExtension,@@ -192,7 +195,9 @@ export class PuppeteerCrawler<> {protected static override optionsShape = {...BrowserCrawler.optionsShape,- browserPoolOptions: schemas.anyObject.optional(),+ // Deliberately looser than the declared type: Puppeteer's own accepted string values have moved over+ // time (`'new'`/`'old'`, now `'shell'`), and the value is forwarded to it verbatim.+ headless: z.union([z.boolean(), z.string()]).optional(),};protected static override optionsSchema = z.strictObject(PuppeteerCrawler.optionsShape);@@ -205,12 +210,14 @@ export class PuppeteerCrawler<) {const parsedOptions = parseArgument(options, PuppeteerCrawler.optionsSchema, 'PuppeteerCrawlerOptions');- const { launchContext, headless, proxyConfiguration, contextPipelineBuilder, ...browserCrawlerOptions } =- parsedOptions;-- const browserPoolOptions = {- ...parsedOptions.browserPoolOptions,- } as BrowserPoolOptions;+ const {+ launchContext,+ headless,+ configuration,+ proxyConfiguration,+ contextPipelineBuilder,+ ...browserCrawlerOptions+ } = parsedOptions;if (launchContext.proxyUrl) {throw new Error(@@ -219,21 +226,14 @@ export class PuppeteerCrawler<);}- // `browserPlugins` is working when it's not overridden by `launchContext`,- // which for crawlers it is always overridden. Hence the error to use the other option.- if (browserPoolOptions.browserPlugins) {- throw new Error('browserPoolOptions.browserPlugins is disallowed. Use launchContext.launcher instead.');- }-- if (headless != null) {- launchContext.launchOptions ??= {} as LaunchOptions;- launchContext.launchOptions.headless = headless as boolean;+ if (options.browserPool) {+ // The raw options, not the parsed ones: `launchContext` has a default, so by now it is always set.+ assertBrowserPoolNotConfigured(new.target.name, {+ launchContext: options.launchContext,+ headless: options.headless,+ });}- const puppeteerLauncher = new PuppeteerLauncher(launchContext, parsedOptions.configuration);-- browserPoolOptions.browserPlugins = [puppeteerLauncher.createBrowserPlugin()];-super({...(browserCrawlerOptions as BrowserCrawlerOptions<Page,@@ -241,13 +241,16 @@ export class PuppeteerCrawler<PuppeteerCrawlingContext,ContextExtension,ExtendedContext,- BrowserPoolOptions,Routes,StatisticStateExtension>),launchContext,+ configuration,proxyConfiguration,- browserPoolOptions,+ browserPoolBuilder: (remoteBrowser) =>+ remoteBrowser+ ? remotePuppeteerBrowserPool({ ...remoteBrowser, launchContext, headless, configuration })+ : puppeteerBrowserPool({ launchContext, headless, configuration }),contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),});}diff --git a/packages/stagehand-crawler/src/index.ts b/packages/stagehand-crawler/src/index.tsindex 53cb7e3e6..184441dbd 100644--- a/packages/stagehand-crawler/src/index.ts+++ b/packages/stagehand-crawler/src/index.ts@@ -56,6 +56,14 @@ export * from '@crawlee/browser';// Export main crawler classexport { StagehandCrawler, createStagehandRouter } from './internals/stagehand-crawler';+// Export the browser pool factories, which are how a pool with non-default options reaches the crawler+export { stagehandBrowserPool, remoteStagehandBrowserPool } from './internals/stagehand-browser-pool';+export type {+ StagehandBrowserPool,+ StagehandBrowserPoolOptions,+ RemoteStagehandBrowserPoolOptions,+} from './internals/stagehand-browser-pool';+// Export typesexport type {StagehandOptions,diff --git a/packages/stagehand-crawler/src/internals/stagehand-browser-pool.ts b/packages/stagehand-crawler/src/internals/stagehand-browser-pool.tsnew file mode 100644index 000000000..8e92a8245--- /dev/null+++ b/packages/stagehand-crawler/src/internals/stagehand-browser-pool.ts@@ -0,0 +1,111 @@+import type { Configuration } from '@crawlee/browser';+import type {+ BrowserPool,+ BrowserPoolHooks,+ BrowserPoolOptions,+ RemoteBrowserPool,+ RemoteBrowserPoolOptions,+} from '@crawlee/browser-pool';+import type { Page } from 'playwright';++import type { StagehandOptions } from './stagehand-crawler';+import type { StagehandLaunchContext } from './stagehand-launcher';+import { StagehandLauncher } from './stagehand-launcher';+import type { StagehandPlugin } from './stagehand-plugin';++/** A {@apilink BrowserPool} of Stagehand browsers, as built by {@apilink stagehandBrowserPool}. */+export type StagehandBrowserPool = BrowserPool<{ browserPlugins: [StagehandPlugin] }, [StagehandPlugin]>;++export interface StagehandBrowserPoolOptions+ extends+ Omit<BrowserPoolOptions, 'browserPlugins'>,+ BrowserPoolHooks<+ ReturnType<StagehandPlugin['createController']>,+ ReturnType<StagehandPlugin['createLaunchContext']>,+ Page+ > {+ /** How to launch the browser: which Playwright browser type, proxy, user data dir, ... */+ launchContext?: StagehandLaunchContext;++ /** Stagehand configuration for the AI behavior and Browserbase integration. */+ stagehandOptions?: StagehandOptions;++ /**+ * Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.+ * Defaults to `true`, and can also be set via {@apilink Configuration}.+ */+ headless?: boolean;++ /** Configuration to read the browser defaults from. Defaults to the global configuration. */+ configuration?: Configuration;+}++export interface RemoteStagehandBrowserPoolOptions+ extends+ Pick<StagehandBrowserPoolOptions, 'launchContext' | 'stagehandOptions' | 'headless' | 'configuration'>,+ Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {}++/**+ * Builds a {@apilink BrowserPool} of Stagehand browsers to pass to a {@apilink StagehandCrawler} as its+ * {@apilink BrowserCrawlerOptions.browserPool|`browserPool`}.+ *+ * It accepts every {@apilink BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext`,+ * `stagehandOptions` and `headless`, and derives the browser plugin from them - so a pool built here always+ * matches the crawler it is given to, and configuring one never means assembling a {@apilink StagehandPlugin}+ * by hand.+ *+ * **Example usage:**+ *+ * ```javascript+ * const crawler = new StagehandCrawler({+ * browserPool: stagehandBrowserPool({+ * maxOpenPagesPerBrowser: 1,+ * stagehandOptions: { env: 'LOCAL', model: 'openai/gpt-4.1-mini' },+ * }),+ * requestHandler: async ({ page }) => { ... },+ * });+ * ```+ *+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.+ *+ * @category Browser management+ */+export function stagehandBrowserPool(options: StagehandBrowserPoolOptions = {}): StagehandBrowserPool {+ const { launchContext, stagehandOptions, headless, configuration, ...poolOptions } = options;++ return stagehandLauncher(launchContext, stagehandOptions, headless, configuration).createBrowserPool(poolOptions);+}++/**+ * The {@apilink RemoteBrowserPool} counterpart of {@apilink stagehandBrowserPool}: connects to a remote browser+ * service with a Stagehand plugin derived from `launchContext` and `stagehandOptions`.+ *+ * A {@apilink StagehandCrawler} accepts the same connection details directly via+ * {@apilink BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to+ * tune the wrapping pool, or to share one remote pool between crawlers.+ *+ * @category Browser management+ */+export function remoteStagehandBrowserPool(options: RemoteStagehandBrowserPoolOptions): RemoteBrowserPool<Page> {+ const { launchContext, stagehandOptions, headless, configuration, ...remoteOptions } = options;++ return stagehandLauncher(launchContext, stagehandOptions, headless, configuration).createRemoteBrowserPool<Page>(+ remoteOptions,+ );+}++function stagehandLauncher(+ launchContext: StagehandLaunchContext = {},+ stagehandOptions: StagehandOptions = {},+ headless?: boolean,+ configuration?: Configuration,+): StagehandLauncher {+ return new StagehandLauncher(+ {+ ...launchContext,+ stagehandOptions,+ ...(headless == null ? {} : { launchOptions: { ...launchContext.launchOptions, headless } }),+ },+ configuration,+ );+}diff --git a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts b/packages/stagehand-crawler/src/internals/stagehand-crawler.tsindex 5a26e61a1..62d67b787 100644--- a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts+++ b/packages/stagehand-crawler/src/internals/stagehand-crawler.ts@@ -19,22 +19,21 @@ import type {CrawlingContext,GetUserDataFromRequest,LoadedContext,+ OwnedBrowserPool,RequestHandler,RouterHandler,RouterRoutes,RouteSchemas,RoutesFromSchemas,} from '@crawlee/browser';-import { BrowserCrawler, parseArgument, Router, schemas } from '@crawlee/browser';-import type { BrowserPoolOptions } from '@crawlee/browser-pool';+import { assertBrowserPoolNotConfigured, BrowserCrawler, parseArgument, Router, schemas } from '@crawlee/browser';import type { Dictionary } from '@crawlee/types';import type { LaunchOptions, Page, Response } from 'playwright';import { z } from 'zod';+import { remoteStagehandBrowserPool, stagehandBrowserPool } from './stagehand-browser-pool';import type { StagehandController } from './stagehand-controller';import type { StagehandLaunchContext } from './stagehand-launcher';-import { StagehandLauncher } from './stagehand-launcher';-import type { StagehandPlugin } from './stagehand-plugin';import { enhancePageWithStagehand } from './utils/stagehand-utils';/**@@ -259,7 +258,6 @@ export interface StagehandCrawlerOptions<StagehandCrawlingContext,ContextExtension,ExtendedContext,- { browserPlugins: [StagehandPlugin] },Routes,StatisticStateExtension> {@@ -274,6 +272,12 @@ export interface StagehandCrawlerOptions<*/launchContext?: StagehandLaunchContext;+ /**+ * Whether to run browser in headless mode. Defaults to `true`.+ * Can be also set via {@apilink Configuration}.+ */+ headless?: boolean;+/*** Function that is called to process each request.*@@ -401,7 +405,6 @@ export class StagehandCrawler<> extends BrowserCrawler<StagehandPage,Response,- { browserPlugins: [StagehandPlugin] },LaunchOptions,StagehandCrawlingContext,ContextExtension,@@ -412,7 +415,7 @@ export class StagehandCrawler<protected static override optionsShape = {...BrowserCrawler.optionsShape,stagehandOptions: schemas.anyObject.optional(),- browserPoolOptions: schemas.anyObject.optional(),+ headless: z.boolean().optional(),};protected static override optionsSchema = z.strictObject(StagehandCrawler.optionsShape);@@ -428,25 +431,23 @@ export class StagehandCrawler<const parsedOptions = parseArgument(options, StagehandCrawler.optionsSchema, 'StagehandCrawlerOptions');const {- stagehandOptions = {},+ stagehandOptions,launchContext,+ headless,+ configuration,contextPipelineBuilder,...browserCrawlerOptions} = parsedOptions;- const browserPoolOptions = {- ...parsedOptions.browserPoolOptions,- } as BrowserPoolOptions;-- // Create launcher with Stagehand plugin- const launcher = new StagehandLauncher({- ...launchContext,- stagehandOptions,- });-- browserPoolOptions.browserPlugins = [launcher.createBrowserPlugin()];+ if (options.browserPool) {+ // The raw options, not the parsed ones: `launchContext` has a default, so by now it is always set.+ assertBrowserPoolNotConfigured(new.target.name, {+ launchContext: options.launchContext,+ stagehandOptions: options.stagehandOptions,+ headless: options.headless,+ });+ }- // Initialize BrowserCrawler with Stagehand plugin and fingerprinting enabledsuper({...(browserCrawlerOptions as StagehandCrawlerOptions<ContextExtension,@@ -455,7 +456,24 @@ export class StagehandCrawler<StatisticStateExtension>),launchContext,- browserPoolOptions,+ configuration,+ // The pool serves plain Playwright pages - a page only becomes a `StagehandPage` further down the+ // pipeline, once `setUpStagehand` enhances it - so its page type is narrower than the crawler's.+ browserPoolBuilder: (remoteBrowser) =>+ (remoteBrowser+ ? remoteStagehandBrowserPool({+ ...remoteBrowser,+ launchContext,+ stagehandOptions,+ headless,+ configuration,+ })+ : stagehandBrowserPool({+ launchContext,+ stagehandOptions,+ headless,+ configuration,+ })) as unknown as OwnedBrowserPool<StagehandPage>,contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),});}diff --git a/packages/templates/templates/camoufox-ts/src/main.ts b/packages/templates/templates/camoufox-ts/src/main.tsindex 8d29644ff..34f06fb8d 100644--- a/packages/templates/templates/camoufox-ts/src/main.ts+++ b/packages/templates/templates/camoufox-ts/src/main.ts@@ -1,7 +1,7 @@// For more information, see https://crawlee.dev/import { Browser, ImpitHttpClient } from '@crawlee/impit-client';import { launchOptions } from 'camoufox-js';-import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';+import { PlaywrightCrawler, playwrightBrowserPool, ProxyConfiguration } from 'crawlee';import { firefox } from 'playwright';import { router } from './routes.js';@@ -14,20 +14,20 @@ const crawler = new PlaywrightCrawler({requestHandler: router,// Comment this option to scrape the full website.maxRequestsPerCrawl: 20,- browserPoolOptions: {+ browserPool: playwrightBrowserPool({// Disable the default fingerprint spoofing to avoid conflicts with Camoufox.useFingerprints: false,- },- launchContext: {- launcher: firefox,- launchOptions: await launchOptions({- headless: false,- // Pass your own Camoufox parameters here...- // block_images: true,- // fonts: ['Times New Roman'],- // ...- }),- },+ launchContext: {+ launcher: firefox,+ launchOptions: await launchOptions({+ headless: false,+ // Pass your own Camoufox parameters here...+ // block_images: true,+ // fonts: ['Times New Roman'],+ // ...+ }),+ },+ }),});await crawler.run(startUrls);diff --git a/test/core/crawlers/basic_browser_crawler.ts b/test/core/crawlers/basic_browser_crawler.tsindex 0dc756f41..41ddf3827 100644--- a/test/core/crawlers/basic_browser_crawler.ts+++ b/test/core/crawlers/basic_browser_crawler.ts@@ -1,4 +1,5 @@-import type { PuppeteerPlugin } from '@crawlee/browser-pool';+import type { BrowserPlugin, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin } from '@crawlee/browser-pool';+import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';import type {BrowserCrawlerOptions,BrowserCrawlingContext,@@ -12,16 +13,34 @@ import type { HTTPResponse, LaunchOptions, Page } from 'puppeteer';export type TestCrawlingContext = BrowserCrawlingContext<Page, HTTPResponse, Dictionary>;-export class BrowserCrawlerTest extends BrowserCrawler<- Page,- HTTPResponse,- { browserPlugins: [PuppeteerPlugin] },- LaunchOptions,- TestCrawlingContext-> {- constructor(options: Partial<BrowserCrawlerOptions<Page, HTTPResponse, TestCrawlingContext>> = {}) {+type TestBrowserPoolOptions = BrowserPoolOptions<PuppeteerPlugin> &+ BrowserPoolHooks<+ ReturnType<PuppeteerPlugin['createController']>,+ ReturnType<PuppeteerPlugin['createLaunchContext']>,+ Page+ >;++export class BrowserCrawlerTest extends BrowserCrawler<Page, HTTPResponse, LaunchOptions, TestCrawlingContext> {+ constructor(+ options: Partial<BrowserCrawlerOptions<Page, HTTPResponse, TestCrawlingContext>> & {+ /**+ * The concrete crawlers derive their plugin from a `launchContext`; this bare test subclass has no+ * launcher, so tests hand it the pool options - `browserPlugins` included - directly.+ */+ browserPoolOptions?: TestBrowserPoolOptions;+ } = {},+ ) {+ const { browserPoolOptions, ...browserCrawlerOptions } = options;+super({- ...options,+ ...browserCrawlerOptions,+ browserPoolBuilder: (remoteBrowser) =>+ remoteBrowser+ ? new RemoteBrowserPool<Page>({+ ...remoteBrowser,+ browserPlugins: browserPoolOptions!.browserPlugins as unknown as BrowserPlugin[],+ })+ : new BrowserPool(browserPoolOptions!),contextPipelineBuilder: () => this.buildContextPipeline(),});}diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.tsindex 462ba9bbb..3445eac20 100644--- a/test/core/crawlers/browser_crawler.test.ts+++ b/test/core/crawlers/browser_crawler.test.ts@@ -191,17 +191,18 @@ describe('BrowserCrawler', () => {await (crawler.browserPool as RemoteBrowserPool).destroy();});- test.concurrent('uses browserPool and ignores remoteBrowser when both are set', async () => {+ test.concurrent('rejects remoteBrowser when a browserPool is passed in', async () => {const externalPool = new BrowserPoolClass({ browserPlugins: [new PuppeteerPlugin(puppeteer)] });try {- const crawler = new BrowserCrawlerTest({- browserPool: externalPool,- remoteBrowser: { endpoint: 'ws://remote:9222' },- requestHandler: async () => {},- });-- expect(crawler.browserPool).toBe(externalPool);+ expect(+ () =>+ new BrowserCrawlerTest({+ browserPool: externalPool,+ remoteBrowser: { endpoint: 'ws://remote:9222' },+ requestHandler: async () => {},+ }),+ ).toThrow('`remoteBrowser` cannot be combined with `browserPool`');} finally {await externalPool.destroy();}diff --git a/test/core/crawlers/playwright_crawler.test.ts b/test/core/crawlers/playwright_crawler.test.tsindex 3b3ecabc6..109b21c9d 100644--- a/test/core/crawlers/playwright_crawler.test.ts+++ b/test/core/crawlers/playwright_crawler.test.ts@@ -4,7 +4,13 @@ import os from 'node:os';import type { PlaywrightCrawlingContext, PlaywrightGotoOptions, Request } from '@crawlee/playwright';import { type ConcurrencySystem, MemoryStorageBackend, serviceLocator } from '@crawlee/core';-import { createPlaywrightRouter, PlaywrightCrawler, RequestList, RequestValidationError } from '@crawlee/playwright';+import {+ createPlaywrightRouter,+ PlaywrightCrawler,+ playwrightBrowserPool,+ RequestList,+ RequestValidationError,+} from '@crawlee/playwright';import type { Cheerio, CheerioAPI, CheerioRoot, Element } from '@crawlee/utils/internal';import { sleep } from '@crawlee/utils';import express from 'express';@@ -101,10 +107,10 @@ describe('PlaywrightCrawler', () => {};const playwrightCrawler = new PlaywrightCrawler({- launchContext: {- launcher: playwright[browser],- },- browserPoolOptions: { useFingerprints: false },+ browserPool: playwrightBrowserPool({+ launchContext: { launcher: playwright[browser] },+ useFingerprints: false,+ }),requestList: requestListLarge,minConcurrency: 1,maxConcurrency: 1,@@ -134,9 +140,7 @@ describe('PlaywrightCrawler', () => {const crawler = new PlaywrightCrawler({maxRequestRetries: 0,- browserPoolOptions: {- operationTimeoutSecs: 0.001,- },+ browserPool: playwrightBrowserPool({ operationTimeoutSecs: 0.001 }),requestHandler: async ({ request }) => {success.push(request.url);},@@ -182,16 +186,49 @@ describe('PlaywrightCrawler', () => {expect(options!.timeout).toEqual(timeoutSecs * 1000);});- test('shallow clones browserPoolOptions before normalization', () => {+ test('does not mutate the launchContext it was given', () => {const options = {- browserPoolOptions: {},+ launchContext: {},+ headless: false,requestHandler: async () => {},};- void new PlaywrightCrawler(options);void new PlaywrightCrawler(options);- expect(Object.keys(options.browserPoolOptions).length).toBe(0);+ expect(options.launchContext).toEqual({});+ });++ describe('playwrightBrowserPool', () => {+ test('runs a plugin for the requested browser and forwards the pool options', () => {+ const browserPool = playwrightBrowserPool({+ maxOpenPagesPerBrowser: 3,+ headless: false,+ launchContext: { launcher: playwright.firefox },+ });++ expect(browserPool.maxOpenPagesPerBrowser).toBe(3);+ expect(browserPool.browserPlugins).toHaveLength(1);+ expect(browserPool.browserPlugins[0].library).toBe(playwright.firefox);+ expect(browserPool.browserPlugins[0].launchOptions).toMatchObject({ headless: false });+ });++ test('turns off fingerprint injection when a custom userAgent is given', () => {+ expect(+ playwrightBrowserPool({ launchContext: { userAgent: 'Definitely Not A Crawler' } }).useFingerprints,+ ).toBe(false);+ });++ test('is rejected by the crawler alongside the options that would configure its own pool', () => {+ const browserPool = playwrightBrowserPool();++ expect(() => new PlaywrightCrawler({ browserPool, headless: false })).toThrow(+ 'PlaywrightCrawler: `headless` cannot be combined with `browserPool`',+ );+ expect(+ () => new PlaywrightCrawler({ browserPool, launchContext: { launcher: playwright.firefox } }),+ ).toThrow('PlaywrightCrawler: `launchContext` cannot be combined with `browserPool`');+ expect(() => new PlaywrightCrawler({ browserPool })).not.toThrow();+ });});test.each([{ useIncognitoPages: true }, { useIncognitoPages: false }])(@@ -209,14 +246,11 @@ describe('PlaywrightCrawler', () => {const playwrightCrawler = new PlaywrightCrawler({maxConcurrency: 1,- launchContext: {- useIncognitoPages,- launchOptions,- },- browserPoolOptions: {+ browserPool: playwrightBrowserPool({+ launchContext: { useIncognitoPages, launchOptions },// don't overwrite locale with fingerprint's localeuseFingerprints: false,- },+ }),requestHandler: async ({ page }) => {[timezone, locale, reducedMotion] = await Promise.all([page.evaluate(() => Intl.DateTimeFormat().resolvedOptions().timeZone),diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.tsindex a5344be85..f90960644 100644--- a/test/core/crawlers/puppeteer_crawler.test.ts+++ b/test/core/crawlers/puppeteer_crawler.test.ts@@ -18,6 +18,7 @@ import {createPuppeteerRouter,ProxyConfiguration,PuppeteerCrawler,+ puppeteerBrowserPool,RequestList,RequestQueue,RequestValidationError,@@ -115,7 +116,7 @@ describe('PuppeteerCrawler', () => {const puppeteerCrawler = new PuppeteerCrawler({requestList: requestListLarge,- browserPoolOptions: { useFingerprints: false },+ browserPool: puppeteerBrowserPool({ useFingerprints: false }),minConcurrency: 1,maxConcurrency: 1,requestHandler,@@ -275,14 +276,14 @@ describe('PuppeteerCrawler', () => {const crawler = new PuppeteerCrawler({requestQueue,navigationTimeoutSecs: 0.005,- browserPoolOptions: {+ browserPool: puppeteerBrowserPool({preLaunchHooks: [async () => {// Do some async work that's longer than navigationTimeoutSecsawait sleep(20);},],- },+ }),requestHandler,});@@ -363,16 +364,16 @@ describe('PuppeteerCrawler', () => {expect(sessions.size).toBe(3); // 3 different sessions used});- test('shallow clones browserPoolOptions before normalization', () => {+ test('does not mutate the launchContext it was given', () => {const options = {- browserPoolOptions: {},+ launchContext: {},+ headless: false,requestHandler: async () => {},};- void new PuppeteerCrawler(options);void new PuppeteerCrawler(options);- expect(Object.keys(options.browserPoolOptions).length).toBe(0);+ expect(options.launchContext).toEqual({});});if (os.platform() !== 'darwin') {@@ -397,16 +398,14 @@ describe('PuppeteerCrawler', () => {const puppeteerCrawler = new PuppeteerCrawler({requestList: requestListLarge,- launchContext: {- useIncognitoPages: true,- },- browserPoolOptions: {+ browserPool: puppeteerBrowserPool({+ launchContext: { useIncognitoPages: true },prePageCreateHooks: [(_id, _controller, options) => {options!.proxyBypassList = ['<-loopback>'];},],- },+ }),proxyConfiguration,requestHandler: async ({ page }) => {const content = await page.content();diff --git a/test/e2e/camoufox-cloudflare/actor/main.js b/test/e2e/camoufox-cloudflare/actor/main.jsindex ac6c77954..6c0a1c017 100644--- a/test/e2e/camoufox-cloudflare/actor/main.js+++ b/test/e2e/camoufox-cloudflare/actor/main.js@@ -1,4 +1,4 @@-import { Dataset, handleCloudflareChallengeHook, PlaywrightCrawler } from '@crawlee/playwright';+import { Dataset, handleCloudflareChallengeHook, PlaywrightCrawler, playwrightBrowserPool } from '@crawlee/playwright';import { Actor } from 'apify';import { launchOptions } from 'camoufox-js';import { firefox } from 'playwright';@@ -16,9 +16,17 @@ await Actor.main(async () => {// The target hard-blocks datacenter IP ranges outright (block page, not a solvable// challenge), so this test needs residential exit nodes.proxyConfiguration: await Actor.createProxyConfiguration({ groups: ['RESIDENTIAL'] }),- // Camoufox ships its own anti-detection; Crawlee's fingerprint injection conflicts with it- // and keeps Cloudflare from ever clearing the challenge.- browserPoolOptions: { useFingerprints: false },+ browserPool: playwrightBrowserPool({+ // Camoufox ships its own anti-detection; Crawlee's fingerprint injection conflicts with it+ // and keeps Cloudflare from ever clearing the challenge.+ useFingerprints: false,+ launchContext: {+ launcher: firefox,+ launchOptions: await launchOptions({+ headless: true,+ }),+ },+ }),preNavigationHooks: [async ({ page }) => {// TODO: remove this hook once a Camoufox build with daijro/camoufox#625 is released.@@ -42,12 +50,6 @@ await Actor.main(async () => {],// verbose keeps the challenge detection visible in the nightly run logspostNavigationHooks: [handleCloudflareChallengeHook({ verbose: true })],- launchContext: {- launcher: firefox,- launchOptions: await launchOptions({- headless: true,- }),- },async requestHandler({ page, parseWithCheerio }) {const isBlocked = await page.evaluate(async () => {diff --git a/test/e2e/stagehand-concurrent/actor/main.js b/test/e2e/stagehand-concurrent/actor/main.jsindex b5cc03550..8693d5284 100644--- a/test/e2e/stagehand-concurrent/actor/main.js+++ b/test/e2e/stagehand-concurrent/actor/main.js@@ -1,4 +1,4 @@-import { StagehandCrawler } from '@crawlee/stagehand';+import { StagehandCrawler, stagehandBrowserPool } from '@crawlee/stagehand';import { Actor } from 'apify';import { z } from 'zod';@@ -21,15 +21,15 @@ await Actor.main(async () => {const crawler = new StagehandCrawler({maxConcurrency: 3,maxRequestsPerCrawl: 3,- // Force one page per browser to ensure multiple browsers are used- browserPoolOptions: {+ browserPool: stagehandBrowserPool({+ // Force one page per browser to ensure multiple browsers are usedmaxOpenPagesPerBrowser: 1,- },- stagehandOptions: {- env: 'LOCAL',- model: 'anthropic/claude-haiku-4-5-20251001',- verbose: 0,- },+ stagehandOptions: {+ env: 'LOCAL',+ model: 'anthropic/claude-haiku-4-5-20251001',+ verbose: 0,+ },+ }),async requestHandler({ page, request, log, pushData }) {log.info(`Processing ${request.loadedUrl}`);diff --git a/test/integration/remote-browser-incognito.test.ts b/test/integration/remote-browser-incognito.test.tsindex 4c176870d..daae0453d 100644--- a/test/integration/remote-browser-incognito.test.ts+++ b/test/integration/remote-browser-incognito.test.ts@@ -11,7 +11,7 @@** With the wrapper removed, request 2's body should report no cookies.*/-import { PlaywrightCrawler } from 'crawlee';+import { PlaywrightCrawler, remotePlaywrightBrowserPool } from 'crawlee';import { expect, test } from 'vitest';import { BROWSERLESS_URL, httpbin } from './helpers.js';@@ -24,11 +24,11 @@ test.skipIf(!process.env.CRAWLEE_DIFFICULT_TESTS)(const observations: { controllerId: string; body: { cookies: Record<string, string> } }[] = [];const controllerIdByPage = new WeakMap<object, string>();- const crawler = new PlaywrightCrawler({- remoteBrowser: {- endpoint: BROWSERLESS_URL,- maxOpenBrowsers: 1,- },+ // Tuning the pool that wraps the remote connection means building it here rather than using the+ // crawler's terser `remoteBrowser` option - and owning its teardown as a result.+ const browserPool = remotePlaywrightBrowserPool({+ endpoint: BROWSERLESS_URL,+ maxOpenBrowsers: 1,browserPoolOptions: {retireBrowserAfterPageCount: 10, // keep the same browser across both requestsmaxOpenPagesPerBrowser: 2,@@ -38,6 +38,10 @@ test.skipIf(!process.env.CRAWLEE_DIFFICULT_TESTS)(},],},+ });++ const crawler = new PlaywrightCrawler({+ browserPool,saveResponseCookies: false, // remove Session-based propagationmaxConcurrency: 1,maxRequestsPerCrawl: 2,@@ -50,7 +54,11 @@ test.skipIf(!process.env.CRAWLEE_DIFFICULT_TESTS)(},});- await crawler.run([httpbin('/cookies/set?TOKEN=integration-test'), httpbin('/cookies')]);+ try {+ await crawler.run([httpbin('/cookies/set?TOKEN=integration-test'), httpbin('/cookies')]);+ } finally {+ await browserPool.destroy();+ }expect(observations).toHaveLength(2);// Same browser handled both requests — otherwise the assertion below proves nothing.diff --git a/test/stagehand-crawler/stagehand-crawler.test.ts b/test/stagehand-crawler/stagehand-crawler.test.tsindex 319bfede4..037afce65 100644--- a/test/stagehand-crawler/stagehand-crawler.test.ts+++ b/test/stagehand-crawler/stagehand-crawler.test.ts@@ -1,6 +1,13 @@import { z } from 'zod';-import { createStagehandRouter, StagehandCrawler } from '../../packages/stagehand-crawler/src/index.js';+import type { IBrowserPool } from '@crawlee/types';++import type { StagehandBrowserPool, StagehandPage } from '../../packages/stagehand-crawler/src/index.js';+import {+ createStagehandRouter,+ stagehandBrowserPool,+ StagehandCrawler,+} from '../../packages/stagehand-crawler/src/index.js';import { enhancePageWithStagehand } from '../../packages/stagehand-crawler/src/internals/utils/stagehand-utils.js';// Mock Stagehand to avoid actual browser launches and API calls@@ -107,6 +114,26 @@ describe('StagehandCrawler', () => {expect(crawler).toBeDefined();});++ // The launcher used to be built before `BrowserCrawler` folded `headless` into the launch context, so the+ // option never reached the plugin.+ test('forwards headless to the browser plugin', () => {+ const crawler = new StagehandCrawler({ headless: false, stagehandOptions: { env: 'LOCAL' } });++ expect((crawler.browserPool as unknown as StagehandBrowserPool).browserPlugins[0].launchOptions).toMatchObject({+ headless: false,+ });+ });++ test('rejects the options that would configure its own pool when a browserPool is passed in', () => {+ // The pool serves plain Playwright pages, the crawler types its own as enhanced `StagehandPage`s.+ const browserPool = stagehandBrowserPool() as unknown as IBrowserPool<StagehandPage>;++ expect(() => new StagehandCrawler({ browserPool, stagehandOptions: { env: 'LOCAL' } })).toThrow(+ 'StagehandCrawler: `stagehandOptions` cannot be combined with `browserPool`',+ );+ expect(() => new StagehandCrawler({ browserPool })).not.toThrow();+ });});describe('createStagehandRouter', () => {