Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: fix `v4` CI build issues (#3856)
Jindřich Bär3090359bc917
diff --git a/docs/public-api/crawlee-browser-pool.api.md b/docs/public-api/crawlee-browser-pool.api.mdindex d6c3fd809..21dae4afc 100644--- a/docs/public-api/crawlee-browser-pool.api.md+++ b/docs/public-api/crawlee-browser-pool.api.md@@ -127,6 +127,7 @@ export abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrarprotected abstract _addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;// (undocumented)browserPerProxy?: boolean;+ protected _connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, connect: (url: string) => Promise<LaunchResult>): Promise<LaunchResult>;// (undocumented)abstract createController(): BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;createLaunchContext(options?: CreateLaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;@@ -147,12 +148,18 @@ export abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrarname: string;// (undocumented)proxyUrl?: string;+ // @internal+ remoteConnection?: RemoteConnection;+ // @internal+ remoteConnectionParameters?: RemoteConnectionParameters;// (undocumented)protected _throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;// (undocumented)useIncognitoPages: boolean;// (undocumented)userDataDir?: string;+ // @internal+ useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;}// @public (undocumented)@@ -191,8 +198,12 @@ export class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptionsgetBrowserControllerByPage(page: PageReturn): BrowserControllerReturn | undefined;getPage(id: string): PageReturn | undefined;getPageId(page: PageReturn): string | undefined;+ hasActiveBrowserWithFreeCapacity(): boolean;+ hasFreeBrowserSlot(): boolean;injectPageState(page: PageReturn, state: PageState): Promise<void>;// (undocumented)+ maxOpenBrowsers: number;+ // (undocumented)maxOpenPagesPerBrowser: number;newPage(options?: BrowserPoolNewPageOptions<PageOptions, BrowserPlugins[number]>): Promise<PageReturn>;newPageInNewBrowser(options?: BrowserPoolNewPageInNewBrowserOptions<PageOptions, BrowserPlugins[number]>): Promise<PageReturn>;@@ -309,6 +320,9 @@ export interface CommonPage {url(): string | Promise<string>;}+// @public+export type CrawlerRemoteBrowserOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins' | 'browserPoolOptions'>;+// @public (undocumented)export interface CreateLaunchContextOptions<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends Partial<Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>> {}@@ -387,9 +401,13 @@ export class LaunchContext<Library extends CommonLibrary = CommonLibrary, Librar// (undocumented)ignoreProxyCertificate?: boolean;// (undocumented)+ readonly isRemote: boolean;+ // (undocumented)launchOptions: LibraryOptions;set proxyUrl(url: string | undefined);get proxyUrl(): string | undefined;+ // @internal+ _remoteToken?: number;// (undocumented)useIncognitoPages: boolean;// (undocumented)@@ -402,6 +420,7 @@ export interface LaunchContextOptions<Library extends CommonLibrary = CommonLibrbrowserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;id?: string;ignoreProxyCertificate?: boolean;+ isRemote?: boolean;launchOptions: LibraryOptions;// (undocumented)proxyUrl?: string;@@ -480,6 +499,7 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<protected _isChromiumBasedBrowser(): boolean;// (undocumented)protected _launch(launchContext: LaunchContext<BrowserType>): Promise<Browser>;+ useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;}// @public@@ -526,6 +546,82 @@ export class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer, PuppeteerTyprotected _isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;// (undocumented)protected _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<PuppeteerTypes.Browser>;+ useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;+}++// @public+export type RemoteBrowserEndpoint = string | ((options?: {+ proxyUrl?: string;+}) => string | ResolvedRemoteEndpoint | Promise<string | ResolvedRemoteEndpoint>);++// @public+export class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {+ constructor(options: RemoteBrowserPoolOptions);+ readonly browserPool: BrowserPool;+ // (undocumented)+ closePage(page: Page, options?: {+ error?: Error;+ }): Promise<void>;+ destroy(): Promise<void>;+ // (undocumented)+ extractPageState(page: Page): Promise<PageState>;+ // (undocumented)+ injectPageState(page: Page, state: PageState): Promise<void>;+ get maxOpenBrowsers(): number;+ set maxOpenBrowsers(value: number);+ newPage(options?: NewPageOptions): Promise<Page>;+}++// @public (undocumented)+export interface RemoteBrowserPoolOptions {+ browserPlugins: BrowserPlugin[];+ browserPoolOptions?: Omit<BrowserPoolOptions, 'browserPlugins'> & BrowserPoolHooks<any, any, any>;+ connection?: RemoteConnectionParameters;+ endpoint: RemoteBrowserEndpoint | RemoteBrowserProvider<any>;+ maxOpenBrowsers?: number;+ release?: (info: {+ endpoint: string;+ context?: Record<string, unknown>;+ }) => unknown;+ slotPollIntervalMillis?: number;+}++// @public+export abstract class RemoteBrowserProvider<TContext extends Record<string, unknown> = Record<string, unknown>> {+ abstract connect(options?: {+ proxyUrl?: string;+ }): Promise<{+ url: string;+ context?: TContext;+ }> | {+ url: string;+ context?: TContext;+ };+ maxOpenBrowsers?: number;+ release(_context: TContext): Promise<void>;+}++// @internal+export interface RemoteConnection {+ release(token: number): Promise<void>;+ resolve(options?: {+ proxyUrl?: string;+ }): Promise<{+ url: string;+ token: number;+ }>;+}++// @public+export interface RemoteConnectionParameters {+ connectOptions?: Record<string, unknown>;+ protocol?: 'cdp' | 'playwright';+}++// @public+export interface ResolvedRemoteEndpoint {+ context?: Record<string, unknown>;+ url: string;}// @public (undocumented)diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.mdindex 877395556..206a12090 100644--- a/docs/public-api/crawlee-playwright.api.md+++ b/docs/public-api/crawlee-playwright.api.md@@ -290,6 +290,7 @@ export class PlaywrightCrawler<ContextExtension = Dictionary_2<never>, ExtendedClaunchContext: ObjectPredicate<object> & BasePredicate<object | undefined>;headless: AnyPredicate<string | boolean>;browserPool: ObjectPredicate<object> & BasePredicate<object | undefined>;+ remoteBrowser: ObjectPredicate<object> & BasePredicate<object | undefined>;saveResponseCookies: BooleanPredicate & BasePredicate<boolean | undefined>;proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;diff --git a/docs/public-api/crawlee-puppeteer.api.md b/docs/public-api/crawlee-puppeteer.api.mdindex d1de76588..384be6478 100644--- a/docs/public-api/crawlee-puppeteer.api.md+++ b/docs/public-api/crawlee-puppeteer.api.md@@ -212,6 +212,7 @@ export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContlaunchContext: ObjectPredicate<object> & BasePredicate<object | undefined>;headless: AnyPredicate<string | boolean>;browserPool: ObjectPredicate<object> & BasePredicate<object | undefined>;+ remoteBrowser: ObjectPredicate<object> & BasePredicate<object | undefined>;saveResponseCookies: BooleanPredicate & BasePredicate<boolean | undefined>;proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;diff --git a/docs/public-api/crawlee-stagehand.api.md b/docs/public-api/crawlee-stagehand.api.mdindex 8742ca6a1..9d6870052 100644--- a/docs/public-api/crawlee-stagehand.api.md+++ b/docs/public-api/crawlee-stagehand.api.md@@ -90,6 +90,7 @@ export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedContlaunchContext: ObjectPredicate<object> & BasePredicate<object | undefined>;headless: AnyPredicate<string | boolean>;browserPool: ObjectPredicate<object> & BasePredicate<object | undefined>;+ remoteBrowser: ObjectPredicate<object> & BasePredicate<object | undefined>;saveResponseCookies: BooleanPredicate & BasePredicate<boolean | undefined>;proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.tsindex 5be691309..2b8827b2a 100644--- a/test/core/crawlers/browser_crawler.test.ts+++ b/test/core/crawlers/browser_crawler.test.ts@@ -172,28 +172,19 @@ describe('BrowserCrawler', () => {});test.concurrent('builds and owns a RemoteBrowserPool from the remoteBrowser option', async () => {- const localStorageEmulator = new MemoryStorageEmulator();- await localStorageEmulator.init();-- try {- const crawler = new BrowserCrawlerTest({- remoteBrowser: { endpoint: 'ws://remote:9222', maxOpenBrowsers: 2 },- browserPoolOptions: { browserPlugins: [new PuppeteerPlugin(puppeteer)] },- requestHandler: async () => {},- });+ const crawler = new BrowserCrawlerTest({+ remoteBrowser: { endpoint: 'ws://remote:9222', maxOpenBrowsers: 2 },+ browserPoolOptions: { browserPlugins: [new PuppeteerPlugin(puppeteer)] },+ requestHandler: async () => {},+ });- expect(crawler.browserPool).toBeInstanceOf(RemoteBrowserPool);- expect((crawler.browserPool as RemoteBrowserPool).maxOpenBrowsers).toBe(2);+ expect(crawler.browserPool).toBeInstanceOf(RemoteBrowserPool);+ expect((crawler.browserPool as RemoteBrowserPool).maxOpenBrowsers).toBe(2);- await (crawler.browserPool as RemoteBrowserPool).destroy();- } finally {- await localStorageEmulator.destroy();- }+ await (crawler.browserPool as RemoteBrowserPool).destroy();});test.concurrent('uses browserPool and ignores remoteBrowser when both are set', async () => {- const localStorageEmulator = new MemoryStorageEmulator();- await localStorageEmulator.init();const externalPool = new BrowserPoolClass({ browserPlugins: [new PuppeteerPlugin(puppeteer)] });try {@@ -206,7 +197,6 @@ describe('BrowserCrawler', () => {expect(crawler.browserPool).toBe(externalPool);} finally {await externalPool.destroy();- await localStorageEmulator.destroy();}});