Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: `browserPerProxy` browser launch option (#2418)

Jindřich Bärdf57b2965ac8
diff --git a/packages/browser-crawler/src/internals/browser-launcher.ts b/packages/browser-crawler/src/internals/browser-launcher.ts
index 3ec11a1c6..22bb37b4d 100644
--- a/packages/browser-crawler/src/internals/browser-launcher.ts
+++ b/packages/browser-crawler/src/internals/browser-launcher.ts
@@ -31,6 +31,14 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginO
*/
useChrome?: boolean;
+ /**
+ * If set to `true`, the crawler respects the proxy url generated for the given request.
+ * This aligns the browser-based crawlers with the `HttpCrawler`.
+ *
+ * Might cause performance issues, as Crawlee might launch too many browser instances.
+ */
+ browserPerProxy?: boolean;
+
/**
* With this option selected, all pages will be opened in a new incognito browser context.
* This means they will not share cookies nor cache and their resources will not be throttled by one another.
@@ -98,6 +106,7 @@ export abstract class BrowserLauncher<
proxyUrl: ow.optional.string.url,
useChrome: ow.optional.boolean,
useIncognitoPages: ow.optional.boolean,
+ browserPerProxy: ow.optional.boolean,
experimentalContainers: ow.optional.boolean,
userDataDir: ow.optional.string,
launchOptions: ow.optional.object,
diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.ts
index 25a040410..901a5ad23 100644
--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts
+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts
@@ -75,6 +75,13 @@ export interface BrowserPluginOptions<LibraryOptions> {
* Path to a User Data Directory, which stores browser session data like cookies and local storage.
*/
userDataDir?: string;
+ /**
+ * If set to `true`, the crawler respects the proxy url generated for the given request.
+ * This aligns the browser-based crawlers with the `HttpCrawler`.
+ *
+ * Might cause performance issues, as Crawlee might launch too many browser instances.
+ */
+ browserPerProxy?: boolean;
}
export interface CreateLaunchContextOptions<
@@ -112,6 +119,8 @@ export abstract class BrowserPlugin<
experimentalContainers: boolean;
+ browserPerProxy?: boolean;
+
constructor(library: Library, options: BrowserPluginOptions<LibraryOptions> = {}) {
const {
launchOptions = {} as LibraryOptions,
@@ -119,6 +128,7 @@ export abstract class BrowserPlugin<
userDataDir,
useIncognitoPages = false,
experimentalContainers = false,
+ browserPerProxy = false,
} = options;
this.library = library;
@@ -127,6 +137,7 @@ export abstract class BrowserPlugin<
this.userDataDir = userDataDir;
this.useIncognitoPages = useIncognitoPages;
this.experimentalContainers = experimentalContainers;
+ this.browserPerProxy = browserPerProxy;
}
/**
@@ -145,6 +156,7 @@ export abstract class BrowserPlugin<
useIncognitoPages = this.useIncognitoPages,
userDataDir = this.userDataDir,
experimentalContainers = this.experimentalContainers,
+ browserPerProxy = this.browserPerProxy,
proxyTier,
} = options;
@@ -156,6 +168,7 @@ export abstract class BrowserPlugin<
useIncognitoPages,
experimentalContainers,
userDataDir,
+ browserPerProxy,
proxyTier,
});
}
diff --git a/packages/browser-pool/src/browser-pool.ts b/packages/browser-pool/src/browser-pool.ts
index 9171fb022..d2908c282 100644
--- a/packages/browser-pool/src/browser-pool.ts
+++ b/packages/browser-pool/src/browser-pool.ts
@@ -94,6 +94,16 @@ export interface BrowserPoolOptions<Plugin extends BrowserPlugin = BrowserPlugin
* @default 300
*/
closeInactiveBrowserAfterSecs?: number;
+ /**
+ * Browsers are marked as retired after they have been inactive for a certain
+ * amount of time. This option sets the interval at which the browsers
+ * are checked and retired if they are inactive.
+ *
+ * Retired browsers are closed after all their pages are closed.
+ *
+ * @default 1
+ */
+ retireInactiveBrowserAfterSecs?: number;
/**
* @default true
*/
@@ -306,6 +316,8 @@ export class BrowserPool<
BROWSER_KILLER_INTERVAL_MILLIS,
);
+ private browserRetireInterval?: NodeJS.Timeout;
+
private limiter = pLimit(1);
constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>) {
@@ -319,6 +331,7 @@ export class BrowserPool<
retireBrowserAfterPageCount: ow.optional.number,
operationTimeoutSecs: ow.optional.number,
closeInactiveBrowserAfterSecs: ow.optional.number,
+ retireInactiveBrowserAfterSecs: ow.optional.number,
preLaunchHooks: ow.optional.array,
postLaunchHooks: ow.optional.array,
prePageCreateHooks: ow.optional.array,
@@ -335,6 +348,7 @@ export class BrowserPool<
retireBrowserAfterPageCount = 100,
operationTimeoutSecs = 15,
closeInactiveBrowserAfterSecs = 300,
+ retireInactiveBrowserAfterSecs = 1,
preLaunchHooks = [],
postLaunchHooks = [],
prePageCreateHooks = [],
@@ -367,6 +381,18 @@ export class BrowserPool<
this.useFingerprints = useFingerprints;
this.fingerprintOptions = fingerprintOptions;
+ this.browserRetireInterval = setInterval(
+ async () => this.activeBrowserControllers.forEach((controller) => {
+ if (
+ controller.activePages === 0
+ && controller.lastPageOpenedAt < (Date.now() - retireInactiveBrowserAfterSecs * 1000)
+ ) {
+ this.retireBrowserController(controller);
+ }
+ }), retireInactiveBrowserAfterSecs * 1000);
+
+ this.browserRetireInterval!.unref();
+
// hooks
this.preLaunchHooks = preLaunchHooks;
this.postLaunchHooks = postLaunchHooks;
@@ -613,7 +639,9 @@ export class BrowserPool<
*/
async destroy(): Promise<void> {
clearInterval(this.browserKillerInterval!);
+ clearInterval(this.browserRetireInterval!);
this.browserKillerInterval = undefined;
+ this.browserRetireInterval = undefined;
await this.closeAllBrowsers();
@@ -706,13 +734,14 @@ export class BrowserPool<
return [...this.activeBrowserControllers].find((controller) => {
const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
const isCorrectPlugin = controller.browserPlugin === browserPlugin;
- const isSameProxyUrl = (controller.proxyUrl === options?.proxyUrl);
+ const isSameProxyUrl = controller.proxyUrl === options?.proxyUrl;
const isCorrectProxyTier = controller.proxyTier === options?.proxyTier;
return isCorrectPlugin
&& hasCapacity
&& (
- (options?.proxyTier && isCorrectProxyTier)
+ (!controller.launchContext.browserPerProxy && !options?.proxyTier)
+ || (options?.proxyTier && isCorrectProxyTier)
|| (options?.proxyUrl && isSameProxyUrl)
|| (!options?.proxyUrl && !options?.proxyTier && !controller.proxyUrl && !controller.proxyTier)
);
diff --git a/packages/browser-pool/src/launch-context.ts b/packages/browser-pool/src/launch-context.ts
index a89d31edf..e7cbdfbb4 100644
--- a/packages/browser-pool/src/launch-context.ts
+++ b/packages/browser-pool/src/launch-context.ts
@@ -34,6 +34,13 @@ export interface LaunchContextOptions<
* Those changes would be typically made in pre-launch hooks.
*/
launchOptions: LibraryOptions;
+ /**
+ * If set to `true`, the crawler respects the proxy url generated for the given request.
+ * This aligns the browser-based crawlers with the `HttpCrawler`.
+ *
+ * Might cause performance issues, as Crawlee might launch too many browser instances.
+ */
+ browserPerProxy?: boolean;
/**
* By default pages share the same browser context.
* If set to `true` each page uses its own context that is destroyed once the page is closed or crashes.
@@ -64,6 +71,7 @@ export class LaunchContext<
browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
launchOptions: LibraryOptions;
useIncognitoPages: boolean;
+ browserPerProxy?: boolean;
experimentalContainers: boolean;
userDataDir: string;
proxyTier?: number;
@@ -81,6 +89,7 @@ export class LaunchContext<
launchOptions,
proxyUrl,
useIncognitoPages,
+ browserPerProxy,
experimentalContainers,
userDataDir = '',
proxyTier,
@@ -89,6 +98,7 @@ export class LaunchContext<
this.id = id;
this.browserPlugin = browserPlugin;
this.launchOptions = launchOptions;
+ this.browserPerProxy = browserPerProxy ?? false;
this.useIncognitoPages = useIncognitoPages ?? false;
this.experimentalContainers = experimentalContainers ?? false;
this.userDataDir = userDataDir;
diff --git a/packages/browser-pool/test/multiple-plugins.test.ts b/packages/browser-pool/test/multiple-plugins.test.ts
index 6390c0a39..25db43d39 100644
--- a/packages/browser-pool/test/multiple-plugins.test.ts
+++ b/packages/browser-pool/test/multiple-plugins.test.ts
@@ -2,7 +2,11 @@ import { BrowserPool, PlaywrightPlugin } from '@crawlee/browser-pool';
import playwright from 'playwright';
describe('BrowserPool - Using multiple plugins', () => {
- let browserPool: BrowserPool<{ browserPlugins: [PlaywrightPlugin, PlaywrightPlugin]; closeInactiveBrowserAfterSecs: 2 }>;
+ let browserPool: BrowserPool<{
+ browserPlugins: [PlaywrightPlugin, PlaywrightPlugin];
+ closeInactiveBrowserAfterSecs: 2;
+ retireInactiveBrowserAfterSecs: 30;
+ }>;
const chromePlugin = new PlaywrightPlugin(playwright.chromium);
const firefoxPlugin = new PlaywrightPlugin(playwright.firefox);
@@ -14,6 +18,7 @@ describe('BrowserPool - Using multiple plugins', () => {
firefoxPlugin,
],
closeInactiveBrowserAfterSecs: 2,
+ retireInactiveBrowserAfterSecs: 30,
});
});