Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: don't retire browsers with long-running `pre|postLaunchHooks` prematurely (#3062)

Jindřich Bär681660e35a1c
diff --git a/packages/browser-pool/src/browser-pool.ts b/packages/browser-pool/src/browser-pool.ts
index c0be47dcc..8c26ee8b8 100644
--- a/packages/browser-pool/src/browser-pool.ts
+++ b/packages/browser-pool/src/browser-pool.ts
@@ -318,6 +318,7 @@ export class BrowserPool<
pageCounter = 0;
pages = new Map<string, PageReturn>();
pageIds = new WeakMap<PageReturn, string>();
+ startingBrowserControllers = new Set<BrowserControllerReturn>();
activeBrowserControllers = new Set<BrowserControllerReturn>();
retiredBrowserControllers = new Set<BrowserControllerReturn>();
pageToBrowserController = new WeakMap<PageReturn, BrowserControllerReturn>();
@@ -610,11 +611,15 @@ export class BrowserPool<
*
*/
retireBrowserController(browserController: BrowserControllerReturn): void {
- const hasBeenRetiredOrKilled = !this.activeBrowserControllers.has(browserController);
+ const isStarting = this.startingBrowserControllers.has(browserController);
+ const isActive = this.activeBrowserControllers.has(browserController);
+
+ const hasBeenRetiredOrKilled = !isStarting && !isActive;
if (hasBeenRetiredOrKilled) return;
this.retiredBrowserControllers.add(browserController);
this.emit(BROWSER_POOL_EVENTS.BROWSER_RETIRED, browserController);
+ this.startingBrowserControllers.delete(browserController);
this.activeBrowserControllers.delete(browserController);
}
@@ -632,7 +637,7 @@ export class BrowserPool<
* closed after all their pages are closed.
*/
retireAllBrowsers(): void {
- this.activeBrowserControllers.forEach((controller) => {
+ [...this.startingBrowserControllers, ...this.activeBrowserControllers].forEach((controller) => {
this.retireBrowserController(controller);
});
}
@@ -665,6 +670,7 @@ export class BrowserPool<
}
private _teardown() {
+ this.startingBrowserControllers.clear();
this.activeBrowserControllers.clear();
this.retiredBrowserControllers.clear();
@@ -672,14 +678,18 @@ export class BrowserPool<
}
private _getAllBrowserControllers() {
- return new Set([...this.activeBrowserControllers, ...this.retiredBrowserControllers]);
+ return new Set([
+ ...this.startingBrowserControllers,
+ ...this.activeBrowserControllers,
+ ...this.retiredBrowserControllers,
+ ]);
}
private async _launchBrowser(pageId: string, options: InternalLaunchBrowserOptions<BrowserPlugins[number]>) {
const { browserPlugin, launchOptions, proxyTier, proxyUrl } = options;
const browserController = browserPlugin.createController() as BrowserControllerReturn;
- this.activeBrowserControllers.add(browserController);
+ this.startingBrowserControllers.add(browserController);
const launchContext = browserPlugin.createLaunchContext({
id: pageId,
@@ -697,7 +707,7 @@ export class BrowserPool<
tryCancel();
browserController.assignBrowser(browser, launchContext);
} catch (err) {
- this.activeBrowserControllers.delete(browserController);
+ this.startingBrowserControllers.delete(browserController);
throw err;
}
@@ -710,7 +720,7 @@ export class BrowserPool<
// both the controller and the browser before throwing.
await this._executeHooks(this.postLaunchHooks, pageId, browserController);
} catch (err) {
- this.activeBrowserControllers.delete(browserController);
+ this.startingBrowserControllers.delete(browserController);
browserController.close().catch((closeErr) => {
log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
id: browserController.id,
@@ -721,6 +731,8 @@ export class BrowserPool<
tryCancel();
browserController.activate();
+ this.startingBrowserControllers.delete(browserController);
+ this.activeBrowserControllers.add(browserController);
this.emit(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, browserController);
return browserController;
diff --git a/test/browser-pool/browser-pool.test.ts b/test/browser-pool/browser-pool.test.ts
index 655fde4cb..c20cf65bd 100644
--- a/test/browser-pool/browser-pool.test.ts
+++ b/test/browser-pool/browser-pool.test.ts
@@ -2,6 +2,7 @@
import http from 'node:http';
import { promisify } from 'node:util';
+import { sleep } from 'crawlee';
import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';
import playwright from 'playwright';
import type { Server as ProxyChainServer } from 'proxy-chain';
@@ -56,6 +57,7 @@ describe.each([
browserPool = new BrowserPool({
browserPlugins: [plugin],
closeInactiveBrowserAfterSecs: 2,
+ retireInactiveBrowserAfterSecs: 2,
});
});
@@ -92,6 +94,7 @@ describe.each([
await browserPool.newPage();
browserPool.retireAllBrowsers();
+ expect(browserPool.startingBrowserControllers.size).toBe(0);
expect(browserPool.activeBrowserControllers.size).toBe(0);
expect(browserPool.retiredBrowserControllers.size).toBe(1);
});
@@ -104,6 +107,7 @@ describe.each([
await browserPool.destroy();
expect(browserController.close).toHaveBeenCalled();
+ expect(browserPool.startingBrowserControllers.size).toBe(0);
expect(browserPool.activeBrowserControllers.size).toBe(0);
expect(browserPool.retiredBrowserControllers.size).toBe(0);
expect(browserPool['browserKillerInterval']).toBeUndefined();
@@ -164,6 +168,7 @@ describe.each([
await browserPool.newPageInNewBrowser();
await browserPool.newPageInNewBrowser();
+ expect(browserPool.startingBrowserControllers.size).toBe(0);
expect(browserPool.activeBrowserControllers.size).toBe(3);
expect(plugin.launch).toHaveBeenCalledTimes(3);
});
@@ -284,6 +289,44 @@ describe.each([
indexArray.forEach((v, index) => expect(v).toEqual(index));
});
+ test('browser lifecycle works correctly', async () => {
+ let resolvePreLaunchHook: (() => void) | null = null;
+ let resolvePostLaunchHook: (() => void) | null = null;
+
+ const preLaunchPromise = new Promise<void>((resolve) => {
+ resolvePreLaunchHook = resolve;
+ });
+ const postLaunchPromise = new Promise<void>((resolve) => {
+ resolvePostLaunchHook = resolve;
+ });
+
+ browserPool.preLaunchHooks = [...browserPool.preLaunchHooks, async () => preLaunchPromise];
+
+ browserPool.postLaunchHooks = [...browserPool.postLaunchHooks, async () => postLaunchPromise];
+
+ const newPagePromise = browserPool.newPage();
+
+ await sleep(200);
+
+ expect(browserPool.startingBrowserControllers.size).toBe(1);
+ expect(browserPool.activeBrowserControllers.size).toBe(0);
+ expect(browserPool.retiredBrowserControllers.size).toBe(0);
+
+ await sleep(5e3); // Make the wait longer than the browser pool's retireInactiveBrowserAfterSecs + closeInactiveBrowserAfterSecs
+
+ resolvePreLaunchHook!();
+ resolvePostLaunchHook!();
+
+ const page = await newPagePromise;
+
+ expect(browserPool.startingBrowserControllers.size).toBe(0);
+ expect(browserPool.activeBrowserControllers.size).toBe(1);
+ expect(browserPool.retiredBrowserControllers.size).toBe(0);
+
+ await (page.evaluate as any)('() => {}'); // Make sure the page is usable
+ await page.close();
+ });
+
describe('preLaunchHooks', () => {
test('should evaluate hook before launching browser with correct args', async () => {
const myAsyncHook = async () => Promise.resolve();