Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(browser-pool): improve error handling when browser is not found (#2050)

Sviatozar Petrenko282527f31bb3
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 74616eaf3..f19053c84 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -1152,7 +1152,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
);
request.state = RequestState.DONE;
} catch (secondaryError: any) {
- if (!secondaryError.triggeredFromUserHandler) {
+ if (!secondaryError.triggeredFromUserHandler
+ // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
+ && !(secondaryError instanceof CriticalError)) {
const apifySpecific = process.env.APIFY_IS_AT_HOME
? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.` : '';
this.log.exception(secondaryError as Error, 'An exception occurred during handling of failed request. '
diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.json
index a05a86766..5b2720bfe 100644
--- a/packages/browser-pool/package.json
+++ b/packages/browser-pool/package.json
@@ -38,6 +38,7 @@
"dependencies": {
"@apify/log": "^2.4.0",
"@apify/timeout": "^0.3.0",
+ "@crawlee/core": "3.5.2",
"@crawlee/types": "3.5.2",
"fingerprint-generator": "^2.0.6",
"fingerprint-injector": "^2.0.5",
diff --git a/packages/browser-pool/src/playwright/playwright-plugin.ts b/packages/browser-pool/src/playwright/playwright-plugin.ts
index 2a3b72d02..fa1abf4e0 100644
--- a/packages/browser-pool/src/playwright/playwright-plugin.ts
+++ b/packages/browser-pool/src/playwright/playwright-plugin.ts
@@ -3,6 +3,7 @@ import net from 'net';
import os from 'os';
import path from 'path';
+import { CriticalError } from '@crawlee/core';
import type { Browser as PlaywrightBrowser, BrowserType } from 'playwright';
import { loadFirefoxAddon } from './load-firefox-addon';
@@ -63,7 +64,9 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
try {
if (useIncognitoPages) {
- browser = await this.library.launch(launchOptions);
+ browser = await this.library.launch(launchOptions).catch((error) => {
+ return this._throwOnFailedLaunch(launchContext, error);
+ });
if (anonymizedProxyUrl) {
browser.on('disconnected', async () => {
@@ -108,7 +111,9 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
}
}
- const browserContext = await this.library.launchPersistentContext(userDataDir, launchOptions);
+ const browserContext = await this.library.launchPersistentContext(userDataDir, launchOptions).catch((error) => {
+ return this._throwOnFailedLaunch(launchContext, error);
+ });
browserContext.once('close', () => {
if (userDataDir.includes('apify-playwright-firefox-taac-')) {
@@ -174,6 +179,22 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
return browser;
}
+ private _throwOnFailedLaunch(launchContext: LaunchContext<BrowserType>, cause: unknown): never {
+ let debugMessage = `Failed to launch browser.`
+ + `${launchContext.launchOptions?.executablePath
+ ? ` Check whether the provided executable path is correct: ${launchContext.launchOptions?.executablePath}.` : ''}`;
+ if (process.env.APIFY_IS_AT_HOME) {
+ debugMessage += ' Make sure your Dockerfile extends apify/actor-node-playwright-*` (with a correct browser name). Or install';
+ } else {
+ debugMessage += ' Try installing';
+ }
+ debugMessage += ' the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).'
+ + ' The original error will be displayed at the bottom as the [cause].';
+ throw new CriticalError(debugMessage, {
+ cause,
+ });
+ }
+
protected _createController(): BrowserController<BrowserType, SafeParameters<BrowserType['launch']>[0], PlaywrightBrowser> {
return new PlaywrightController(this);
}
diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
index 5e0de0f1c..8d011e615 100644
--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
@@ -1,3 +1,4 @@
+import { CriticalError } from '@crawlee/core';
import type { Dictionary } from '@crawlee/types';
import type Puppeteer from 'puppeteer';
import type * as PuppeteerTypes from 'puppeteer';
@@ -72,10 +73,22 @@ export class PuppeteerPlugin extends BrowserPlugin<
await close();
});
}
- } catch (error) {
+ } catch (error: any) {
await close();
-
- throw error;
+ let debugMessage = `Failed to launch browser.`
+ + `${launchContext.launchOptions?.executablePath
+ ? ` Check whether the provided executable path is correct: ${launchContext.launchOptions?.executablePath}.` : ''}`;
+ if (process.env.APIFY_IS_AT_HOME) {
+ debugMessage += ' Make sure your Dockerfile extends `apify/actor-node-puppeteer-chrome. Or install';
+ } else {
+ debugMessage += ' Try installing';
+ }
+ debugMessage += ` a browser, if it's missing, by running \`npx @puppeteer/browsers install chromium --path [path]\``
+ + ` and pointing \`executablePath\` to the downloaded executable (https://pptr.dev/browsers-api).`
+ + ` The original error will be displayed at the bottom as the [cause].`;
+ throw new CriticalError(debugMessage, {
+ cause: error,
+ });
}
}
diff --git a/packages/core/src/autoscaling/autoscaled_pool.ts b/packages/core/src/autoscaling/autoscaled_pool.ts
index 8e1c601f6..376fdc992 100644
--- a/packages/core/src/autoscaling/autoscaled_pool.ts
+++ b/packages/core/src/autoscaling/autoscaled_pool.ts
@@ -9,6 +9,7 @@ import { Snapshotter } from './snapshotter';
import type { SystemInfo, SystemStatusOptions } from './system_status';
import { SystemStatus } from './system_status';
import { Configuration } from '../configuration';
+import { CriticalError } from '../errors';
import { log as defaultLog } from '../log';
export interface AutoscaledPoolOptions {
@@ -551,7 +552,12 @@ export class AutoscaledPool {
// We might have already rejected this promise.
if (this.reject) {
// No need to log all concurrent errors.
- this.log.exception(err, 'runTaskFunction failed.');
+ if (
+ // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
+ !(e instanceof CriticalError)
+ ) {
+ this.log.exception(err, 'runTaskFunction failed.');
+ }
this.reject(err);
}
}
diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.ts
index f8a0e1499..3e55610ca 100644
--- a/packages/core/src/errors.ts
+++ b/packages/core/src/errors.ts
@@ -5,6 +5,7 @@ export class NonRetryableError extends Error {}
/**
* Errors of `CriticalError` type will shut down the whole crawler.
+ * Error handlers catching CriticalError should avoid logging it, as it will be logged by Node.js itself at the end
*/
export class CriticalError extends NonRetryableError {}
diff --git a/yarn.lock b/yarn.lock
index 56d44b1eb..97c8de16b 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -761,6 +761,7 @@ __metadata:
dependencies:
"@apify/log": ^2.4.0
"@apify/timeout": ^0.3.0
+ "@crawlee/core": 3.5.2
"@crawlee/types": 3.5.2
fingerprint-generator: ^2.0.6
fingerprint-injector: ^2.0.5