Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { type CrawleeLogger, CriticalError, serviceLocator } from '@crawlee/core';
import type { Dictionary } from '@crawlee/types';
import merge from 'lodash.merge';
import type { LaunchContextOptions } from '../launch-context.js';
import { LaunchContext } from '../launch-context.js';
import type { UnwrapPromise } from '../utils.js';
import type { BrowserController } from './browser-controller.js';
/**
* The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
* when Chromium/Chrome browser is launched:
* - in headless mode,
* - without using a fingerprint,
* - without specifying a user agent.
* Last updated on 2022-05-05.
*/
export const DEFAULT_USER_AGENT =
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36';
/**
* Each plugin expects an instance of the object with the `.launch()` property.
* For Puppeteer, it is the `puppeteer` module itself, whereas for Playwright
* it is one of the browser types, such as `puppeteer.chromium`.
* `BrowserPlugin` does not include the library. You can choose any version
* or fork of the library. It also keeps `browser-pool` installation small.
*/
export interface CommonLibrary {
product?: string;
launch(opts?: Dictionary): Promise<CommonBrowser>;
name?: () => string;
}
export interface CommonBrowser {
newPage(...args: unknown[]): Promise<CommonPage>;
}
export interface CommonPage {
close(...args: unknown[]): Promise<unknown>;
url(): string | Promise<string>;
evaluate(pageFunction: ((...args: any[]) => unknown) | string, ...args: unknown[]): Promise<unknown>;
}
export interface BrowserPluginOptions<LibraryOptions> {
/**
* Options that will be passed down to the automation library. E.g.
* `puppeteer.launch(launchOptions);`. This is a good place to set
* options that you want to apply as defaults. To dynamically override
* those options per-browser, see the `preLaunchHooks` of {@apilink BrowserPool}.
*/
launchOptions?: LibraryOptions;
/**
* Automation libraries configure proxies differently. This helper allows you
* to set a proxy URL without worrying about specific implementations.
* It also allows you use an authenticated proxy without extra code.
*/
proxyUrl?: string;
/**
* By default pages share the same browser context.
* If set to true each page uses its own context that is destroyed once the page is closed or crashes.
*
* @default false
*/
useIncognitoPages?: boolean;
/**
* Path to a User Data Directory, which stores browser session data like cookies and local storage.
*/
userDataDir?: string;
/**
* If set to `true`, the crawler respects the proxy url generated for the given request.
* This aligns the browser-based crawlers with the `HttpCrawler`.
*
* Might cause performance issues, as Crawlee might launch too many browser instances.
*/
browserPerProxy?: boolean;
/**
* If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
* This is useful when using HTTPS proxies with self-signed certificates.
*/
ignoreProxyCertificate?: boolean;
}
export interface CreateLaunchContextOptions<
Library extends CommonLibrary,
LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],
LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,
NewPageOptions = Parameters<LaunchResult['newPage']>[0],
NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
> extends Partial<
Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>
> {}
/**
* The `BrowserPlugin` serves two purposes. First, it is the base class that
* specialized controllers like `PuppeteerPlugin` or `PlaywrightPlugin` extend.
* Second, it allows the user to configure the automation libraries and
* feed them to {@apilink BrowserPool} for use.
*/
export abstract class BrowserPlugin<
Library extends CommonLibrary = CommonLibrary,
LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0],
LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,
NewPageOptions = Parameters<LaunchResult['newPage']>[0],
NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
> {
readonly name = this.constructor.name;
/**
* Kept `protected` rather than `#private` because the concrete plugins in this package are separate
* classes — an ES private field would not be visible to them.
*
* @internal
*/
protected readonly log!: CrawleeLogger;
readonly library: Library;
readonly launchOptions: LibraryOptions;
readonly proxyUrl?: string;
readonly userDataDir?: string;
useIncognitoPages: boolean;
readonly browserPerProxy?: boolean;
readonly ignoreProxyCertificate?: boolean;
constructor(library: Library, options: BrowserPluginOptions<LibraryOptions> = {}) {
const {
launchOptions = {} as LibraryOptions,
proxyUrl,
userDataDir,
useIncognitoPages = false,
browserPerProxy = false,
ignoreProxyCertificate = false,
} = options;
this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
this.library = library;
this.launchOptions = launchOptions;
this.proxyUrl = proxyUrl && new URL(proxyUrl).href.replace(/\/$/, '');
this.userDataDir = userDataDir;
this.useIncognitoPages = useIncognitoPages;
this.browserPerProxy = browserPerProxy;
this.ignoreProxyCertificate = ignoreProxyCertificate;
}
/**
* Creates a `LaunchContext` with all the information needed
* to launch a browser. Aside from library specific launch options,
* it also includes internal properties used by `BrowserPool` for
* management of the pool and extra features.
*/
createLaunchContext(
options: CreateLaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult> = {},
): LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult> {
const {
id,
launchOptions = {},
proxyUrl = this.proxyUrl,
useIncognitoPages = this.useIncognitoPages,
userDataDir = this.userDataDir,
browserPerProxy = this.browserPerProxy,
ignoreProxyCertificate = this.ignoreProxyCertificate,
isRemote,
} = options;
return new LaunchContext({
id,
launchOptions: merge({}, this.launchOptions, launchOptions),
browserPlugin: this,
proxyUrl,
useIncognitoPages,
userDataDir,
browserPerProxy,
ignoreProxyCertificate,
isRemote,
});
}
abstract createController(): BrowserController<
Library,
LibraryOptions,
LaunchResult,
NewPageOptions,
NewPageResult
>;
/**
* Launches the browser using provided launch context.
*/
async launch(
launchContext: LaunchContext<
Library,
LibraryOptions,
LaunchResult,
NewPageOptions,
NewPageResult
> = this.createLaunchContext(),
): Promise<LaunchResult> {
launchContext.launchOptions ??= {} as LibraryOptions;
const { proxyUrl, launchOptions } = launchContext;
if (proxyUrl) {
await this.addProxyToLaunchOptions(launchContext);
}
if (this.isChromiumBasedBrowser(launchContext)) {
// This will set the args for chromium based browsers to hide the webdriver.
(launchOptions as Dictionary).args = this.#mergeArgsToHideWebdriver(launchOptions!.args);
// When User-Agent is not set, and we're using Chromium in headless mode,
// it is better to use DEFAULT_USER_AGENT to reduce chance of detection,
// as otherwise 'HeadlessChrome' is present in User-Agent string.
const userAgent = launchOptions!.args.find((arg: string) => arg.startsWith('--user-agent'));
if (launchOptions!.headless && !launchContext.fingerprint && !userAgent) {
launchOptions!.args.push(`--user-agent=${DEFAULT_USER_AGENT}`);
}
}
return this._launch(launchContext);
}
#mergeArgsToHideWebdriver(originalArgs?: string[]): string[] {
if (!originalArgs?.length) {
return ['--disable-blink-features=AutomationControlled'];
}
const argumentIndex = originalArgs.findIndex((arg: string) => arg.startsWith('--disable-blink-features='));
if (argumentIndex !== -1) {
originalArgs[argumentIndex] += ',AutomationControlled';
} else {
originalArgs.push('--disable-blink-features=AutomationControlled');
}
return originalArgs;
}
protected throwAugmentedLaunchError(
cause: unknown,
executablePath: string | undefined,
dockerImage: string,
moduleInstallCommand: string,
): never {
const errorMessage = ['Failed to launch browser. Please check the following:'];
if (executablePath) {
errorMessage.push(`- Check whether the provided executable path "${executablePath}" is correct.`);
}
if (process.env.APIFY_IS_AT_HOME) {
errorMessage.push(`- Make sure your Dockerfile extends ${dockerImage}.`);
}
errorMessage.push(`- ${moduleInstallCommand}`);
errorMessage.push(
'',
'The original error is available in the `cause` property. Below is the error received when trying to launch a browser:',
'',
);
// Add in a zero-width space so we can remove it later when printing the error stack
throw new BrowserLaunchError(`${errorMessage.join('\n')}\u200b`, { cause });
}
protected abstract addProxyToLaunchOptions(
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>,
): Promise<void>;
protected abstract isChromiumBasedBrowser(
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>,
): boolean;
protected abstract _launch(
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>,
): Promise<LaunchResult>;
}
export class BrowserLaunchError extends CriticalError {
public constructor(...args: ConstructorParameters<typeof CriticalError>) {
super(...args);
this.name = 'BrowserLaunchError';
const [, oldStack] = this.stack?.split('\u200b') ?? [null, ''];
Object.defineProperty(this, 'stack', {
get: () => {
if (this.cause instanceof Error) {
return `${this.message}\n${this.cause.stack}\nError thrown at:\n${oldStack}`;
}
return `${this.message}\n${oldStack}`;
},
});
}
}