Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import type { Cookie } from '@crawlee/types';
import type { Browser, BrowserType, Page } from 'playwright';
import { tryCancel } from '@apify/timeout';
import { BrowserController } from '../abstract-classes/browser-controller.js';
import { anonymizeProxySugar } from '../anonymize-proxy.js';
import type { SafeParameters } from '../utils.js';
export class PlaywrightController extends BrowserController<
BrowserType,
SafeParameters<BrowserType['launch']>[0],
Browser
> {
/** @internal */
normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown> {
if (!proxyUrl) {
return {};
}
const url = new URL(proxyUrl);
const username = decodeURIComponent(url.username);
const password = decodeURIComponent(url.password);
return {
proxy: {
server: `${url.protocol}//${url.host}`,
username,
password,
bypass: pageOptions?.proxy?.bypass,
},
};
}
protected async _newPage(contextOptions?: SafeParameters<Browser['newPage']>[0]): Promise<Page> {
if (contextOptions !== undefined && !this.launchContext.useIncognitoPages) {
throw new Error('A new page can be created with provided context only when using incognito pages.');
}
let close = async () => {};
if (this.launchContext.useIncognitoPages) {
// Each page requires to have all the context options applied
contextOptions = {
...this.launchContext.launchOptions,
...contextOptions,
};
// Remote browsers handle their own proxy — don't inject local proxy settings into context
if (this.launchContext.isRemote) {
delete contextOptions?.proxy;
}
if (contextOptions?.proxy) {
const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(
contextOptions.proxy.server,
contextOptions.proxy.username,
contextOptions.proxy.password,
{ ignoreProxyCertificate: this.launchContext.ignoreProxyCertificate },
);
if (anonymizedProxyUrl) {
contextOptions.proxy = {
server: anonymizedProxyUrl,
bypass: contextOptions.proxy.bypass,
};
}
close = closeProxy;
}
}
try {
const page = await this.browser.newPage(contextOptions);
this.registerPageTeardown(page, close);
page.once('close', () => {
this.registerPageClosed(page);
});
tryCancel();
return page;
} catch (error) {
await close();
throw error;
}
}
protected async _close(): Promise<void> {
await this.browser.close();
}
protected async _kill(): Promise<void> {
// TODO: We need to be absolutely sure the browser dies.
await this.browser.close(); // Playwright does not have the browser child process attached to normal browser server
}
protected async _getCookies(page: Page): Promise<Cookie[]> {
const context = page.context();
return context.cookies();
}
protected async _setCookies(page: Page, cookies: Cookie[]): Promise<void> {
const context = page.context();
return context.addCookies(cookies);
}
}