Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: add `ignoreProxyCertificate` option for the internal `proxy-chain` instance (#3418)
Jindřich Bär02eec66ecfae
diff --git a/packages/browser-crawler/src/internals/browser-launcher.ts b/packages/browser-crawler/src/internals/browser-launcher.tsindex 1b68b1bb3..e20eb7348 100644--- a/packages/browser-crawler/src/internals/browser-launcher.ts+++ b/packages/browser-crawler/src/internals/browser-launcher.ts@@ -67,6 +67,12 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginO*/userAgent?: string;+ /**+ * If set to `true`, TLS certificate errors from the upstream proxy will be ignored.+ * This is useful when using HTTPS proxies with self-signed certificates.+ */+ ignoreProxyCertificate?: boolean;+/*** The type of browser to be launched.* By default, `chromium` is used. Other browsers like `webkit` or `firefox` can be used.@@ -108,6 +114,7 @@ export abstract class BrowserLauncher<useIncognitoPages: ow.optional.boolean,browserPerProxy: ow.optional.boolean,experimentalContainers: ow.optional.boolean,+ ignoreProxyCertificate: ow.optional.boolean,userDataDir: ow.optional.string,launchOptions: ow.optional.object,userAgent: ow.optional.string,diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.tsindex 7b95dd755..da13dc8c0 100644--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts@@ -82,6 +82,11 @@ export interface BrowserPluginOptions<LibraryOptions> {* Might cause performance issues, as Crawlee might launch too many browser instances.*/browserPerProxy?: boolean;+ /**+ * If set to `true`, TLS certificate errors from the upstream proxy will be ignored.+ * This is useful when using HTTPS proxies with self-signed certificates.+ */+ ignoreProxyCertificate?: boolean;}export interface CreateLaunchContextOptions<@@ -126,6 +131,8 @@ export abstract class BrowserPlugin<browserPerProxy?: boolean;+ ignoreProxyCertificate?: boolean;+constructor(library: Library, options: BrowserPluginOptions<LibraryOptions> = {}) {const {launchOptions = {} as LibraryOptions,@@ -134,6 +141,7 @@ export abstract class BrowserPlugin<useIncognitoPages = false,experimentalContainers = false,browserPerProxy = false,+ ignoreProxyCertificate = false,} = options;this.library = library;@@ -143,6 +151,7 @@ export abstract class BrowserPlugin<this.useIncognitoPages = useIncognitoPages;this.experimentalContainers = experimentalContainers;this.browserPerProxy = browserPerProxy;+ this.ignoreProxyCertificate = ignoreProxyCertificate;}/**@@ -162,6 +171,7 @@ export abstract class BrowserPlugin<userDataDir = this.userDataDir,experimentalContainers = this.experimentalContainers,browserPerProxy = this.browserPerProxy,+ ignoreProxyCertificate = this.ignoreProxyCertificate,proxyTier,} = options;@@ -174,6 +184,7 @@ export abstract class BrowserPlugin<experimentalContainers,userDataDir,browserPerProxy,+ ignoreProxyCertificate,proxyTier,});}diff --git a/packages/browser-pool/src/anonymize-proxy.ts b/packages/browser-pool/src/anonymize-proxy.tsindex ae3d39839..d8bc12caf 100644--- a/packages/browser-pool/src/anonymize-proxy.ts+++ b/packages/browser-pool/src/anonymize-proxy.ts@@ -2,11 +2,16 @@ import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';type PromiseVoid = () => Promise<void>;+export interface AnonymizeProxySugarOptions {+ ignoreProxyCertificate?: boolean;+}+/** @internal */export const anonymizeProxySugar = async (proxyUrl?: string,username?: string,password?: string,+ options?: AnonymizeProxySugarOptions,): Promise<[string | undefined, PromiseVoid]> => {if (proxyUrl) {const url = new URL(proxyUrl);@@ -16,10 +21,14 @@ export const anonymizeProxySugar = async (url.password = password ?? '';}- if (url.username || url.password) {+ if (url.username || url.password || options?.ignoreProxyCertificate) {// trim off trailing slash if it's presentconst proxyUrlString = url.href.endsWith('/') ? url.href.slice(0, -1) : url.href;- const anonymized = await anonymizeProxy(proxyUrlString);+ const anonymized = await anonymizeProxy({+ url: proxyUrlString,+ port: 0,+ ignoreProxyCertificate: options?.ignoreProxyCertificate ?? false,+ });return [anonymized,diff --git a/packages/browser-pool/src/index.ts b/packages/browser-pool/src/index.tsindex 9e23f2e05..81c8c4e62 100644--- a/packages/browser-pool/src/index.ts+++ b/packages/browser-pool/src/index.ts@@ -52,4 +52,4 @@ export {GetFingerprintReturn,} from './fingerprinting/types';export { InferBrowserPluginArray, UnwrapPromise } from './utils';-export { anonymizeProxySugar } from './anonymize-proxy';+export { anonymizeProxySugar, type AnonymizeProxySugarOptions } from './anonymize-proxy';diff --git a/packages/browser-pool/src/launch-context.ts b/packages/browser-pool/src/launch-context.tsindex e7cbdfbb4..669d78937 100644--- a/packages/browser-pool/src/launch-context.ts+++ b/packages/browser-pool/src/launch-context.ts@@ -58,6 +58,11 @@ export interface LaunchContextOptions<userDataDir?: string;proxyUrl?: string;proxyTier?: number;+ /**+ * If set to `true`, TLS certificate errors from the upstream proxy will be ignored.+ * This is useful when using HTTPS proxies with self-signed certificates.+ */+ ignoreProxyCertificate?: boolean;}export class LaunchContext<@@ -75,6 +80,7 @@ export class LaunchContext<experimentalContainers: boolean;userDataDir: string;proxyTier?: number;+ ignoreProxyCertificate?: boolean;private _proxyUrl?: string;private readonly _reservedFieldNames = [...Reflect.ownKeys(this), 'extend'];@@ -93,6 +99,7 @@ export class LaunchContext<experimentalContainers,userDataDir = '',proxyTier,+ ignoreProxyCertificate,} = options;this.id = id;@@ -103,6 +110,7 @@ export class LaunchContext<this.experimentalContainers = experimentalContainers ?? false;this.userDataDir = userDataDir;this.proxyTier = proxyTier;+ this.ignoreProxyCertificate = ignoreProxyCertificate ?? false;this._proxyUrl = proxyUrl;}diff --git a/packages/browser-pool/src/playwright/playwright-controller.ts b/packages/browser-pool/src/playwright/playwright-controller.tsindex 7dc27f7c4..a04d007ca 100644--- a/packages/browser-pool/src/playwright/playwright-controller.ts+++ b/packages/browser-pool/src/playwright/playwright-controller.ts@@ -60,6 +60,7 @@ export class PlaywrightController extends BrowserController<contextOptions.proxy.server,contextOptions.proxy.username,contextOptions.proxy.password,+ { ignoreProxyCertificate: this.launchContext.ignoreProxyCertificate },);if (anonymizedProxyUrl) {diff --git a/packages/browser-pool/src/playwright/playwright-plugin.ts b/packages/browser-pool/src/playwright/playwright-plugin.tsindex 5e59d4656..0781dcac7 100644--- a/packages/browser-pool/src/playwright/playwright-plugin.ts+++ b/packages/browser-pool/src/playwright/playwright-plugin.ts@@ -59,7 +59,9 @@ export class PlaywrightPlugin extends BrowserPlugin<launchOptions!.args = launchOptions!.args?.filter((arg) => arg !== '--no-sandbox');}- const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);+ const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {+ ignoreProxyCertificate: launchContext.ignoreProxyCertificate,+ });if (anonymizedProxyUrl) {launchOptions!.proxy = {server: anonymizedProxyUrl,diff --git a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts b/packages/browser-pool/src/puppeteer/puppeteer-controller.tsindex d51a8ef1d..1e825f4e6 100644--- a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts+++ b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts@@ -52,6 +52,7 @@ export class PuppeteerController extends BrowserController<contextOptions.proxyServer,contextOptions.proxyUsername,contextOptions.proxyPassword,+ { ignoreProxyCertificate: this.launchContext.ignoreProxyCertificate },);if (anonymizedProxyUrl) {diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.tsindex f273916c8..8a5698a2c 100644--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts@@ -39,7 +39,14 @@ export class PuppeteerPlugin extends BrowserPlugin<} catch {// ignore}- const { launchOptions, userDataDir, useIncognitoPages, experimentalContainers, proxyUrl } = launchContext;+ const {+ launchOptions,+ userDataDir,+ useIncognitoPages,+ experimentalContainers,+ proxyUrl,+ ignoreProxyCertificate,+ } = launchContext;if (experimentalContainers) {throw new Error('Experimental containers are only available with Playwright');@@ -62,7 +69,9 @@ export class PuppeteerPlugin extends BrowserPlugin<let browser: PuppeteerTypes.Browser;{- const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);+ const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {+ ignoreProxyCertificate: launchContext.ignoreProxyCertificate,+ });if (proxyUrl) {const proxyArg = `${PROXY_SERVER_ARG}${anonymizedProxyUrl ?? proxyUrl}`;@@ -133,7 +142,12 @@ export class PuppeteerPlugin extends BrowserPlugin<let page: PuppeteerTypes.Page;if (useIncognitoPages) {- const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);+ const [anonymizedProxyUrl, close] = await anonymizeProxySugar(+ proxyUrl,+ undefined,+ undefined,+ { ignoreProxyCertificate },+ );try {const context = (await (browser as any)[method]({diff --git a/packages/stagehand-crawler/src/internals/stagehand-plugin.ts b/packages/stagehand-crawler/src/internals/stagehand-plugin.tsindex 763fdc247..2cdc9bec9 100644--- a/packages/stagehand-crawler/src/internals/stagehand-plugin.ts+++ b/packages/stagehand-crawler/src/internals/stagehand-plugin.ts@@ -57,7 +57,9 @@ export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, Pconst isLocal = this.stagehandOptions.env === 'LOCAL' || !this.stagehandOptions.env;// Use anonymizeProxy to handle proxy authentication transparently- const [anonymizedProxyUrl, closeAnonymizedProxy] = await anonymizeProxySugar(proxyUrl);+ const [anonymizedProxyUrl, closeAnonymizedProxy] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {+ ignoreProxyCertificate: launchContext.ignoreProxyCertificate,+ });// Build model configuration// For LOCAL env, we merge apiKey into the model config since Stagehand expects it therediff --git a/test/browser-pool/anonymize-proxy-sugar.test.ts b/test/browser-pool/anonymize-proxy-sugar.test.tsindex af16a757e..6b439ece5 100644--- a/test/browser-pool/anonymize-proxy-sugar.test.ts+++ b/test/browser-pool/anonymize-proxy-sugar.test.ts@@ -7,7 +7,7 @@ describe('anonymizeProxySugar', () => {// Mock the anonymizeProxy function from proxy-chainbeforeEach(() => {vi.mock('proxy-chain', () => ({- anonymizeProxy: vi.fn((url) => Promise.resolve(`anonymized-${url}`)),+ anonymizeProxy: vi.fn((opts) => Promise.resolve(`anonymized-${opts.url}`)),}));});@@ -23,7 +23,33 @@ describe('anonymizeProxySugar', () => {])('should call anonymizeProxy from proxy-chain with correctly pre-processed URL: %s', async (input, expectedOutput) => {const [anonymized] = await anonymizeProxySugar(input);- expect(anonymizeProxy).toHaveBeenCalledWith(expectedOutput);+ expect(anonymizeProxy).toHaveBeenCalledWith(expect.objectContaining({ url: expectedOutput }));expect(anonymized).toBeTypeOf('string');});++ test('should pass ignoreProxyCertificate to anonymizeProxy', async () => {+ await anonymizeProxySugar('http://username:password@proxy:1000', undefined, undefined, {+ ignoreProxyCertificate: true,+ });++ expect(anonymizeProxy).toHaveBeenCalledWith(expect.objectContaining({ ignoreProxyCertificate: true }));+ });++ test('should anonymize proxy without credentials when ignoreProxyCertificate is set', async () => {+ const [anonymized] = await anonymizeProxySugar('http://proxy:1000', undefined, undefined, {+ ignoreProxyCertificate: true,+ });++ expect(anonymizeProxy).toHaveBeenCalledWith(+ expect.objectContaining({ url: 'http://proxy:1000', ignoreProxyCertificate: true }),+ );+ expect(anonymized).toBeTypeOf('string');+ });++ test('should not anonymize proxy without credentials and without ignoreProxyCertificate', async () => {+ const [anonymized] = await anonymizeProxySugar('http://proxy:1000');++ expect(anonymizeProxy).not.toHaveBeenCalled();+ expect(anonymized).toBeUndefined();+ });});