Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add `ignoreProxyCertificate` option for the internal `proxy-chain` instance (#3418)

Jindřich Bär02eec66ecfae
diff --git a/packages/browser-crawler/src/internals/browser-launcher.ts b/packages/browser-crawler/src/internals/browser-launcher.ts
index 1b68b1bb3..e20eb7348 100644
--- a/packages/browser-crawler/src/internals/browser-launcher.ts
+++ b/packages/browser-crawler/src/internals/browser-launcher.ts
@@ -67,6 +67,12 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginO
*/
userAgent?: string;
+ /**
+ * If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
+ * This is useful when using HTTPS proxies with self-signed certificates.
+ */
+ ignoreProxyCertificate?: boolean;
+
/**
* The type of browser to be launched.
* By default, `chromium` is used. Other browsers like `webkit` or `firefox` can be used.
@@ -108,6 +114,7 @@ export abstract class BrowserLauncher<
useIncognitoPages: ow.optional.boolean,
browserPerProxy: ow.optional.boolean,
experimentalContainers: ow.optional.boolean,
+ ignoreProxyCertificate: ow.optional.boolean,
userDataDir: ow.optional.string,
launchOptions: ow.optional.object,
userAgent: ow.optional.string,
diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.ts
index 7b95dd755..da13dc8c0 100644
--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts
+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts
@@ -82,6 +82,11 @@ export interface BrowserPluginOptions<LibraryOptions> {
* Might cause performance issues, as Crawlee might launch too many browser instances.
*/
browserPerProxy?: boolean;
+ /**
+ * If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
+ * This is useful when using HTTPS proxies with self-signed certificates.
+ */
+ ignoreProxyCertificate?: boolean;
}
export interface CreateLaunchContextOptions<
@@ -126,6 +131,8 @@ export abstract class BrowserPlugin<
browserPerProxy?: boolean;
+ ignoreProxyCertificate?: boolean;
+
constructor(library: Library, options: BrowserPluginOptions<LibraryOptions> = {}) {
const {
launchOptions = {} as LibraryOptions,
@@ -134,6 +141,7 @@ export abstract class BrowserPlugin<
useIncognitoPages = false,
experimentalContainers = false,
browserPerProxy = false,
+ ignoreProxyCertificate = false,
} = options;
this.library = library;
@@ -143,6 +151,7 @@ export abstract class BrowserPlugin<
this.useIncognitoPages = useIncognitoPages;
this.experimentalContainers = experimentalContainers;
this.browserPerProxy = browserPerProxy;
+ this.ignoreProxyCertificate = ignoreProxyCertificate;
}
/**
@@ -162,6 +171,7 @@ export abstract class BrowserPlugin<
userDataDir = this.userDataDir,
experimentalContainers = this.experimentalContainers,
browserPerProxy = this.browserPerProxy,
+ ignoreProxyCertificate = this.ignoreProxyCertificate,
proxyTier,
} = options;
@@ -174,6 +184,7 @@ export abstract class BrowserPlugin<
experimentalContainers,
userDataDir,
browserPerProxy,
+ ignoreProxyCertificate,
proxyTier,
});
}
diff --git a/packages/browser-pool/src/anonymize-proxy.ts b/packages/browser-pool/src/anonymize-proxy.ts
index ae3d39839..d8bc12caf 100644
--- a/packages/browser-pool/src/anonymize-proxy.ts
+++ b/packages/browser-pool/src/anonymize-proxy.ts
@@ -2,11 +2,16 @@ import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';
type PromiseVoid = () => Promise<void>;
+export interface AnonymizeProxySugarOptions {
+ ignoreProxyCertificate?: boolean;
+}
+
/** @internal */
export const anonymizeProxySugar = async (
proxyUrl?: string,
username?: string,
password?: string,
+ options?: AnonymizeProxySugarOptions,
): Promise<[string | undefined, PromiseVoid]> => {
if (proxyUrl) {
const url = new URL(proxyUrl);
@@ -16,10 +21,14 @@ export const anonymizeProxySugar = async (
url.password = password ?? '';
}
- if (url.username || url.password) {
+ if (url.username || url.password || options?.ignoreProxyCertificate) {
// trim off trailing slash if it's present
const proxyUrlString = url.href.endsWith('/') ? url.href.slice(0, -1) : url.href;
- const anonymized = await anonymizeProxy(proxyUrlString);
+ const anonymized = await anonymizeProxy({
+ url: proxyUrlString,
+ port: 0,
+ ignoreProxyCertificate: options?.ignoreProxyCertificate ?? false,
+ });
return [
anonymized,
diff --git a/packages/browser-pool/src/index.ts b/packages/browser-pool/src/index.ts
index 9e23f2e05..81c8c4e62 100644
--- a/packages/browser-pool/src/index.ts
+++ b/packages/browser-pool/src/index.ts
@@ -52,4 +52,4 @@ export {
GetFingerprintReturn,
} from './fingerprinting/types';
export { InferBrowserPluginArray, UnwrapPromise } from './utils';
-export { anonymizeProxySugar } from './anonymize-proxy';
+export { anonymizeProxySugar, type AnonymizeProxySugarOptions } from './anonymize-proxy';
diff --git a/packages/browser-pool/src/launch-context.ts b/packages/browser-pool/src/launch-context.ts
index e7cbdfbb4..669d78937 100644
--- a/packages/browser-pool/src/launch-context.ts
+++ b/packages/browser-pool/src/launch-context.ts
@@ -58,6 +58,11 @@ export interface LaunchContextOptions<
userDataDir?: string;
proxyUrl?: string;
proxyTier?: number;
+ /**
+ * If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
+ * This is useful when using HTTPS proxies with self-signed certificates.
+ */
+ ignoreProxyCertificate?: boolean;
}
export class LaunchContext<
@@ -75,6 +80,7 @@ export class LaunchContext<
experimentalContainers: boolean;
userDataDir: string;
proxyTier?: number;
+ ignoreProxyCertificate?: boolean;
private _proxyUrl?: string;
private readonly _reservedFieldNames = [...Reflect.ownKeys(this), 'extend'];
@@ -93,6 +99,7 @@ export class LaunchContext<
experimentalContainers,
userDataDir = '',
proxyTier,
+ ignoreProxyCertificate,
} = options;
this.id = id;
@@ -103,6 +110,7 @@ export class LaunchContext<
this.experimentalContainers = experimentalContainers ?? false;
this.userDataDir = userDataDir;
this.proxyTier = proxyTier;
+ this.ignoreProxyCertificate = ignoreProxyCertificate ?? false;
this._proxyUrl = proxyUrl;
}
diff --git a/packages/browser-pool/src/playwright/playwright-controller.ts b/packages/browser-pool/src/playwright/playwright-controller.ts
index 7dc27f7c4..a04d007ca 100644
--- a/packages/browser-pool/src/playwright/playwright-controller.ts
+++ b/packages/browser-pool/src/playwright/playwright-controller.ts
@@ -60,6 +60,7 @@ export class PlaywrightController extends BrowserController<
contextOptions.proxy.server,
contextOptions.proxy.username,
contextOptions.proxy.password,
+ { ignoreProxyCertificate: this.launchContext.ignoreProxyCertificate },
);
if (anonymizedProxyUrl) {
diff --git a/packages/browser-pool/src/playwright/playwright-plugin.ts b/packages/browser-pool/src/playwright/playwright-plugin.ts
index 5e59d4656..0781dcac7 100644
--- a/packages/browser-pool/src/playwright/playwright-plugin.ts
+++ b/packages/browser-pool/src/playwright/playwright-plugin.ts
@@ -59,7 +59,9 @@ export class PlaywrightPlugin extends BrowserPlugin<
launchOptions!.args = launchOptions!.args?.filter((arg) => arg !== '--no-sandbox');
}
- const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);
+ const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {
+ ignoreProxyCertificate: launchContext.ignoreProxyCertificate,
+ });
if (anonymizedProxyUrl) {
launchOptions!.proxy = {
server: anonymizedProxyUrl,
diff --git a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts
index d51a8ef1d..1e825f4e6 100644
--- a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts
+++ b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts
@@ -52,6 +52,7 @@ export class PuppeteerController extends BrowserController<
contextOptions.proxyServer,
contextOptions.proxyUsername,
contextOptions.proxyPassword,
+ { ignoreProxyCertificate: this.launchContext.ignoreProxyCertificate },
);
if (anonymizedProxyUrl) {
diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
index f273916c8..8a5698a2c 100644
--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
@@ -39,7 +39,14 @@ export class PuppeteerPlugin extends BrowserPlugin<
} catch {
// ignore
}
- const { launchOptions, userDataDir, useIncognitoPages, experimentalContainers, proxyUrl } = launchContext;
+ const {
+ launchOptions,
+ userDataDir,
+ useIncognitoPages,
+ experimentalContainers,
+ proxyUrl,
+ ignoreProxyCertificate,
+ } = launchContext;
if (experimentalContainers) {
throw new Error('Experimental containers are only available with Playwright');
@@ -62,7 +69,9 @@ export class PuppeteerPlugin extends BrowserPlugin<
let browser: PuppeteerTypes.Browser;
{
- const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);
+ const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {
+ ignoreProxyCertificate: launchContext.ignoreProxyCertificate,
+ });
if (proxyUrl) {
const proxyArg = `${PROXY_SERVER_ARG}${anonymizedProxyUrl ?? proxyUrl}`;
@@ -133,7 +142,12 @@ export class PuppeteerPlugin extends BrowserPlugin<
let page: PuppeteerTypes.Page;
if (useIncognitoPages) {
- const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);
+ const [anonymizedProxyUrl, close] = await anonymizeProxySugar(
+ proxyUrl,
+ undefined,
+ undefined,
+ { ignoreProxyCertificate },
+ );
try {
const context = (await (browser as any)[method]({
diff --git a/packages/stagehand-crawler/src/internals/stagehand-plugin.ts b/packages/stagehand-crawler/src/internals/stagehand-plugin.ts
index 763fdc247..2cdc9bec9 100644
--- a/packages/stagehand-crawler/src/internals/stagehand-plugin.ts
+++ b/packages/stagehand-crawler/src/internals/stagehand-plugin.ts
@@ -57,7 +57,9 @@ export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, P
const isLocal = this.stagehandOptions.env === 'LOCAL' || !this.stagehandOptions.env;
// Use anonymizeProxy to handle proxy authentication transparently
- const [anonymizedProxyUrl, closeAnonymizedProxy] = await anonymizeProxySugar(proxyUrl);
+ const [anonymizedProxyUrl, closeAnonymizedProxy] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {
+ ignoreProxyCertificate: launchContext.ignoreProxyCertificate,
+ });
// Build model configuration
// For LOCAL env, we merge apiKey into the model config since Stagehand expects it there
diff --git a/test/browser-pool/anonymize-proxy-sugar.test.ts b/test/browser-pool/anonymize-proxy-sugar.test.ts
index af16a757e..6b439ece5 100644
--- a/test/browser-pool/anonymize-proxy-sugar.test.ts
+++ b/test/browser-pool/anonymize-proxy-sugar.test.ts
@@ -7,7 +7,7 @@ describe('anonymizeProxySugar', () => {
// Mock the anonymizeProxy function from proxy-chain
beforeEach(() => {
vi.mock('proxy-chain', () => ({
- anonymizeProxy: vi.fn((url) => Promise.resolve(`anonymized-${url}`)),
+ anonymizeProxy: vi.fn((opts) => Promise.resolve(`anonymized-${opts.url}`)),
}));
});
@@ -23,7 +23,33 @@ describe('anonymizeProxySugar', () => {
])('should call anonymizeProxy from proxy-chain with correctly pre-processed URL: %s', async (input, expectedOutput) => {
const [anonymized] = await anonymizeProxySugar(input);
- expect(anonymizeProxy).toHaveBeenCalledWith(expectedOutput);
+ expect(anonymizeProxy).toHaveBeenCalledWith(expect.objectContaining({ url: expectedOutput }));
expect(anonymized).toBeTypeOf('string');
});
+
+ test('should pass ignoreProxyCertificate to anonymizeProxy', async () => {
+ await anonymizeProxySugar('http://username:password@proxy:1000', undefined, undefined, {
+ ignoreProxyCertificate: true,
+ });
+
+ expect(anonymizeProxy).toHaveBeenCalledWith(expect.objectContaining({ ignoreProxyCertificate: true }));
+ });
+
+ test('should anonymize proxy without credentials when ignoreProxyCertificate is set', async () => {
+ const [anonymized] = await anonymizeProxySugar('http://proxy:1000', undefined, undefined, {
+ ignoreProxyCertificate: true,
+ });
+
+ expect(anonymizeProxy).toHaveBeenCalledWith(
+ expect.objectContaining({ url: 'http://proxy:1000', ignoreProxyCertificate: true }),
+ );
+ expect(anonymized).toBeTypeOf('string');
+ });
+
+ test('should not anonymize proxy without credentials and without ignoreProxyCertificate', async () => {
+ const [anonymized] = await anonymizeProxySugar('http://proxy:1000');
+
+ expect(anonymizeProxy).not.toHaveBeenCalled();
+ expect(anonymized).toBeUndefined();
+ });
});