Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Disable SSL validation on MITM proxies (#1117)

Szymon Marczak853c5cd474bf
diff --git a/src/crawlers/browser_crawler.js b/src/crawlers/browser_crawler.js
index 53383353d..dca7108d4 100644
--- a/src/crawlers/browser_crawler.js
+++ b/src/crawlers/browser_crawler.js
@@ -508,6 +508,15 @@ export default class BrowserCrawler extends BasicCrawler {
const proxyInfo = this.proxyConfiguration.newProxyInfo(launchContextExtends.session && launchContextExtends.session.id);
launchContext.proxyUrl = proxyInfo.url;
launchContextExtends.proxyInfo = proxyInfo;
+
+ // Disable SSL verification for MITM proxies
+ if (this.proxyConfiguration.isManInTheMiddle) {
+ /**
+ * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
+ * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
+ */
+ launchContext.launchOptions.ignoreHTTPSErrors = true;
+ }
}
launchContext.extend(launchContextExtends);
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
index fd3453ce7..32ebecd77 100644
--- a/src/crawlers/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -740,6 +740,14 @@ class CheerioCrawler extends BasicCrawler {
},
};
+ // Disable SSL verification for MITM proxies
+ if (this.proxyConfiguration && this.proxyConfiguration.isManInTheMiddle) {
+ mandatoryRequestOptions.https = {
+ ...mandatoryRequestOptions.https,
+ rejectUnauthorized: false,
+ };
+ }
+
if (/PATCH|POST|PUT/.test(request.method)) mandatoryRequestOptions.payload = request.payload;
return { ...this.requestOptions, ...mandatoryRequestOptions };
diff --git a/src/proxy_configuration.js b/src/proxy_configuration.js
index 6167d21db..234df27a0 100644
--- a/src/proxy_configuration.js
+++ b/src/proxy_configuration.js
@@ -195,6 +195,7 @@ export class ProxyConfiguration {
this.usesApifyProxy = !this.proxyUrls && !this.newUrlFunction;
this.log = defaultLog.child({ prefix: 'ProxyConfiguration' });
this.config = config;
+ this.isManInTheMiddle = false;
}
/**
@@ -348,7 +349,9 @@ export class ProxyConfiguration {
async _checkAccess() {
const status = await this._fetchStatus();
if (status) {
- const { connected, connectionError } = status;
+ const { connected, connectionError, isManInTheMiddle } = status;
+ this.isManInTheMiddle = isManInTheMiddle;
+
if (!connected) this._throwApifyProxyConnectionError(connectionError);
} else {
this.log.warning('Apify Proxy access check timed out. Watch out for errors with status code 407. '