Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
force skipTlsVerification to false for zdr (#5196)
rowinsg7ccc8f4e9957
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 6f8a7d79c..b0703656b 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -94,13 +94,18 @@ export class RemoveFeatureError extends Error {}export class SSLError extends TransportableError {- constructor(public skipTlsVerification: boolean) {+ constructor(+ public skipTlsVerification: boolean,+ public zeroDataRetention = false,+ ) {super("SCRAPE_SSL_ERROR","An SSL/TLS certificate error occurred while trying to establish a secure connection to this website. " +- (skipTlsVerification- ? "You already have `skipTlsVerification: true` enabled, which means the website's TLS configuration is severely broken (not just an expired or self-signed certificate). Possible solutions: (1) Try the plain HTTP version of the URL (http:// instead of https://), (2) The website may be completely down, or (3) Contact the website administrator about their broken SSL configuration."- : "This usually happens when a website has an expired, self-signed, or misconfigured SSL certificate. If you trust this website and are not submitting sensitive data, you can bypass this error by setting `skipTlsVerification: true` in your scrape request. Note: Only do this for trusted sites as it disables certificate validation."),+ (zeroDataRetention+ ? "This usually happens when a website has an expired, self-signed, or misconfigured SSL certificate. TLS verification can't be bypassed for zero data retention requests, so `skipTlsVerification` has no effect."+ : skipTlsVerification+ ? "You already have `skipTlsVerification: true` enabled, which means the website's TLS configuration is severely broken (not just an expired or self-signed certificate). Possible solutions: (1) Try the plain HTTP version of the URL (http:// instead of https://), (2) The website may be completely down, or (3) Contact the website administrator about their broken SSL configuration."+ : "This usually happens when a website has an expired, self-signed, or misconfigured SSL certificate. If you trust this website and are not submitting sensitive data, you can bypass this error by setting `skipTlsVerification: true` in your scrape request. Note: Only do this for trusted sites as it disables certificate validation."),);}@@ -108,6 +113,7 @@ export class SSLError extends TransportableError {return {...super.serialize(),skipTlsVerification: this.skipTlsVerification,+ zeroDataRetention: this.zeroDataRetention,};}@@ -115,7 +121,7 @@ export class SSLError extends TransportableError {_: ErrorCodes,data: ReturnType<typeof this.prototype.serialize>,) {- const x = new SSLError(data.skipTlsVerification);+ const x = new SSLError(data.skipTlsVerification, data.zeroDataRetention);x.stack = data.stack;return x;}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex c5d7bda25..257d20071 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -1307,6 +1307,12 @@ export async function scrapeURL(}}+ // ZDR always verifies TLS, for every engine. Explicit false also wins over+ // the skip-by-default in applyScrapeOptionsDefaults.+ if (internalOptions.zeroDataRetention) {+ options.skipTlsVerification = false;+ }+const meta = await buildMetaObject(id,url,@@ -1852,6 +1858,13 @@ export async function scrapeURL(});} else if (error instanceof SSLError) {errorType = "SSLError";+ // Engines don't know about ZDR; swap in the message that doesn't+ // suggest skipTlsVerification, which ZDR overrides above.+ if (meta.internalOptions.zeroDataRetention) {+ const zdrError = new SSLError(false, true);+ zdrError.stack = error.stack;+ error = zdrError;+ }meta.logger.warn("scrapeURL: SSL error", { error });} else if (error instanceof ActionError) {errorType = "ActionError";