Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

force skipTlsVerification to false for zdr (#5196)

rowinsg7ccc8f4e9957
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 6f8a7d79c..b0703656b 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -94,13 +94,18 @@ export class RemoveFeatureError extends Error {
}
export class SSLError extends TransportableError {
- constructor(public skipTlsVerification: boolean) {
+ constructor(
+ public skipTlsVerification: boolean,
+ public zeroDataRetention = false,
+ ) {
super(
"SCRAPE_SSL_ERROR",
"An SSL/TLS certificate error occurred while trying to establish a secure connection to this website. " +
- (skipTlsVerification
- ? "You already have `skipTlsVerification: true` enabled, which means the website's TLS configuration is severely broken (not just an expired or self-signed certificate). Possible solutions: (1) Try the plain HTTP version of the URL (http:// instead of https://), (2) The website may be completely down, or (3) Contact the website administrator about their broken SSL configuration."
- : "This usually happens when a website has an expired, self-signed, or misconfigured SSL certificate. If you trust this website and are not submitting sensitive data, you can bypass this error by setting `skipTlsVerification: true` in your scrape request. Note: Only do this for trusted sites as it disables certificate validation."),
+ (zeroDataRetention
+ ? "This usually happens when a website has an expired, self-signed, or misconfigured SSL certificate. TLS verification can't be bypassed for zero data retention requests, so `skipTlsVerification` has no effect."
+ : skipTlsVerification
+ ? "You already have `skipTlsVerification: true` enabled, which means the website's TLS configuration is severely broken (not just an expired or self-signed certificate). Possible solutions: (1) Try the plain HTTP version of the URL (http:// instead of https://), (2) The website may be completely down, or (3) Contact the website administrator about their broken SSL configuration."
+ : "This usually happens when a website has an expired, self-signed, or misconfigured SSL certificate. If you trust this website and are not submitting sensitive data, you can bypass this error by setting `skipTlsVerification: true` in your scrape request. Note: Only do this for trusted sites as it disables certificate validation."),
);
}
@@ -108,6 +113,7 @@ export class SSLError extends TransportableError {
return {
...super.serialize(),
skipTlsVerification: this.skipTlsVerification,
+ zeroDataRetention: this.zeroDataRetention,
};
}
@@ -115,7 +121,7 @@ export class SSLError extends TransportableError {
_: ErrorCodes,
data: ReturnType<typeof this.prototype.serialize>,
) {
- const x = new SSLError(data.skipTlsVerification);
+ const x = new SSLError(data.skipTlsVerification, data.zeroDataRetention);
x.stack = data.stack;
return x;
}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index c5d7bda25..257d20071 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -1307,6 +1307,12 @@ export async function scrapeURL(
}
}
+ // ZDR always verifies TLS, for every engine. Explicit false also wins over
+ // the skip-by-default in applyScrapeOptionsDefaults.
+ if (internalOptions.zeroDataRetention) {
+ options.skipTlsVerification = false;
+ }
+
const meta = await buildMetaObject(
id,
url,
@@ -1852,6 +1858,13 @@ export async function scrapeURL(
});
} else if (error instanceof SSLError) {
errorType = "SSLError";
+ // Engines don't know about ZDR; swap in the message that doesn't
+ // suggest skipTlsVerification, which ZDR overrides above.
+ if (meta.internalOptions.zeroDataRetention) {
+ const zdrError = new SSLError(false, true);
+ zdrError.stack = error.stack;
+ error = zdrError;
+ }
meta.logger.warn("scrapeURL: SSL error", { error });
} else if (error instanceof ActionError) {
errorType = "ActionError";