Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrapeURL): better error for SSL failures (#1552)

Gergő Móricz3db2294b9712
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index fadce7793..4e64f74e8 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -7,6 +7,7 @@ import {
ActionError,
EngineError,
SiteError,
+ SSLError,
UnsupportedFileError,
} from "../../error";
import { MockState } from "../../lib/mock";
@@ -169,7 +170,13 @@ export async function fireEngineCheckStatus(
typeof status.error === "string" &&
status.error.includes("Chrome error: ")
) {
- throw new SiteError(status.error.split("Chrome error: ")[1]);
+ const code = status.error.split("Chrome error: ")[1];
+
+ if (code.includes("ERR_CERT_") || code.includes("ERR_SSL_") || code.includes("ERR_BAD_SSL_")) {
+ throw new SSLError();
+ } else {
+ throw new SiteError(code);
+ }
} else if (
typeof status.error === "string" &&
status.error.includes("File size exceeds")
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index ff445f8dd..29b8970ec 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -49,6 +49,12 @@ export class RemoveFeatureError extends Error {
}
}
+export class SSLError extends Error {
+ constructor() {
+ super("An SSL error occurred while scraping the URL. If you're not inputting any sensitive data, try scraping with `skipTlsVerification: true`.");
+ }
+}
+
export class SiteError extends Error {
public code: string;
constructor(code: string) {
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 7dfc821fe..baf29752a 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -21,6 +21,7 @@ import {
SiteError,
TimeoutError,
UnsupportedFileError,
+ SSLError,
} from "./error";
import { executeTransformers } from "./transformers";
import { LLMRefusalError } from "./transformers/llmExtract";
@@ -323,6 +324,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
throw error;
} else if (error instanceof SiteError) {
throw error;
+ } else if (error instanceof SSLError) {
+ throw error;
} else if (error instanceof ActionError) {
throw error;
} else if (error instanceof UnsupportedFileError) {
@@ -470,6 +473,8 @@ export async function scrapeURL(
// TODO: results?
} else if (error instanceof SiteError) {
meta.logger.warn("scrapeURL: Site failed to load in browser", { error });
+ } else if (error instanceof SSLError) {
+ meta.logger.warn("scrapeURL: SSL error", { error });
} else if (error instanceof ActionError) {
meta.logger.warn("scrapeURL: Action(s) failed to complete", { error });
} else if (error instanceof UnsupportedFileError) {