Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL): better error for SSL failures (#1552)
Gergő Móricz3db2294b9712
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex fadce7793..4e64f74e8 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -7,6 +7,7 @@ import {ActionError,EngineError,SiteError,+ SSLError,UnsupportedFileError,} from "../../error";import { MockState } from "../../lib/mock";@@ -169,7 +170,13 @@ export async function fireEngineCheckStatus(typeof status.error === "string" &&status.error.includes("Chrome error: ")) {- throw new SiteError(status.error.split("Chrome error: ")[1]);+ const code = status.error.split("Chrome error: ")[1];++ if (code.includes("ERR_CERT_") || code.includes("ERR_SSL_") || code.includes("ERR_BAD_SSL_")) {+ throw new SSLError();+ } else {+ throw new SiteError(code);+ }} else if (typeof status.error === "string" &&status.error.includes("File size exceeds")diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex ff445f8dd..29b8970ec 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -49,6 +49,12 @@ export class RemoveFeatureError extends Error {}}+export class SSLError extends Error {+ constructor() {+ super("An SSL error occurred while scraping the URL. If you're not inputting any sensitive data, try scraping with `skipTlsVerification: true`.");+ }+}+export class SiteError extends Error {public code: string;constructor(code: string) {diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 7dfc821fe..baf29752a 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -21,6 +21,7 @@ import {SiteError,TimeoutError,UnsupportedFileError,+ SSLError,} from "./error";import { executeTransformers } from "./transformers";import { LLMRefusalError } from "./transformers/llmExtract";@@ -323,6 +324,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {throw error;} else if (error instanceof SiteError) {throw error;+ } else if (error instanceof SSLError) {+ throw error;} else if (error instanceof ActionError) {throw error;} else if (error instanceof UnsupportedFileError) {@@ -470,6 +473,8 @@ export async function scrapeURL(// TODO: results?} else if (error instanceof SiteError) {meta.logger.warn("scrapeURL: Site failed to load in browser", { error });+ } else if (error instanceof SSLError) {+ meta.logger.warn("scrapeURL: SSL error", { error });} else if (error instanceof ActionError) {meta.logger.warn("scrapeURL: Action(s) failed to complete", { error });} else if (error instanceof UnsupportedFileError) {