Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): use TransportableError for branding validation errors (#2923)
Abimael Martell7b7a736d7038
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex da33785a4..f93fd1b9a 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -27,6 +27,7 @@ import {NoCachedDataError,ScrapeJobCancelledError,ScrapeRetryLimitError,+ BrandingNotSupportedError,} from "../scraper/scrapeURL/error";// TODO: figure out correct typing for this@@ -52,6 +53,7 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_NO_CACHED_DATA: NoCachedDataError,SCRAPE_ACTION_ERROR: ActionError,SCRAPE_ACTIONS_NOT_SUPPORTED: ActionsNotSupportedError,+ SCRAPE_BRANDING_NOT_SUPPORTED: BrandingNotSupportedError,SCRAPE_RACED_REDIRECT_ERROR: RacedRedirectError,SCRAPE_SITEMAP_ERROR: SitemapError,CRAWL_DENIAL: CrawlDenialError,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex 146900f64..6ab3c61d4 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -22,6 +22,7 @@ export type ErrorCodes =| "SCRAPE_NO_CACHED_DATA"| "SCRAPE_SITEMAP_ERROR"| "SCRAPE_ACTIONS_NOT_SUPPORTED"+ | "SCRAPE_BRANDING_NOT_SUPPORTED"| "CRAWL_DENIAL"| "BAD_REQUEST_INVALID_JSON"| "BAD_REQUEST";diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 8f561ff5d..7a6b4b560 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -20,6 +20,7 @@ import { hasFormatOfType } from "../../../lib/format-utils";import { getPDFMaxPages } from "../../../controllers/v2/types";import type { PdfMetadata } from "./pdf/types";import { BrandingProfile } from "../../../types/branding";+import { BrandingNotSupportedError } from "../error";export type Engine =| "fire-engine;chrome-cdp"@@ -610,15 +611,17 @@ export async function buildFallbackList(meta: Meta): Promise<);if (!hasCDPEngine) {if (meta.featureFlags.has("pdf")) {- throw new Error(+ throw new BrandingNotSupportedError("Branding extraction is only supported for HTML web pages. PDFs are not supported.",);} else if (meta.featureFlags.has("document")) {- throw new Error(+ throw new BrandingNotSupportedError("Branding extraction is only supported for HTML web pages. Documents (docx, xlsx, etc.) are not supported.",);}- throw new Error("Branding extraction requires Chrome CDP (fire-engine).");+ throw new BrandingNotSupportedError(+ "Branding extraction requires Chrome CDP (fire-engine).",+ );}}diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 4fcdb1365..d15a4d09d 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -447,6 +447,25 @@ export class DocumentPrefetchFailed extends TransportableError {}}+export class BrandingNotSupportedError extends TransportableError {+ constructor(message: string) {+ super("SCRAPE_BRANDING_NOT_SUPPORTED", message);+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new BrandingNotSupportedError(data.message);+ x.stack = data.stack;+ return x;+ }+}+export class FEPageLoadFailed extends Error {constructor() {super(diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 9102b9b1e..92ab6e64b 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -49,6 +49,7 @@ import {EngineUnsuccessfulError,ProxySelectionError,ScrapeRetryLimitError,+ BrandingNotSupportedError,} from "./error";import { ScrapeRetryTracker } from "./retryTracker";import { executeTransformers } from "./transformers";@@ -1219,6 +1220,11 @@ export async function scrapeURL("scrapeURL: Failed to prefetch document that is protected by anti-bot",{ error },);+ } else if (error instanceof BrandingNotSupportedError) {+ errorType = "BrandingNotSupportedError";+ meta.logger.warn("scrapeURL: Branding not supported for this content", {+ error,+ });} else if (error instanceof ProxySelectionError) {errorType = "ProxySelectionError";meta.logger.warn("scrapeURL: Proxy selection error", { error });