Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): use TransportableError for branding validation errors (#2923)

Abimael Martell7b7a736d7038
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index da33785a4..f93fd1b9a 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -27,6 +27,7 @@ import {
NoCachedDataError,
ScrapeJobCancelledError,
ScrapeRetryLimitError,
+ BrandingNotSupportedError,
} from "../scraper/scrapeURL/error";
// TODO: figure out correct typing for this
@@ -52,6 +53,7 @@ const errorMap: Record<ErrorCodes, any> = {
SCRAPE_NO_CACHED_DATA: NoCachedDataError,
SCRAPE_ACTION_ERROR: ActionError,
SCRAPE_ACTIONS_NOT_SUPPORTED: ActionsNotSupportedError,
+ SCRAPE_BRANDING_NOT_SUPPORTED: BrandingNotSupportedError,
SCRAPE_RACED_REDIRECT_ERROR: RacedRedirectError,
SCRAPE_SITEMAP_ERROR: SitemapError,
CRAWL_DENIAL: CrawlDenialError,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index 146900f64..6ab3c61d4 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -22,6 +22,7 @@ export type ErrorCodes =
| "SCRAPE_NO_CACHED_DATA"
| "SCRAPE_SITEMAP_ERROR"
| "SCRAPE_ACTIONS_NOT_SUPPORTED"
+ | "SCRAPE_BRANDING_NOT_SUPPORTED"
| "CRAWL_DENIAL"
| "BAD_REQUEST_INVALID_JSON"
| "BAD_REQUEST";
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 8f561ff5d..7a6b4b560 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -20,6 +20,7 @@ import { hasFormatOfType } from "../../../lib/format-utils";
import { getPDFMaxPages } from "../../../controllers/v2/types";
import type { PdfMetadata } from "./pdf/types";
import { BrandingProfile } from "../../../types/branding";
+import { BrandingNotSupportedError } from "../error";
export type Engine =
| "fire-engine;chrome-cdp"
@@ -610,15 +611,17 @@ export async function buildFallbackList(meta: Meta): Promise<
);
if (!hasCDPEngine) {
if (meta.featureFlags.has("pdf")) {
- throw new Error(
+ throw new BrandingNotSupportedError(
"Branding extraction is only supported for HTML web pages. PDFs are not supported.",
);
} else if (meta.featureFlags.has("document")) {
- throw new Error(
+ throw new BrandingNotSupportedError(
"Branding extraction is only supported for HTML web pages. Documents (docx, xlsx, etc.) are not supported.",
);
}
- throw new Error("Branding extraction requires Chrome CDP (fire-engine).");
+ throw new BrandingNotSupportedError(
+ "Branding extraction requires Chrome CDP (fire-engine).",
+ );
}
}
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 4fcdb1365..d15a4d09d 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -447,6 +447,25 @@ export class DocumentPrefetchFailed extends TransportableError {
}
}
+export class BrandingNotSupportedError extends TransportableError {
+ constructor(message: string) {
+ super("SCRAPE_BRANDING_NOT_SUPPORTED", message);
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new BrandingNotSupportedError(data.message);
+ x.stack = data.stack;
+ return x;
+ }
+}
+
export class FEPageLoadFailed extends Error {
constructor() {
super(
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 9102b9b1e..92ab6e64b 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -49,6 +49,7 @@ import {
EngineUnsuccessfulError,
ProxySelectionError,
ScrapeRetryLimitError,
+ BrandingNotSupportedError,
} from "./error";
import { ScrapeRetryTracker } from "./retryTracker";
import { executeTransformers } from "./transformers";
@@ -1219,6 +1220,11 @@ export async function scrapeURL(
"scrapeURL: Failed to prefetch document that is protected by anti-bot",
{ error },
);
+ } else if (error instanceof BrandingNotSupportedError) {
+ errorType = "BrandingNotSupportedError";
+ meta.logger.warn("scrapeURL: Branding not supported for this content", {
+ error,
+ });
} else if (error instanceof ProxySelectionError) {
errorType = "ProxySelectionError";
meta.logger.warn("scrapeURL: Proxy selection error", { error });