Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(branding): skip branding on PDFs, documents and images instead of failing the scrape (#4882)

Abimael Martellbd83f5d8436a
diff --git a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts b/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts
index dc0548639..2b306a0d3 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts
@@ -2,8 +2,21 @@ import {
type CostTrackingCall,
getCostTrackingCalls,
} from "../cost-tracking-helpers";
-import { concurrentIf, HAS_AI, TEST_PRODUCTION } from "../lib";
-import { scrape, scrapeTimeout, idmux, Identity } from "./lib";
+import {
+ ALLOW_TEST_SUITE_WEBSITE,
+ concurrentIf,
+ HAS_AI,
+ TEST_PRODUCTION,
+ TEST_SELF_HOST,
+ TEST_SUITE_WEBSITE,
+} from "../lib";
+import {
+ scrape,
+ scrapeTimeout,
+ scrapeWithFailure,
+ idmux,
+ Identity,
+} from "./lib";
let identity: Identity;
@@ -135,6 +148,64 @@ describe("Branding with Jev", () => {
);
});
+const PDF_URL = "https://www.orimi.com/pdf-test.pdf";
+
+describe("Branding on pages it can't run on", () => {
+ concurrentIf(TEST_PRODUCTION)(
+ "keeps the other formats and warns when the page is a PDF",
+ async () => {
+ const response = await scrape(
+ {
+ url: PDF_URL,
+ formats: ["markdown", "branding"],
+ timeout: scrapeTimeout,
+ },
+ identity,
+ );
+
+ expect(response.markdown?.length).toBeGreaterThan(0);
+ expect(response.branding).toBeUndefined();
+ expect(response.warning).toContain("Branding was skipped");
+ },
+ scrapeTimeout,
+ );
+
+ concurrentIf(TEST_PRODUCTION)(
+ "still fails a branding-only request for a PDF",
+ async () => {
+ const response = await scrapeWithFailure(
+ { url: PDF_URL, formats: ["branding"], timeout: scrapeTimeout },
+ identity,
+ );
+
+ expect(response.error).toContain(
+ "Branding extraction is only supported for HTML web pages",
+ );
+ },
+ scrapeTimeout,
+ );
+
+ // Self-hosted has no fire-engine, so branding can never run there.
+ concurrentIf(TEST_SELF_HOST && ALLOW_TEST_SUITE_WEBSITE)(
+ "keeps the other formats when branding can't run self-hosted",
+ async () => {
+ const response = await scrape(
+ {
+ url: TEST_SUITE_WEBSITE,
+ formats: ["markdown", "branding"],
+ timeout: scrapeTimeout,
+ },
+ identity,
+ );
+
+ expect(response.markdown?.length).toBeGreaterThan(0);
+ expect(response.branding).toBeUndefined();
+ expect(response.warning).toContain("Branding was skipped");
+ },
+ scrapeTimeout,
+ );
+});
+
// TODO: fix this test
// Need to run on fire-engine
describe.skip("Branding extraction", () => {
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 16f6ae239..36d0c3366 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -1477,6 +1477,7 @@ export async function scrapeURL(
try {
let result: ScrapeUrlResponse;
+ let brandingSkippedReason: string | undefined;
while (true) {
try {
result = await scrapeURLLoop(meta);
@@ -1625,12 +1626,41 @@ export async function scrapeURL(
[...meta.featureFlags].filter(x => x !== "document"),
);
}
+ } else if (
+ error instanceof BrandingNotSupportedError &&
+ meta.options.formats.some(f => f.type !== "branding")
+ ) {
+ // The page turned out to be a PDF, document or image. Keep the
+ // other requested formats instead of failing the whole scrape;
+ // branding is dropped with a warning. Branding-only requests
+ // still fail with the error.
+ retryTracker.record("feature_removal", error);
+ meta.logger.info("Skipping branding for a non-HTML page", {
+ reason: error.message,
+ });
+ brandingSkippedReason = error.message;
+ meta.featureFlags = new Set(
+ [...meta.featureFlags].filter(x => x !== "branding"),
+ );
+ meta.options = {
+ ...meta.options,
+ formats: meta.options.formats.filter(
+ f => f.type !== "branding",
+ ),
+ };
} else {
throw error;
}
}
}
+ if (brandingSkippedReason && result.success) {
+ const warning = `Branding was skipped: ${brandingSkippedReason}`;
+ result.document.warning = result.document.warning
+ ? `${result.document.warning} ${warning}`
+ : warning;
+ }
+
// Threat protection: if the scrape ended up on a different URL than
// requested (redirect), re-check the destination URL. This closes the
// "clean URL redirects to a blocked URL" bypass vector — including