Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(branding): skip branding on PDFs, documents and images instead of failing the scrape (#4882)
Abimael Martellbd83f5d8436a
diff --git a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts b/apps/api/src/__tests__/snips/v2/scrape-branding.test.tsindex dc0548639..2b306a0d3 100644--- a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts@@ -2,8 +2,21 @@ import {type CostTrackingCall,getCostTrackingCalls,} from "../cost-tracking-helpers";-import { concurrentIf, HAS_AI, TEST_PRODUCTION } from "../lib";-import { scrape, scrapeTimeout, idmux, Identity } from "./lib";+import {+ ALLOW_TEST_SUITE_WEBSITE,+ concurrentIf,+ HAS_AI,+ TEST_PRODUCTION,+ TEST_SELF_HOST,+ TEST_SUITE_WEBSITE,+} from "../lib";+import {+ scrape,+ scrapeTimeout,+ scrapeWithFailure,+ idmux,+ Identity,+} from "./lib";let identity: Identity;@@ -135,6 +148,64 @@ describe("Branding with Jev", () => {);});+const PDF_URL = "https://www.orimi.com/pdf-test.pdf";++describe("Branding on pages it can't run on", () => {+ concurrentIf(TEST_PRODUCTION)(+ "keeps the other formats and warns when the page is a PDF",+ async () => {+ const response = await scrape(+ {+ url: PDF_URL,+ formats: ["markdown", "branding"],+ timeout: scrapeTimeout,+ },+ identity,+ );++ expect(response.markdown?.length).toBeGreaterThan(0);+ expect(response.branding).toBeUndefined();+ expect(response.warning).toContain("Branding was skipped");+ },+ scrapeTimeout,+ );++ concurrentIf(TEST_PRODUCTION)(+ "still fails a branding-only request for a PDF",+ async () => {+ const response = await scrapeWithFailure(+ { url: PDF_URL, formats: ["branding"], timeout: scrapeTimeout },+ identity,+ );++ expect(response.error).toContain(+ "Branding extraction is only supported for HTML web pages",+ );+ },+ scrapeTimeout,+ );++ // Self-hosted has no fire-engine, so branding can never run there.+ concurrentIf(TEST_SELF_HOST && ALLOW_TEST_SUITE_WEBSITE)(+ "keeps the other formats when branding can't run self-hosted",+ async () => {+ const response = await scrape(+ {+ url: TEST_SUITE_WEBSITE,+ formats: ["markdown", "branding"],+ timeout: scrapeTimeout,+ },+ identity,+ );++ expect(response.markdown?.length).toBeGreaterThan(0);+ expect(response.branding).toBeUndefined();+ expect(response.warning).toContain("Branding was skipped");+ },+ scrapeTimeout,+ );+});+// TODO: fix this test// Need to run on fire-enginedescribe.skip("Branding extraction", () => {diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 16f6ae239..36d0c3366 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -1477,6 +1477,7 @@ export async function scrapeURL(try {let result: ScrapeUrlResponse;+ let brandingSkippedReason: string | undefined;while (true) {try {result = await scrapeURLLoop(meta);@@ -1625,12 +1626,41 @@ export async function scrapeURL([...meta.featureFlags].filter(x => x !== "document"),);}+ } else if (+ error instanceof BrandingNotSupportedError &&+ meta.options.formats.some(f => f.type !== "branding")+ ) {+ // The page turned out to be a PDF, document or image. Keep the+ // other requested formats instead of failing the whole scrape;+ // branding is dropped with a warning. Branding-only requests+ // still fail with the error.+ retryTracker.record("feature_removal", error);+ meta.logger.info("Skipping branding for a non-HTML page", {+ reason: error.message,+ });+ brandingSkippedReason = error.message;+ meta.featureFlags = new Set(+ [...meta.featureFlags].filter(x => x !== "branding"),+ );+ meta.options = {+ ...meta.options,+ formats: meta.options.formats.filter(+ f => f.type !== "branding",+ ),+ };} else {throw error;}}}+ if (brandingSkippedReason && result.success) {+ const warning = `Branding was skipped: ${brandingSkippedReason}`;+ result.document.warning = result.document.warning+ ? `${result.document.warning} ${warning}`+ : warning;+ }+// Threat protection: if the scrape ended up on a different URL than// requested (redirect), re-check the destination URL. This closes the// "clean URL redirects to a blocked URL" bypass vector — including