Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): keep branding-only scrapes off the render engine (#4118)
Abimael Martell30ca53574c33
diff --git a/apps/api/src/__tests__/lib/fire-engine-render-routing.test.ts b/apps/api/src/__tests__/lib/fire-engine-render-routing.test.tsnew file mode 100644index 000000000..5b7007d4f--- /dev/null+++ b/apps/api/src/__tests__/lib/fire-engine-render-routing.test.ts@@ -0,0 +1,111 @@+import { shouldForceNonRender } from "../../scraper/scrapeURL/engines/fire-engine";++const fmt = (types: string[]) => types.map(type => ({ type })) as any;++describe("shouldForceNonRender", () => {+ it("opts branding-only scrapes out of render routing", () => {+ expect(+ shouldForceNonRender({+ formats: fmt(["branding"]),+ youtubePostprocessorWillRun: false,+ }),+ ).toBe(true);+ });++ it("keeps render routing when a screenshot format is requested", () => {+ expect(+ shouldForceNonRender({+ formats: fmt(["branding", "screenshot"]),+ youtubePostprocessorWillRun: false,+ }),+ ).toBe(false);+ });++ it("keeps render routing when a screenshot action is requested", () => {+ expect(+ shouldForceNonRender({+ formats: fmt(["branding"]),+ actions: [{ type: "wait" }, { type: "screenshot" }],+ youtubePostprocessorWillRun: false,+ }),+ ).toBe(false);+ });++ it("still opts out with DOM-only user actions", () => {+ expect(+ shouldForceNonRender({+ formats: fmt(["branding"]),+ actions: [+ { type: "wait" },+ { type: "click" },+ { type: "scroll" },+ { type: "write" },+ { type: "press" },+ { type: "scrape" },+ { type: "executeJavascript" },+ ],+ youtubePostprocessorWillRun: false,+ }),+ ).toBe(true);+ });++ it("keeps render routing for unknown/future action types (fail safe)", () => {+ expect(+ shouldForceNonRender({+ formats: fmt(["branding"]),+ actions: [{ type: "someFutureVisualAction" }],+ youtubePostprocessorWillRun: false,+ }),+ ).toBe(false);+ });++ it("keeps render routing when a pdf action is requested", () => {+ expect(+ shouldForceNonRender({+ formats: fmt(["branding"]),+ actions: [{ type: "pdf" }],+ youtubePostprocessorWillRun: false,+ }),+ ).toBe(false);+ });++ it("keeps render routing for audio/video formats", () => {+ expect(+ shouldForceNonRender({+ formats: fmt(["branding", "audio"]),+ youtubePostprocessorWillRun: false,+ }),+ ).toBe(false);+ expect(+ shouldForceNonRender({+ formats: fmt(["branding", "video"]),+ youtubePostprocessorWillRun: false,+ }),+ ).toBe(false);+ });++ it("keeps render routing when the media postprocessor will run", () => {+ expect(+ shouldForceNonRender({+ formats: fmt(["branding"]),+ youtubePostprocessorWillRun: true,+ }),+ ).toBe(false);+ });++ it("does nothing for non-branding scrapes", () => {+ expect(+ shouldForceNonRender({+ formats: fmt(["markdown"]),+ youtubePostprocessorWillRun: false,+ }),+ ).toBe(false);+ expect(+ shouldForceNonRender({+ formats: fmt(["markdown"]),+ actions: [{ type: "wait" }],+ youtubePostprocessorWillRun: false,+ }),+ ).toBe(false);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 2e513f769..b2a5e86ac 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -273,6 +273,42 @@ async function performFireEngineScrape<});}+// Action types that only read or drive the DOM and don't depend on rendered+// output. Anything not listed here (screenshot, pdf, and any future visual+// action) keeps render-engine routing — fail safe, not open.+const DOM_SAFE_ACTION_TYPES: ReadonlySet<string> = new Set([+ "wait",+ "click",+ "write",+ "press",+ "scroll",+ "scrape",+ "executeJavascript",+]);++// Branding needs media *loaded* (real image dimensions in the DOM), not+// *rendered* — but blockMedia: false routes to the render engine, where+// visually heavy pages can stall the renderer. Opt out of render routing+// unless something actually needs visual output.+export function shouldForceNonRender(input: {+ formats: Meta["options"]["formats"];+ actions?: Array<{ type: string }>;+ youtubePostprocessorWillRun: boolean;+}): boolean {+ if (!hasFormatOfType(input.formats, "branding")) {+ return false;+ }++ const needsVisualRendering =+ hasFormatOfType(input.formats, "screenshot") !== undefined ||+ (input.actions ?? []).some(a => !DOM_SAFE_ACTION_TYPES.has(a.type)) ||+ hasFormatOfType(input.formats, "audio") !== undefined ||+ hasFormatOfType(input.formats, "video") !== undefined ||+ input.youtubePostprocessorWillRun;++ return !needsVisualRendering;+}+export async function scrapeURLWithFireEngineChromeCDP(meta: Meta,): Promise<EngineScrapeResult> {@@ -360,6 +396,12 @@ export async function scrapeURLWithFireEngineChromeCDP(hasFormatOfType(meta.options.formats, "branding") ||shouldRunYoutubePostprocessor;+ const forceNonRender = shouldForceNonRender({+ formats: meta.options.formats,+ actions: meta.options.actions ?? undefined,+ youtubePostprocessorWillRun: shouldRunYoutubePostprocessor,+ });+const request: FireEngineScrapeRequestCommon &FireEngineScrapeRequestChromeCDP = {url: meta.rewrittenUrl ?? meta.url,@@ -385,6 +427,7 @@ export async function scrapeURLWithFireEngineChromeCDP(meta.internalOptions.saveScrapeResultToGCS,zeroDataRetention: meta.internalOptions.zeroDataRetention,...(shouldAllowMedia ? { blockMedia: false } : {}),+ ...(forceNonRender ? { forceNonRender: true } : {}),persistentStorage: meta.options.profile? {uniqueId: `${createHash("sha256").update(meta.internalOptions.teamId).digest("hex").slice(0, 16)}_${meta.options.profile.name}`,diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex ebf59dbda..6eaee9319 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -61,6 +61,8 @@ export type FireEngineScrapeRequestChromeCDP = {skipTlsVerification?: boolean;actions?: InternalAction[];blockMedia?: boolean;+ /** Opt out of render-engine routing (blockMedia: false normally forces it). */+ forceNonRender?: boolean;mobile?: boolean;disableSmartWaitCache?: boolean;persistentStorage?: { uniqueId: string };