Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): keep branding-only scrapes off the render engine (#4118)

Abimael Martell30ca53574c33
diff --git a/apps/api/src/__tests__/lib/fire-engine-render-routing.test.ts b/apps/api/src/__tests__/lib/fire-engine-render-routing.test.ts
new file mode 100644
index 000000000..5b7007d4f
--- /dev/null
+++ b/apps/api/src/__tests__/lib/fire-engine-render-routing.test.ts
@@ -0,0 +1,111 @@
+import { shouldForceNonRender } from "../../scraper/scrapeURL/engines/fire-engine";
+
+const fmt = (types: string[]) => types.map(type => ({ type })) as any;
+
+describe("shouldForceNonRender", () => {
+ it("opts branding-only scrapes out of render routing", () => {
+ expect(
+ shouldForceNonRender({
+ formats: fmt(["branding"]),
+ youtubePostprocessorWillRun: false,
+ }),
+ ).toBe(true);
+ });
+
+ it("keeps render routing when a screenshot format is requested", () => {
+ expect(
+ shouldForceNonRender({
+ formats: fmt(["branding", "screenshot"]),
+ youtubePostprocessorWillRun: false,
+ }),
+ ).toBe(false);
+ });
+
+ it("keeps render routing when a screenshot action is requested", () => {
+ expect(
+ shouldForceNonRender({
+ formats: fmt(["branding"]),
+ actions: [{ type: "wait" }, { type: "screenshot" }],
+ youtubePostprocessorWillRun: false,
+ }),
+ ).toBe(false);
+ });
+
+ it("still opts out with DOM-only user actions", () => {
+ expect(
+ shouldForceNonRender({
+ formats: fmt(["branding"]),
+ actions: [
+ { type: "wait" },
+ { type: "click" },
+ { type: "scroll" },
+ { type: "write" },
+ { type: "press" },
+ { type: "scrape" },
+ { type: "executeJavascript" },
+ ],
+ youtubePostprocessorWillRun: false,
+ }),
+ ).toBe(true);
+ });
+
+ it("keeps render routing for unknown/future action types (fail safe)", () => {
+ expect(
+ shouldForceNonRender({
+ formats: fmt(["branding"]),
+ actions: [{ type: "someFutureVisualAction" }],
+ youtubePostprocessorWillRun: false,
+ }),
+ ).toBe(false);
+ });
+
+ it("keeps render routing when a pdf action is requested", () => {
+ expect(
+ shouldForceNonRender({
+ formats: fmt(["branding"]),
+ actions: [{ type: "pdf" }],
+ youtubePostprocessorWillRun: false,
+ }),
+ ).toBe(false);
+ });
+
+ it("keeps render routing for audio/video formats", () => {
+ expect(
+ shouldForceNonRender({
+ formats: fmt(["branding", "audio"]),
+ youtubePostprocessorWillRun: false,
+ }),
+ ).toBe(false);
+ expect(
+ shouldForceNonRender({
+ formats: fmt(["branding", "video"]),
+ youtubePostprocessorWillRun: false,
+ }),
+ ).toBe(false);
+ });
+
+ it("keeps render routing when the media postprocessor will run", () => {
+ expect(
+ shouldForceNonRender({
+ formats: fmt(["branding"]),
+ youtubePostprocessorWillRun: true,
+ }),
+ ).toBe(false);
+ });
+
+ it("does nothing for non-branding scrapes", () => {
+ expect(
+ shouldForceNonRender({
+ formats: fmt(["markdown"]),
+ youtubePostprocessorWillRun: false,
+ }),
+ ).toBe(false);
+ expect(
+ shouldForceNonRender({
+ formats: fmt(["markdown"]),
+ actions: [{ type: "wait" }],
+ youtubePostprocessorWillRun: false,
+ }),
+ ).toBe(false);
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 2e513f769..b2a5e86ac 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -273,6 +273,42 @@ async function performFireEngineScrape<
});
}
+// Action types that only read or drive the DOM and don't depend on rendered
+// output. Anything not listed here (screenshot, pdf, and any future visual
+// action) keeps render-engine routing — fail safe, not open.
+const DOM_SAFE_ACTION_TYPES: ReadonlySet<string> = new Set([
+ "wait",
+ "click",
+ "write",
+ "press",
+ "scroll",
+ "scrape",
+ "executeJavascript",
+]);
+
+// Branding needs media *loaded* (real image dimensions in the DOM), not
+// *rendered* — but blockMedia: false routes to the render engine, where
+// visually heavy pages can stall the renderer. Opt out of render routing
+// unless something actually needs visual output.
+export function shouldForceNonRender(input: {
+ formats: Meta["options"]["formats"];
+ actions?: Array<{ type: string }>;
+ youtubePostprocessorWillRun: boolean;
+}): boolean {
+ if (!hasFormatOfType(input.formats, "branding")) {
+ return false;
+ }
+
+ const needsVisualRendering =
+ hasFormatOfType(input.formats, "screenshot") !== undefined ||
+ (input.actions ?? []).some(a => !DOM_SAFE_ACTION_TYPES.has(a.type)) ||
+ hasFormatOfType(input.formats, "audio") !== undefined ||
+ hasFormatOfType(input.formats, "video") !== undefined ||
+ input.youtubePostprocessorWillRun;
+
+ return !needsVisualRendering;
+}
+
export async function scrapeURLWithFireEngineChromeCDP(
meta: Meta,
): Promise<EngineScrapeResult> {
@@ -360,6 +396,12 @@ export async function scrapeURLWithFireEngineChromeCDP(
hasFormatOfType(meta.options.formats, "branding") ||
shouldRunYoutubePostprocessor;
+ const forceNonRender = shouldForceNonRender({
+ formats: meta.options.formats,
+ actions: meta.options.actions ?? undefined,
+ youtubePostprocessorWillRun: shouldRunYoutubePostprocessor,
+ });
+
const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestChromeCDP = {
url: meta.rewrittenUrl ?? meta.url,
@@ -385,6 +427,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
meta.internalOptions.saveScrapeResultToGCS,
zeroDataRetention: meta.internalOptions.zeroDataRetention,
...(shouldAllowMedia ? { blockMedia: false } : {}),
+ ...(forceNonRender ? { forceNonRender: true } : {}),
persistentStorage: meta.options.profile
? {
uniqueId: `${createHash("sha256").update(meta.internalOptions.teamId).digest("hex").slice(0, 16)}_${meta.options.profile.name}`,
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index ebf59dbda..6eaee9319 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -61,6 +61,8 @@ export type FireEngineScrapeRequestChromeCDP = {
skipTlsVerification?: boolean;
actions?: InternalAction[];
blockMedia?: boolean;
+ /** Opt out of render-engine routing (blockMedia: false normally forces it). */
+ forceNonRender?: boolean;
mobile?: boolean;
disableSmartWaitCache?: boolean;
persistentStorage?: { uniqueId: string };