Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): accept fire-engine's GCS PDF handoff on the POST /scrape fast path (#4492)

Abimael Martellba47e966d87a
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/fileSchema.test.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/fileSchema.test.ts
new file mode 100644
index 000000000..57a936879
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/fileSchema.test.ts
@@ -0,0 +1,150 @@
+import { describe, expect, it, vi } from "vitest";
+
+// fireEngineScrape talks to fire-engine through robustFetch; swap it for a
+// controllable stub so the POST /scrape response shape is the only variable.
+const fetchFake = vi.hoisted(() => ({
+ status: {} as Record<string, unknown>,
+}));
+
+vi.mock("../../../lib/fetch", () => ({
+ robustFetch: vi.fn(async () => fetchFake.status),
+}));
+
+vi.mock("../../../../../lib/gcs-jobs", () => ({
+ getDocFromGCS: vi.fn(async () => null),
+}));
+
+import { fireEngineFileSchema } from "../fileSchema";
+import { fireEngineScrape } from "../scrape";
+
+const fakeLogger = {
+ debug: vi.fn(),
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ child() {
+ return this;
+ },
+} as any;
+
+const fakeMeta = {
+ options: { proxy: "auto" },
+ featureFlags: new Set<string>(),
+} as any;
+
+const request = {
+ url: "https://example.com/report.pdf",
+ engine: "chrome-cdp",
+ timeout: 30000,
+} as any;
+
+// Mirrors the shape fire-engine returns for a large-PDF handoff: no inline
+// bytes, a GCS reference plus the content identity, and an empty `content`.
+const handoffStatus = {
+ timeTaken: 20.5,
+ content: "",
+ url: "https://example.com/report.pdf",
+ pageStatusCode: 200,
+ responseHeaders: {
+ "content-type": "application/pdf",
+ "content-length": "59163826",
+ },
+ screenshots: [],
+ actionContent: [],
+ actionResults: [],
+ file: {
+ name: "report.pdf",
+ gcs_uri: "gs://fire-engine-handoff/pdf-handoff/0f1e2d3c-job.pdf",
+ sha256: "0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0",
+ size_bytes: 59163826,
+ },
+ usedMobileProxy: true,
+ timezone: "America/New_York",
+};
+
+describe("fireEngineFileSchema", () => {
+ it("accepts an inline file", () => {
+ expect(
+ fireEngineFileSchema.safeParse({ name: "a.pdf", content: "JVBERi0=" })
+ .success,
+ ).toBe(true);
+ });
+
+ it("accepts a GCS handoff reference", () => {
+ expect(fireEngineFileSchema.safeParse(handoffStatus.file).success).toBe(
+ true,
+ );
+ });
+
+ it("accepts a missing or null file", () => {
+ expect(fireEngineFileSchema.safeParse(undefined).success).toBe(true);
+ expect(fireEngineFileSchema.safeParse(null).success).toBe(true);
+ });
+
+ it("rejects a file carrying both or neither transport", () => {
+ expect(
+ fireEngineFileSchema.safeParse({
+ name: "a.pdf",
+ content: "JVBERi0=",
+ gcs_uri: "gs://b/k.pdf",
+ }).success,
+ ).toBe(false);
+ expect(fireEngineFileSchema.safeParse({ name: "a.pdf" }).success).toBe(
+ false,
+ );
+ });
+});
+
+describe("fireEngineScrape response parsing", () => {
+ it("accepts a finished job carrying a GCS handoff on the POST /scrape fast path", async () => {
+ fetchFake.status = handoffStatus;
+ const result = await fireEngineScrape(
+ fakeMeta,
+ fakeLogger,
+ request,
+ null,
+ undefined,
+ "http://fire-engine.test",
+ );
+ expect("file" in result && result.file).toMatchObject({
+ gcs_uri: handoffStatus.file.gcs_uri,
+ sha256: handoffStatus.file.sha256,
+ size_bytes: handoffStatus.file.size_bytes,
+ });
+ });
+
+ it("still accepts a finished job carrying an inline file", async () => {
+ fetchFake.status = {
+ ...handoffStatus,
+ file: { name: "report.pdf", content: "JVBERi0=" },
+ };
+ const result = await fireEngineScrape(
+ fakeMeta,
+ fakeLogger,
+ request,
+ null,
+ undefined,
+ "http://fire-engine.test",
+ );
+ expect("file" in result && result.file).toMatchObject({
+ content: "JVBERi0=",
+ });
+ });
+
+ it("rejects a file that carries both transports as an unmatched response", async () => {
+ fetchFake.status = {
+ ...handoffStatus,
+ file: { ...handoffStatus.file, content: "JVBERi0=" },
+ };
+ await expect(
+ fireEngineScrape(
+ fakeMeta,
+ fakeLogger,
+ request,
+ null,
+ undefined,
+ "http://fire-engine.test",
+ ),
+ ).rejects.toThrow("not matched by any schema");
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index d77b0711b..e847dae1f 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -16,6 +16,7 @@ import {
} from "../../error";
import { MockState } from "../../lib/mock";
import { fireEngineURL } from "./scrape";
+import { fireEngineFileSchema } from "./fileSchema";
import { getDocFromGCS } from "../../../../lib/gcs-jobs";
import { Meta } from "../..";
@@ -104,24 +105,9 @@ const successSchema = z.object({
.array()
.optional(),
- // chrome-cdp only -- file download handler. Small files arrive inline as
- // base64 `content`; large PDFs arrive as a GCS reference instead
- // (fire-engine uploads them to its handoff bucket rather than inlining
- // hundreds of MB of base64 into this response). Exactly one of
- // `content` / `gcs_uri` is expected.
- file: z
- .object({
- name: z.string(),
- content: z.string().optional(),
- gcs_uri: z.string().optional(),
- sha256: z.string().optional(),
- size_bytes: z.number().optional(),
- })
- .refine(f => (f.content !== undefined) !== (f.gcs_uri !== undefined), {
- message: "file must carry exactly one of content or gcs_uri",
- })
- .optional()
- .or(z.null()),
+ // chrome-cdp only -- file download handler (inline base64 or a GCS
+ // handoff reference; see fileSchema.ts).
+ file: fireEngineFileSchema,
docUrl: z.string().optional(),
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/fileSchema.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/fileSchema.ts
new file mode 100644
index 000000000..5bce38da6
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/fileSchema.ts
@@ -0,0 +1,29 @@
+import { z } from "zod";
+
+/**
+ * The `file` member of a fire-engine chrome-cdp success response (the file
+ * download handler). Small files arrive inline as base64 `content`; large
+ * PDFs arrive as a GCS reference instead (fire-engine uploads them to its
+ * handoff bucket rather than inlining hundreds of MB of base64 into the
+ * response). Exactly one of `content` / `gcs_uri` is expected.
+ *
+ * Shared by BOTH response parsers on purpose. fire-engine can return the
+ * finished job from `POST /scrape` directly (fireEngineScrape) or from the
+ * `GET /scrape/:id` poll (fireEngineCheckStatus); a handoff that only one of
+ * them understood was rejected as "response not matched by any schema" on
+ * the fast path, which made every quick large-PDF download look like an
+ * engine failure and re-ran the download on the next engine.
+ */
+export const fireEngineFileSchema = z
+ .object({
+ name: z.string(),
+ content: z.string().optional(),
+ gcs_uri: z.string().optional(),
+ sha256: z.string().optional(),
+ size_bytes: z.number().optional(),
+ })
+ .refine(f => (f.content !== undefined) !== (f.gcs_uri !== undefined), {
+ message: "file must carry exactly one of content or gcs_uri",
+ })
+ .optional()
+ .or(z.null());
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 6c9f32573..de0cc7cb7 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -5,6 +5,7 @@ import { InternalAction } from "../../../../controllers/v1/types";
import { robustFetch } from "../../lib/fetch";
import { MockState } from "../../lib/mock";
import { getDocFromGCS } from "../../../../lib/gcs-jobs";
+import { fireEngineFileSchema } from "./fileSchema";
import {
ActionError,
AddFeatureError,
@@ -155,14 +156,12 @@ const successSchema = z.object({
.array()
.optional(),
- // chrome-cdp only -- file download handler
- file: z
- .object({
- name: z.string(),
- content: z.string(),
- })
- .optional()
- .or(z.null()),
+ // chrome-cdp only -- file download handler (inline base64 or a GCS
+ // handoff reference; see fileSchema.ts). Must accept exactly what the
+ // poll parser in checkStatus.ts accepts: fire-engine returns finished
+ // jobs from POST /scrape too, and a handoff rejected here surfaced as
+ // "response not matched by any schema" -> a spurious engine failure.
+ file: fireEngineFileSchema,
docUrl: z.string().optional(),