Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): accept fire-engine's GCS PDF handoff on the POST /scrape fast path (#4492)
Abimael Martellba47e966d87a
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/fileSchema.test.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/fileSchema.test.tsnew file mode 100644index 000000000..57a936879--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/fileSchema.test.ts@@ -0,0 +1,150 @@+import { describe, expect, it, vi } from "vitest";++// fireEngineScrape talks to fire-engine through robustFetch; swap it for a+// controllable stub so the POST /scrape response shape is the only variable.+const fetchFake = vi.hoisted(() => ({+ status: {} as Record<string, unknown>,+}));++vi.mock("../../../lib/fetch", () => ({+ robustFetch: vi.fn(async () => fetchFake.status),+}));++vi.mock("../../../../../lib/gcs-jobs", () => ({+ getDocFromGCS: vi.fn(async () => null),+}));++import { fireEngineFileSchema } from "../fileSchema";+import { fireEngineScrape } from "../scrape";++const fakeLogger = {+ debug: vi.fn(),+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ child() {+ return this;+ },+} as any;++const fakeMeta = {+ options: { proxy: "auto" },+ featureFlags: new Set<string>(),+} as any;++const request = {+ url: "https://example.com/report.pdf",+ engine: "chrome-cdp",+ timeout: 30000,+} as any;++// Mirrors the shape fire-engine returns for a large-PDF handoff: no inline+// bytes, a GCS reference plus the content identity, and an empty `content`.+const handoffStatus = {+ timeTaken: 20.5,+ content: "",+ url: "https://example.com/report.pdf",+ pageStatusCode: 200,+ responseHeaders: {+ "content-type": "application/pdf",+ "content-length": "59163826",+ },+ screenshots: [],+ actionContent: [],+ actionResults: [],+ file: {+ name: "report.pdf",+ gcs_uri: "gs://fire-engine-handoff/pdf-handoff/0f1e2d3c-job.pdf",+ sha256: "0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0",+ size_bytes: 59163826,+ },+ usedMobileProxy: true,+ timezone: "America/New_York",+};++describe("fireEngineFileSchema", () => {+ it("accepts an inline file", () => {+ expect(+ fireEngineFileSchema.safeParse({ name: "a.pdf", content: "JVBERi0=" })+ .success,+ ).toBe(true);+ });++ it("accepts a GCS handoff reference", () => {+ expect(fireEngineFileSchema.safeParse(handoffStatus.file).success).toBe(+ true,+ );+ });++ it("accepts a missing or null file", () => {+ expect(fireEngineFileSchema.safeParse(undefined).success).toBe(true);+ expect(fireEngineFileSchema.safeParse(null).success).toBe(true);+ });++ it("rejects a file carrying both or neither transport", () => {+ expect(+ fireEngineFileSchema.safeParse({+ name: "a.pdf",+ content: "JVBERi0=",+ gcs_uri: "gs://b/k.pdf",+ }).success,+ ).toBe(false);+ expect(fireEngineFileSchema.safeParse({ name: "a.pdf" }).success).toBe(+ false,+ );+ });+});++describe("fireEngineScrape response parsing", () => {+ it("accepts a finished job carrying a GCS handoff on the POST /scrape fast path", async () => {+ fetchFake.status = handoffStatus;+ const result = await fireEngineScrape(+ fakeMeta,+ fakeLogger,+ request,+ null,+ undefined,+ "http://fire-engine.test",+ );+ expect("file" in result && result.file).toMatchObject({+ gcs_uri: handoffStatus.file.gcs_uri,+ sha256: handoffStatus.file.sha256,+ size_bytes: handoffStatus.file.size_bytes,+ });+ });++ it("still accepts a finished job carrying an inline file", async () => {+ fetchFake.status = {+ ...handoffStatus,+ file: { name: "report.pdf", content: "JVBERi0=" },+ };+ const result = await fireEngineScrape(+ fakeMeta,+ fakeLogger,+ request,+ null,+ undefined,+ "http://fire-engine.test",+ );+ expect("file" in result && result.file).toMatchObject({+ content: "JVBERi0=",+ });+ });++ it("rejects a file that carries both transports as an unmatched response", async () => {+ fetchFake.status = {+ ...handoffStatus,+ file: { ...handoffStatus.file, content: "JVBERi0=" },+ };+ await expect(+ fireEngineScrape(+ fakeMeta,+ fakeLogger,+ request,+ null,+ undefined,+ "http://fire-engine.test",+ ),+ ).rejects.toThrow("not matched by any schema");+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex d77b0711b..e847dae1f 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -16,6 +16,7 @@ import {} from "../../error";import { MockState } from "../../lib/mock";import { fireEngineURL } from "./scrape";+import { fireEngineFileSchema } from "./fileSchema";import { getDocFromGCS } from "../../../../lib/gcs-jobs";import { Meta } from "../..";@@ -104,24 +105,9 @@ const successSchema = z.object({.array().optional(),- // chrome-cdp only -- file download handler. Small files arrive inline as- // base64 `content`; large PDFs arrive as a GCS reference instead- // (fire-engine uploads them to its handoff bucket rather than inlining- // hundreds of MB of base64 into this response). Exactly one of- // `content` / `gcs_uri` is expected.- file: z- .object({- name: z.string(),- content: z.string().optional(),- gcs_uri: z.string().optional(),- sha256: z.string().optional(),- size_bytes: z.number().optional(),- })- .refine(f => (f.content !== undefined) !== (f.gcs_uri !== undefined), {- message: "file must carry exactly one of content or gcs_uri",- })- .optional()- .or(z.null()),+ // chrome-cdp only -- file download handler (inline base64 or a GCS+ // handoff reference; see fileSchema.ts).+ file: fireEngineFileSchema,docUrl: z.string().optional(),diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/fileSchema.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/fileSchema.tsnew file mode 100644index 000000000..5bce38da6--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/fileSchema.ts@@ -0,0 +1,29 @@+import { z } from "zod";++/**+ * The `file` member of a fire-engine chrome-cdp success response (the file+ * download handler). Small files arrive inline as base64 `content`; large+ * PDFs arrive as a GCS reference instead (fire-engine uploads them to its+ * handoff bucket rather than inlining hundreds of MB of base64 into the+ * response). Exactly one of `content` / `gcs_uri` is expected.+ *+ * Shared by BOTH response parsers on purpose. fire-engine can return the+ * finished job from `POST /scrape` directly (fireEngineScrape) or from the+ * `GET /scrape/:id` poll (fireEngineCheckStatus); a handoff that only one of+ * them understood was rejected as "response not matched by any schema" on+ * the fast path, which made every quick large-PDF download look like an+ * engine failure and re-ran the download on the next engine.+ */+export const fireEngineFileSchema = z+ .object({+ name: z.string(),+ content: z.string().optional(),+ gcs_uri: z.string().optional(),+ sha256: z.string().optional(),+ size_bytes: z.number().optional(),+ })+ .refine(f => (f.content !== undefined) !== (f.gcs_uri !== undefined), {+ message: "file must carry exactly one of content or gcs_uri",+ })+ .optional()+ .or(z.null());diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 6c9f32573..de0cc7cb7 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -5,6 +5,7 @@ import { InternalAction } from "../../../../controllers/v1/types";import { robustFetch } from "../../lib/fetch";import { MockState } from "../../lib/mock";import { getDocFromGCS } from "../../../../lib/gcs-jobs";+import { fireEngineFileSchema } from "./fileSchema";import {ActionError,AddFeatureError,@@ -155,14 +156,12 @@ const successSchema = z.object({.array().optional(),- // chrome-cdp only -- file download handler- file: z- .object({- name: z.string(),- content: z.string(),- })- .optional()- .or(z.null()),+ // chrome-cdp only -- file download handler (inline base64 or a GCS+ // handoff reference; see fileSchema.ts). Must accept exactly what the+ // poll parser in checkStatus.ts accepts: fire-engine returns finished+ // jobs from POST /scrape too, and a handoff rejected here surfaced as+ // "response not matched by any schema" -> a spurious engine failure.+ file: fireEngineFileSchema,docUrl: z.string().optional(),