Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore(api): add FirePDF request metadata (#4603)
Abimael Martell94cbdcc9bae2
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.tsindex 4acd9f504..4486e3999 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts@@ -36,6 +36,80 @@ function makeMeta() {} as any;}+describe("scrapePDFWithFirePDF request metadata", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ mockedRobustFetch.mockResolvedValue({+ markdown: "Document",+ failed_pages: null,+ pages_processed: 1,+ } as any);+ });++ it.each([+ [+ "original URL",+ {},+ {},+ { source_endpoint: "scrape", url: "https://example.com/file.pdf" },+ ],+ [+ "rewritten URL",+ { rewrittenUrl: "https://example.com/download/file.pdf" },+ {},+ {+ source_endpoint: "scrape",+ url: "https://example.com/download/file.pdf",+ },+ ],+ ["empty URL", { url: "" }, {}, { source_endpoint: "scrape", url: "" }],+ [+ "non-HTTP URL",+ { url: "file:///document.pdf" },+ {},+ { source_endpoint: "scrape", url: "file:///document.pdf" },+ ],+ [+ "unmodified source value",+ { url: " document source " },+ {},+ { source_endpoint: "scrape", url: " document source " },+ ],+ ["ZDR", {}, { zeroDataRetention: true }, { source_endpoint: "scrape" }],+ ["parse", {}, { isParse: true }, { source_endpoint: "parse" }],+ [+ "uploaded file",+ {},+ {+ uploadedFile: {+ buffer: Buffer.from("document"),+ filename: "document.pdf",+ },+ },+ { source_endpoint: "parse" },+ ],+ ] as const)(+ "sends source metadata for %s",+ async (_name, overrides, internalOptions, expected) => {+ const meta = { ...makeMeta(), ...overrides };+ meta.internalOptions = {+ ...meta.internalOptions,+ zeroDataRetention: false,+ ...internalOptions,+ };++ await scrapePDFWithFirePDF(meta, "BASE64", 1);++ const body = mockedRobustFetch.mock.calls[0][0].body as Record<+ string,+ unknown+ >;+ expect(body).toMatchObject({ source: "firecrawl", ...expected });+ expect(Object.hasOwn(body, "url")).toBe(Object.hasOwn(expected, "url"));+ },+ );+});+describe("reconcilePageCountWithFirePdf", () => {it("uses fire-pdf's count when the upstream pass left it at 0", () => {// The original regression: processPdf threw "Invalid PDF structure" on adiff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.tsindex 5dc1a0293..901cfd132 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts@@ -164,12 +164,41 @@ describe("scrapePDFWithFirePDFAsync", () => {expect(result.pagesProcessed).toBe(12);expect(fallback).not.toHaveBeenCalled();expect(calls).toHaveLength(4);+ expect(calls[0].body).toMatchObject({+ source: "firecrawl",+ source_endpoint: "scrape",+ url: "https://example.com/doc.pdf",+ });// Account context rides the submit body (FirePDF ENG-5049).expect((calls[0].body as { team_concurrency?: number }).team_concurrency,).toBe(12);});+ it("marks parse requests without forwarding their synthetic upload URL", async () => {+ const fetchImpl = vi.fn(async (url: string) =>+ jsonResp({+ status: 200,+ body: url.endsWith("/result")+ ? { schema_version: 1, markdown: "Document", pages_processed: 1 }+ : { scrape_id: "scrape-id-test", status: "done", lane: "fast" },+ }),+ );+ const meta = makeMeta();+ meta.internalOptions.isParse = true;+ meta.url = "https://example.com/uploads/document.pdf";++ await scrapePDFWithFirePDFAsync(meta, "BASE64", 1, 1, "auto", {+ fetchImpl: fetchImpl as any,+ sleepImpl: noopSleep,+ });++ const body = JSON.parse((fetchImpl.mock.calls[0] as any)[1].body);+ expect(body.source).toBe("firecrawl");+ expect(body.source_endpoint).toBe("parse");+ expect(body).not.toHaveProperty("url");+ });+it("requests and returns physical page markdown", async () => {const { fetchImpl, calls } = makeFetchFromSequence([{@@ -1128,6 +1157,9 @@ describe("scrapePDFWithFirePDFAsync", () => {const body = calls[0].body as Record<string, unknown>;expect(body.input_gcs_uri).toBe(BY_REF.gcsUri);expect(body.input_sha256).toBe(BY_REF.sha256);+ expect(body.source).toBe("firecrawl");+ expect(body.source_endpoint).toBe("scrape");+ expect(body.url).toBe(meta.url);expect(body.pdf_b64).toBeUndefined();expect((body.options as { pages_estimate?: number }).pages_estimate).toBe(6543,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.tsnew file mode 100644index 000000000..6b5e12105--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts@@ -0,0 +1,18 @@+import type { Meta } from "../../..";++export function buildFirePdfRequestMetadata(meta: Meta): {+ source_endpoint: "scrape" | "parse";+ url?: string;+} {+ const isParse =+ meta.internalOptions.isParse === true ||+ meta.internalOptions.uploadedFile !== undefined;+ const source_endpoint = isParse ? "parse" : "scrape";++ // Upload requests use synthetic URLs, and ZDR requests omit URL metadata.+ if (isParse || meta.internalOptions.zeroDataRetention === true) {+ return { source_endpoint };+ }++ return { source_endpoint, url: meta.rewrittenUrl ?? meta.url };+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.tsindex 762a99af3..28f7a1aea 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts@@ -2,6 +2,7 @@ import type { Meta } from "../../..";import type { PDFMode } from "../../../../../controllers/v2/types";import { fetch as undiciFetch } from "undici";import { AbortManagerThrownError } from "../../../lib/abortManager";+import { buildFirePdfRequestMetadata } from "./request-metadata";import {firePdfAsyncSubmitRetriesTotal,firePdfAsyncSubmittedTotal,@@ -131,6 +132,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {: { input_gcs_uri: input.gcsUri, input_sha256: input.sha256 }),scrape_id: scrapeId,source: "firecrawl" as const,+ ...buildFirePdfRequestMetadata(meta),zdr: false as const,deadline_at: deadlineAt,...(meta.internalOptions.teamId && {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsindex de9d45779..9ebc4c937 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -8,6 +8,7 @@ import { safeMarkdownToHtml } from "./markdownToHtml";import { createPdfCacheKey } from "../../../../lib/gcs-pdf-cache";import { maybeSaveResult, tryGetCached } from "./fire-pdf/cache";import { firePdfBlocksSchema, firePdfPagesSchema } from "./fire-pdf/schema";+import { buildFirePdfRequestMetadata } from "./fire-pdf/request-metadata";/*** Reconcile an existing page count with what fire-pdf reported.@@ -138,7 +139,7 @@ export async function scrapePDFWithFirePDF(...(meta.internalOptions.crawlId && {crawl_id: meta.internalOptions.crawlId,}),- ...(zdr ? {} : { url: meta.rewrittenUrl ?? meta.url }),+ ...buildFirePdfRequestMetadata(meta),pdf_sha256: pdfSha256,source: "firecrawl",zdr,