Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

chore(api): add FirePDF request metadata (#4603)

Abimael Martell94cbdcc9bae2
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
index 4acd9f504..4486e3999 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
@@ -36,6 +36,80 @@ function makeMeta() {
} as any;
}
+describe("scrapePDFWithFirePDF request metadata", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mockedRobustFetch.mockResolvedValue({
+ markdown: "Document",
+ failed_pages: null,
+ pages_processed: 1,
+ } as any);
+ });
+
+ it.each([
+ [
+ "original URL",
+ {},
+ {},
+ { source_endpoint: "scrape", url: "https://example.com/file.pdf" },
+ ],
+ [
+ "rewritten URL",
+ { rewrittenUrl: "https://example.com/download/file.pdf" },
+ {},
+ {
+ source_endpoint: "scrape",
+ url: "https://example.com/download/file.pdf",
+ },
+ ],
+ ["empty URL", { url: "" }, {}, { source_endpoint: "scrape", url: "" }],
+ [
+ "non-HTTP URL",
+ { url: "file:///document.pdf" },
+ {},
+ { source_endpoint: "scrape", url: "file:///document.pdf" },
+ ],
+ [
+ "unmodified source value",
+ { url: " document source " },
+ {},
+ { source_endpoint: "scrape", url: " document source " },
+ ],
+ ["ZDR", {}, { zeroDataRetention: true }, { source_endpoint: "scrape" }],
+ ["parse", {}, { isParse: true }, { source_endpoint: "parse" }],
+ [
+ "uploaded file",
+ {},
+ {
+ uploadedFile: {
+ buffer: Buffer.from("document"),
+ filename: "document.pdf",
+ },
+ },
+ { source_endpoint: "parse" },
+ ],
+ ] as const)(
+ "sends source metadata for %s",
+ async (_name, overrides, internalOptions, expected) => {
+ const meta = { ...makeMeta(), ...overrides };
+ meta.internalOptions = {
+ ...meta.internalOptions,
+ zeroDataRetention: false,
+ ...internalOptions,
+ };
+
+ await scrapePDFWithFirePDF(meta, "BASE64", 1);
+
+ const body = mockedRobustFetch.mock.calls[0][0].body as Record<
+ string,
+ unknown
+ >;
+ expect(body).toMatchObject({ source: "firecrawl", ...expected });
+ expect(Object.hasOwn(body, "url")).toBe(Object.hasOwn(expected, "url"));
+ },
+ );
+});
+
describe("reconcilePageCountWithFirePdf", () => {
it("uses fire-pdf's count when the upstream pass left it at 0", () => {
// The original regression: processPdf threw "Invalid PDF structure" on a
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
index 5dc1a0293..901cfd132 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
@@ -164,12 +164,41 @@ describe("scrapePDFWithFirePDFAsync", () => {
expect(result.pagesProcessed).toBe(12);
expect(fallback).not.toHaveBeenCalled();
expect(calls).toHaveLength(4);
+ expect(calls[0].body).toMatchObject({
+ source: "firecrawl",
+ source_endpoint: "scrape",
+ url: "https://example.com/doc.pdf",
+ });
// Account context rides the submit body (FirePDF ENG-5049).
expect(
(calls[0].body as { team_concurrency?: number }).team_concurrency,
).toBe(12);
});
+ it("marks parse requests without forwarding their synthetic upload URL", async () => {
+ const fetchImpl = vi.fn(async (url: string) =>
+ jsonResp({
+ status: 200,
+ body: url.endsWith("/result")
+ ? { schema_version: 1, markdown: "Document", pages_processed: 1 }
+ : { scrape_id: "scrape-id-test", status: "done", lane: "fast" },
+ }),
+ );
+ const meta = makeMeta();
+ meta.internalOptions.isParse = true;
+ meta.url = "https://example.com/uploads/document.pdf";
+
+ await scrapePDFWithFirePDFAsync(meta, "BASE64", 1, 1, "auto", {
+ fetchImpl: fetchImpl as any,
+ sleepImpl: noopSleep,
+ });
+
+ const body = JSON.parse((fetchImpl.mock.calls[0] as any)[1].body);
+ expect(body.source).toBe("firecrawl");
+ expect(body.source_endpoint).toBe("parse");
+ expect(body).not.toHaveProperty("url");
+ });
+
it("requests and returns physical page markdown", async () => {
const { fetchImpl, calls } = makeFetchFromSequence([
{
@@ -1128,6 +1157,9 @@ describe("scrapePDFWithFirePDFAsync", () => {
const body = calls[0].body as Record<string, unknown>;
expect(body.input_gcs_uri).toBe(BY_REF.gcsUri);
expect(body.input_sha256).toBe(BY_REF.sha256);
+ expect(body.source).toBe("firecrawl");
+ expect(body.source_endpoint).toBe("scrape");
+ expect(body.url).toBe(meta.url);
expect(body.pdf_b64).toBeUndefined();
expect((body.options as { pages_estimate?: number }).pages_estimate).toBe(
6543,
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts
new file mode 100644
index 000000000..6b5e12105
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts
@@ -0,0 +1,18 @@
+import type { Meta } from "../../..";
+
+export function buildFirePdfRequestMetadata(meta: Meta): {
+ source_endpoint: "scrape" | "parse";
+ url?: string;
+} {
+ const isParse =
+ meta.internalOptions.isParse === true ||
+ meta.internalOptions.uploadedFile !== undefined;
+ const source_endpoint = isParse ? "parse" : "scrape";
+
+ // Upload requests use synthetic URLs, and ZDR requests omit URL metadata.
+ if (isParse || meta.internalOptions.zeroDataRetention === true) {
+ return { source_endpoint };
+ }
+
+ return { source_endpoint, url: meta.rewrittenUrl ?? meta.url };
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
index 762a99af3..28f7a1aea 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
@@ -2,6 +2,7 @@ import type { Meta } from "../../..";
import type { PDFMode } from "../../../../../controllers/v2/types";
import { fetch as undiciFetch } from "undici";
import { AbortManagerThrownError } from "../../../lib/abortManager";
+import { buildFirePdfRequestMetadata } from "./request-metadata";
import {
firePdfAsyncSubmitRetriesTotal,
firePdfAsyncSubmittedTotal,
@@ -131,6 +132,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
: { input_gcs_uri: input.gcsUri, input_sha256: input.sha256 }),
scrape_id: scrapeId,
source: "firecrawl" as const,
+ ...buildFirePdfRequestMetadata(meta),
zdr: false as const,
deadline_at: deadlineAt,
...(meta.internalOptions.teamId && {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
index de9d45779..9ebc4c937 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
@@ -8,6 +8,7 @@ import { safeMarkdownToHtml } from "./markdownToHtml";
import { createPdfCacheKey } from "../../../../lib/gcs-pdf-cache";
import { maybeSaveResult, tryGetCached } from "./fire-pdf/cache";
import { firePdfBlocksSchema, firePdfPagesSchema } from "./fire-pdf/schema";
+import { buildFirePdfRequestMetadata } from "./fire-pdf/request-metadata";
/**
* Reconcile an existing page count with what fire-pdf reported.
@@ -138,7 +139,7 @@ export async function scrapePDFWithFirePDF(
...(meta.internalOptions.crawlId && {
crawl_id: meta.internalOptions.crawlId,
}),
- ...(zdr ? {} : { url: meta.rewrittenUrl ?? meta.url }),
+ ...buildFirePdfRequestMetadata(meta),
pdf_sha256: pdfSha256,
source: "firecrawl",
zdr,