Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(pdf): update request metadata (#4609)

Abimael Martellbe71a2182659
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 335a60c9e..c9ac123f3 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -42,6 +42,7 @@ import {
NoCachedDataError,
} from "../error";
import { isUrlBlocked } from "../../WebScraper/utils/blocklist";
+import { hasCustomRequestContext } from "../lib/request-context";
import {
canUseExchangeForRequest,
type ExchangeScrapeMetadata,
@@ -629,10 +630,7 @@ export function shouldUseIndex(meta: Meta) {
!getPDFPageMarkers(meta.options.parsers) &&
!hasCustomScreenshotSettings &&
meta.options.maxAge !== 0 &&
- (meta.options.headers === undefined ||
- Object.keys(meta.options.headers).length === 0) &&
- (meta.options.actions === undefined || meta.options.actions.length === 0) &&
- meta.options.profile === undefined
+ !hasCustomRequestContext(meta.options)
);
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index a46d29dde..c0bbfd5e3 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -44,6 +44,7 @@ import {
shouldParsePDF,
} from "../../../../controllers/v2/types";
import { hasFormatOfType } from "../../../../lib/format-utils";
+import { hasCustomRequestContext } from "../../lib/request-context";
export async function sendDocumentToIndex(meta: Meta, document: Document) {
// Skip caching if screenshot format has custom viewport or quality settings
@@ -85,9 +86,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
meta.winnerEngine !== "fetch")) &&
!meta.featureFlags.has("actions") &&
!hasCustomScreenshotSettings &&
- (meta.options.headers === undefined ||
- Object.keys(meta.options.headers).length === 0) &&
- meta.options.profile === undefined;
+ !hasCustomRequestContext(meta.options);
if (!shouldCache) {
return document;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
index 4486e3999..4b62c4a63 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
@@ -22,6 +22,7 @@ function makeMeta() {
id: "sync-page-markdown-test",
url: "https://example.com/file.pdf",
rewrittenUrl: undefined,
+ options: {},
logger,
mock: null,
abort: {
@@ -104,10 +105,48 @@ describe("scrapePDFWithFirePDF request metadata", () => {
string,
unknown
>;
- expect(body).toMatchObject({ source: "firecrawl", ...expected });
+ expect(body).toMatchObject({
+ source: "firecrawl",
+ source_request_context: "default",
+ ...expected,
+ });
expect(Object.hasOwn(body, "url")).toBe(Object.hasOwn(expected, "url"));
},
);
+
+ it.each([
+ ["empty options", {}, "default"],
+ ["empty headers and actions", { headers: {}, actions: [] }, "default"],
+ ["header", { headers: { "X-Example": "example-value" } }, "custom"],
+ ["empty header value", { headers: { "X-Example": "" } }, "custom"],
+ ["cookie header", { headers: { Cookie: "example=value" } }, "custom"],
+ ["action", { actions: [{ type: "wait", milliseconds: 1 }] }, "custom"],
+ ["profile", { profile: { name: "example-profile" } }, "custom"],
+ [
+ "profile without saving changes",
+ { profile: { name: "example-profile", saveChanges: false } },
+ "custom",
+ ],
+ ] as const)(
+ "describes %s without forwarding option values",
+ async (_name, options, expected) => {
+ const meta = { ...makeMeta(), options };
+
+ await scrapePDFWithFirePDF(meta, "BASE64", 1);
+
+ const body = mockedRobustFetch.mock.calls[0][0].body as Record<
+ string,
+ unknown
+ >;
+ expect(body.source_request_context).toBe(expected);
+ expect(body).not.toHaveProperty("headers");
+ expect(body).not.toHaveProperty("actions");
+ expect(body).not.toHaveProperty("profile");
+ expect(JSON.stringify(body)).not.toContain("example-value");
+ expect(JSON.stringify(body)).not.toContain("example=value");
+ expect(JSON.stringify(body)).not.toContain("example-profile");
+ },
+ );
});
describe("reconcilePageCountWithFirePdf", () => {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
index 901cfd132..008e14084 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
@@ -167,6 +167,7 @@ describe("scrapePDFWithFirePDFAsync", () => {
expect(calls[0].body).toMatchObject({
source: "firecrawl",
source_endpoint: "scrape",
+ source_request_context: "default",
url: "https://example.com/doc.pdf",
});
// Account context rides the submit body (FirePDF ENG-5049).
@@ -196,9 +197,41 @@ describe("scrapePDFWithFirePDFAsync", () => {
const body = JSON.parse((fetchImpl.mock.calls[0] as any)[1].body);
expect(body.source).toBe("firecrawl");
expect(body.source_endpoint).toBe("parse");
+ expect(body.source_request_context).toBe("default");
expect(body).not.toHaveProperty("url");
});
+ it("describes custom options without forwarding their values", async () => {
+ const fetchImpl = vi.fn(async (url: string) =>
+ jsonResp({
+ status: 200,
+ body: url.endsWith("/result")
+ ? { schema_version: 1, markdown: "Document", pages_processed: 1 }
+ : { scrape_id: "scrape-id-test", status: "done", lane: "fast" },
+ }),
+ );
+ const meta = makeMeta();
+ meta.options.headers = { "X-Example": "example-value" };
+ meta.options.actions = [{ type: "write", text: "example-text" }];
+ meta.options.profile = { name: "example-profile", saveChanges: false };
+ meta.url = " document source ";
+
+ await scrapePDFWithFirePDFAsync(meta, "BASE64", 1, 1, "auto", {
+ fetchImpl: fetchImpl as any,
+ sleepImpl: noopSleep,
+ });
+
+ const body = JSON.parse((fetchImpl.mock.calls[0] as any)[1].body);
+ expect(body.source_request_context).toBe("custom");
+ expect(body.url).toBe(meta.url);
+ expect(body).not.toHaveProperty("headers");
+ expect(body).not.toHaveProperty("actions");
+ expect(body).not.toHaveProperty("profile");
+ expect(JSON.stringify(body)).not.toContain("example-value");
+ expect(JSON.stringify(body)).not.toContain("example-text");
+ expect(JSON.stringify(body)).not.toContain("example-profile");
+ });
+
it("requests and returns physical page markdown", async () => {
const { fetchImpl, calls } = makeFetchFromSequence([
{
@@ -1159,6 +1192,7 @@ describe("scrapePDFWithFirePDFAsync", () => {
expect(body.input_sha256).toBe(BY_REF.sha256);
expect(body.source).toBe("firecrawl");
expect(body.source_endpoint).toBe("scrape");
+ expect(body.source_request_context).toBe("default");
expect(body.url).toBe(meta.url);
expect(body.pdf_b64).toBeUndefined();
expect((body.options as { pages_estimate?: number }).pages_estimate).toBe(
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts
index 6e80a2c5f..55085b244 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts
@@ -7,6 +7,46 @@ import { config } from "../../../../../config";
import { shouldUseIndex } from "../../index";
import { sendDocumentToIndex } from "../../index/index";
+describe("URL index request context", () => {
+ it.each([
+ ["default options", {}, true],
+ ["empty headers and actions", { headers: {}, actions: [] }, true],
+ ["custom header", { headers: { "X-Example": "example-value" } }, false],
+ ["empty header value", { headers: { "X-Example": "" } }, false],
+ ["action", { actions: [{ type: "wait", milliseconds: 1 }] }, false],
+ ["profile", { profile: { name: "example-profile" } }, false],
+ ])("preserves index eligibility for %s", async (_name, options, expected) => {
+ const originalWriteOnly = config.FIRECRAWL_INDEX_WRITE_ONLY;
+ (
+ config as { FIRECRAWL_INDEX_WRITE_ONLY?: boolean }
+ ).FIRECRAWL_INDEX_WRITE_ONLY = false;
+ try {
+ const meta = {
+ internalOptions: {},
+ winnerEngine: "fire-engine;chrome-cdp",
+ featureFlags: new Set(),
+ options: {
+ formats: ["markdown"],
+ maxAge: 60_000,
+ storeInCache: true,
+ ...options,
+ },
+ } as any;
+ expect(shouldUseIndex(meta)).toBe(expected);
+ if (!expected) {
+ const document = { markdown: "Example", metadata: {} } as any;
+ const result = await sendDocumentToIndex(meta, document);
+ expect(result).toBe(document);
+ expect(result.metadata.indexId).toBeUndefined();
+ }
+ } finally {
+ (
+ config as { FIRECRAWL_INDEX_WRITE_ONLY?: boolean }
+ ).FIRECRAWL_INDEX_WRITE_ONLY = originalWriteOnly;
+ }
+ });
+});
+
describe("PDF page-markdown URL index policy", () => {
it("bypasses URL-index reads for page-aware requests", () => {
const originalWriteOnly = config.FIRECRAWL_INDEX_WRITE_ONLY;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts
index 6b5e12105..9b3d954a9 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts
@@ -1,18 +1,28 @@
import type { Meta } from "../../..";
+import { hasCustomRequestContext } from "../../../lib/request-context";
export function buildFirePdfRequestMetadata(meta: Meta): {
source_endpoint: "scrape" | "parse";
+ source_request_context: "default" | "custom";
url?: string;
} {
const isParse =
meta.internalOptions.isParse === true ||
meta.internalOptions.uploadedFile !== undefined;
const source_endpoint = isParse ? "parse" : "scrape";
+ // Describe supplied request options without forwarding their values.
+ const source_request_context = hasCustomRequestContext(meta.options)
+ ? "custom"
+ : "default";
// Upload requests use synthetic URLs, and ZDR requests omit URL metadata.
if (isParse || meta.internalOptions.zeroDataRetention === true) {
- return { source_endpoint };
+ return { source_endpoint, source_request_context };
}
- return { source_endpoint, url: meta.rewrittenUrl ?? meta.url };
+ return {
+ source_endpoint,
+ source_request_context,
+ url: meta.rewrittenUrl ?? meta.url,
+ };
}
diff --git a/apps/api/src/scraper/scrapeURL/lib/request-context.ts b/apps/api/src/scraper/scrapeURL/lib/request-context.ts
new file mode 100644
index 000000000..4057adcfc
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/lib/request-context.ts
@@ -0,0 +1,11 @@
+import type { Meta } from "..";
+
+export function hasCustomRequestContext(
+ options: Pick<Meta["options"], "headers" | "actions" | "profile">,
+): boolean {
+ return (
+ Object.keys(options.headers ?? {}).length > 0 ||
+ (options.actions?.length ?? 0) > 0 ||
+ options.profile !== undefined
+ );
+}