Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(pdf): update request metadata (#4609)
Abimael Martellbe71a2182659
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 335a60c9e..c9ac123f3 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -42,6 +42,7 @@ import {NoCachedDataError,} from "../error";import { isUrlBlocked } from "../../WebScraper/utils/blocklist";+import { hasCustomRequestContext } from "../lib/request-context";import {canUseExchangeForRequest,type ExchangeScrapeMetadata,@@ -629,10 +630,7 @@ export function shouldUseIndex(meta: Meta) {!getPDFPageMarkers(meta.options.parsers) &&!hasCustomScreenshotSettings &&meta.options.maxAge !== 0 &&- (meta.options.headers === undefined ||- Object.keys(meta.options.headers).length === 0) &&- (meta.options.actions === undefined || meta.options.actions.length === 0) &&- meta.options.profile === undefined+ !hasCustomRequestContext(meta.options));}diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex a46d29dde..c0bbfd5e3 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -44,6 +44,7 @@ import {shouldParsePDF,} from "../../../../controllers/v2/types";import { hasFormatOfType } from "../../../../lib/format-utils";+import { hasCustomRequestContext } from "../../lib/request-context";export async function sendDocumentToIndex(meta: Meta, document: Document) {// Skip caching if screenshot format has custom viewport or quality settings@@ -85,9 +86,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {meta.winnerEngine !== "fetch")) &&!meta.featureFlags.has("actions") &&!hasCustomScreenshotSettings &&- (meta.options.headers === undefined ||- Object.keys(meta.options.headers).length === 0) &&- meta.options.profile === undefined;+ !hasCustomRequestContext(meta.options);if (!shouldCache) {return document;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.tsindex 4486e3999..4b62c4a63 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts@@ -22,6 +22,7 @@ function makeMeta() {id: "sync-page-markdown-test",url: "https://example.com/file.pdf",rewrittenUrl: undefined,+ options: {},logger,mock: null,abort: {@@ -104,10 +105,48 @@ describe("scrapePDFWithFirePDF request metadata", () => {string,unknown>;- expect(body).toMatchObject({ source: "firecrawl", ...expected });+ expect(body).toMatchObject({+ source: "firecrawl",+ source_request_context: "default",+ ...expected,+ });expect(Object.hasOwn(body, "url")).toBe(Object.hasOwn(expected, "url"));},);++ it.each([+ ["empty options", {}, "default"],+ ["empty headers and actions", { headers: {}, actions: [] }, "default"],+ ["header", { headers: { "X-Example": "example-value" } }, "custom"],+ ["empty header value", { headers: { "X-Example": "" } }, "custom"],+ ["cookie header", { headers: { Cookie: "example=value" } }, "custom"],+ ["action", { actions: [{ type: "wait", milliseconds: 1 }] }, "custom"],+ ["profile", { profile: { name: "example-profile" } }, "custom"],+ [+ "profile without saving changes",+ { profile: { name: "example-profile", saveChanges: false } },+ "custom",+ ],+ ] as const)(+ "describes %s without forwarding option values",+ async (_name, options, expected) => {+ const meta = { ...makeMeta(), options };++ await scrapePDFWithFirePDF(meta, "BASE64", 1);++ const body = mockedRobustFetch.mock.calls[0][0].body as Record<+ string,+ unknown+ >;+ expect(body.source_request_context).toBe(expected);+ expect(body).not.toHaveProperty("headers");+ expect(body).not.toHaveProperty("actions");+ expect(body).not.toHaveProperty("profile");+ expect(JSON.stringify(body)).not.toContain("example-value");+ expect(JSON.stringify(body)).not.toContain("example=value");+ expect(JSON.stringify(body)).not.toContain("example-profile");+ },+ );});describe("reconcilePageCountWithFirePdf", () => {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.tsindex 901cfd132..008e14084 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts@@ -167,6 +167,7 @@ describe("scrapePDFWithFirePDFAsync", () => {expect(calls[0].body).toMatchObject({source: "firecrawl",source_endpoint: "scrape",+ source_request_context: "default",url: "https://example.com/doc.pdf",});// Account context rides the submit body (FirePDF ENG-5049).@@ -196,9 +197,41 @@ describe("scrapePDFWithFirePDFAsync", () => {const body = JSON.parse((fetchImpl.mock.calls[0] as any)[1].body);expect(body.source).toBe("firecrawl");expect(body.source_endpoint).toBe("parse");+ expect(body.source_request_context).toBe("default");expect(body).not.toHaveProperty("url");});+ it("describes custom options without forwarding their values", async () => {+ const fetchImpl = vi.fn(async (url: string) =>+ jsonResp({+ status: 200,+ body: url.endsWith("/result")+ ? { schema_version: 1, markdown: "Document", pages_processed: 1 }+ : { scrape_id: "scrape-id-test", status: "done", lane: "fast" },+ }),+ );+ const meta = makeMeta();+ meta.options.headers = { "X-Example": "example-value" };+ meta.options.actions = [{ type: "write", text: "example-text" }];+ meta.options.profile = { name: "example-profile", saveChanges: false };+ meta.url = " document source ";++ await scrapePDFWithFirePDFAsync(meta, "BASE64", 1, 1, "auto", {+ fetchImpl: fetchImpl as any,+ sleepImpl: noopSleep,+ });++ const body = JSON.parse((fetchImpl.mock.calls[0] as any)[1].body);+ expect(body.source_request_context).toBe("custom");+ expect(body.url).toBe(meta.url);+ expect(body).not.toHaveProperty("headers");+ expect(body).not.toHaveProperty("actions");+ expect(body).not.toHaveProperty("profile");+ expect(JSON.stringify(body)).not.toContain("example-value");+ expect(JSON.stringify(body)).not.toContain("example-text");+ expect(JSON.stringify(body)).not.toContain("example-profile");+ });+it("requests and returns physical page markdown", async () => {const { fetchImpl, calls } = makeFetchFromSequence([{@@ -1159,6 +1192,7 @@ describe("scrapePDFWithFirePDFAsync", () => {expect(body.input_sha256).toBe(BY_REF.sha256);expect(body.source).toBe("firecrawl");expect(body.source_endpoint).toBe("scrape");+ expect(body.source_request_context).toBe("default");expect(body.url).toBe(meta.url);expect(body.pdf_b64).toBeUndefined();expect((body.options as { pages_estimate?: number }).pages_estimate).toBe(diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.tsindex 6e80a2c5f..55085b244 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts@@ -7,6 +7,46 @@ import { config } from "../../../../../config";import { shouldUseIndex } from "../../index";import { sendDocumentToIndex } from "../../index/index";+describe("URL index request context", () => {+ it.each([+ ["default options", {}, true],+ ["empty headers and actions", { headers: {}, actions: [] }, true],+ ["custom header", { headers: { "X-Example": "example-value" } }, false],+ ["empty header value", { headers: { "X-Example": "" } }, false],+ ["action", { actions: [{ type: "wait", milliseconds: 1 }] }, false],+ ["profile", { profile: { name: "example-profile" } }, false],+ ])("preserves index eligibility for %s", async (_name, options, expected) => {+ const originalWriteOnly = config.FIRECRAWL_INDEX_WRITE_ONLY;+ (+ config as { FIRECRAWL_INDEX_WRITE_ONLY?: boolean }+ ).FIRECRAWL_INDEX_WRITE_ONLY = false;+ try {+ const meta = {+ internalOptions: {},+ winnerEngine: "fire-engine;chrome-cdp",+ featureFlags: new Set(),+ options: {+ formats: ["markdown"],+ maxAge: 60_000,+ storeInCache: true,+ ...options,+ },+ } as any;+ expect(shouldUseIndex(meta)).toBe(expected);+ if (!expected) {+ const document = { markdown: "Example", metadata: {} } as any;+ const result = await sendDocumentToIndex(meta, document);+ expect(result).toBe(document);+ expect(result.metadata.indexId).toBeUndefined();+ }+ } finally {+ (+ config as { FIRECRAWL_INDEX_WRITE_ONLY?: boolean }+ ).FIRECRAWL_INDEX_WRITE_ONLY = originalWriteOnly;+ }+ });+});+describe("PDF page-markdown URL index policy", () => {it("bypasses URL-index reads for page-aware requests", () => {const originalWriteOnly = config.FIRECRAWL_INDEX_WRITE_ONLY;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.tsindex 6b5e12105..9b3d954a9 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts@@ -1,18 +1,28 @@import type { Meta } from "../../..";+import { hasCustomRequestContext } from "../../../lib/request-context";export function buildFirePdfRequestMetadata(meta: Meta): {source_endpoint: "scrape" | "parse";+ source_request_context: "default" | "custom";url?: string;} {const isParse =meta.internalOptions.isParse === true ||meta.internalOptions.uploadedFile !== undefined;const source_endpoint = isParse ? "parse" : "scrape";+ // Describe supplied request options without forwarding their values.+ const source_request_context = hasCustomRequestContext(meta.options)+ ? "custom"+ : "default";// Upload requests use synthetic URLs, and ZDR requests omit URL metadata.if (isParse || meta.internalOptions.zeroDataRetention === true) {- return { source_endpoint };+ return { source_endpoint, source_request_context };}- return { source_endpoint, url: meta.rewrittenUrl ?? meta.url };+ return {+ source_endpoint,+ source_request_context,+ url: meta.rewrittenUrl ?? meta.url,+ };}diff --git a/apps/api/src/scraper/scrapeURL/lib/request-context.ts b/apps/api/src/scraper/scrapeURL/lib/request-context.tsnew file mode 100644index 000000000..4057adcfc--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/lib/request-context.ts@@ -0,0 +1,11 @@+import type { Meta } from "..";++export function hasCustomRequestContext(+ options: Pick<Meta["options"], "headers" | "actions" | "profile">,+): boolean {+ return (+ Object.keys(options.headers ?? {}).length > 0 ||+ (options.actions?.length ?? 0) > 0 ||+ options.profile !== undefined+ );+}