Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(api): expose PDF page markdown (#4219)

Abimael Martellfd240843e19d
diff --git a/apps/api/src/__tests__/snips/v2/parse.test.ts b/apps/api/src/__tests__/snips/v2/parse.test.ts
index ba2586e41..ee8a3f07d 100644
--- a/apps/api/src/__tests__/snips/v2/parse.test.ts
+++ b/apps/api/src/__tests__/snips/v2/parse.test.ts
@@ -606,6 +606,45 @@ describe("/v2/parse", () => {
);
});
+ describeIf(ALLOW_TEST_SUITE_WEBSITE && !!config.FIRE_PDF_BASE_URL)(
+ "PDF upload physical page markdown",
+ () => {
+ it(
+ "returns physical pages for an uploaded PDF",
+ async () => {
+ expect(pdfFixture).not.toBeNull();
+
+ const result = await parse(
+ {
+ options: {
+ formats: ["markdown"],
+ parsers: [{ type: "pdf", mode: "auto", pageMarkdown: true }],
+ },
+ file: {
+ content: pdfFixture!,
+ filename: "upload.pdf",
+ contentType: "application/pdf",
+ },
+ },
+ identity,
+ );
+
+ expect(result.markdown).toContain("PDF Test File");
+ expect(result.pages).toHaveLength(result.metadata.numPages!);
+ result.pages?.forEach((page, index) => {
+ expect(page).toEqual(
+ expect.objectContaining({
+ pageNumber: index + 1,
+ markdown: expect.any(String),
+ }),
+ );
+ });
+ },
+ scrapeTimeout * 3,
+ );
+ },
+ );
+
it(
"returns a validation error when file is missing",
async () => {
diff --git a/apps/api/src/__tests__/snips/v2/parsers-pdf-page-markdown.test.ts b/apps/api/src/__tests__/snips/v2/parsers-pdf-page-markdown.test.ts
new file mode 100644
index 000000000..57b243457
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/parsers-pdf-page-markdown.test.ts
@@ -0,0 +1,47 @@
+import {
+ ALLOW_TEST_SUITE_WEBSITE,
+ describeIf,
+ TEST_SUITE_WEBSITE,
+} from "../lib";
+import { config } from "../../../config";
+import { idmux, Identity, scrape, scrapeTimeout } from "./lib";
+
+let identity: Identity;
+
+beforeAll(async () => {
+ identity = await idmux({
+ name: "parsers-pdf-page-markdown",
+ concurrency: 100,
+ credits: 1000000,
+ });
+}, 10000 + scrapeTimeout);
+
+const SHOULD_RUN = ALLOW_TEST_SUITE_WEBSITE && !!config.FIRE_PDF_BASE_URL;
+
+describeIf(SHOULD_RUN)("PDF physical page markdown", () => {
+ it(
+ "returns additive pages without changing document markdown",
+ async () => {
+ const response = await scrape(
+ {
+ url: `${TEST_SUITE_WEBSITE}/example.pdf`,
+ formats: ["markdown"],
+ parsers: [{ type: "pdf", mode: "auto", pageMarkdown: true }],
+ },
+ identity,
+ );
+
+ expect(response.markdown).toContain("PDF Test File");
+ expect(response.pages).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ pageNumber: 1,
+ markdown: expect.any(String),
+ }),
+ ]),
+ );
+ expect(response.pages).toHaveLength(response.metadata.numPages!);
+ },
+ scrapeTimeout * 3,
+ );
+});
diff --git a/apps/api/src/__tests__/snips/v2/parsers.test.ts b/apps/api/src/__tests__/snips/v2/parsers.test.ts
index bd7b20681..dc8392353 100644
--- a/apps/api/src/__tests__/snips/v2/parsers.test.ts
+++ b/apps/api/src/__tests__/snips/v2/parsers.test.ts
@@ -387,6 +387,24 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("Parsers parameter tests", () => {
scrapeTimeout,
);
+ it.concurrent(
+ "rejects non-boolean pageMarkdown",
+ async () => {
+ const raw = await scrapeRaw(
+ {
+ url: pdfUrl,
+ parsers: [{ type: "pdf", pageMarkdown: "yes" } as any],
+ },
+ identity,
+ );
+
+ expect(raw.statusCode).toBe(400);
+ expect(raw.body.success).toBe(false);
+ expect(raw.body.error).toBe("Bad Request");
+ },
+ scrapeTimeout,
+ );
+
it.concurrent(
"rejects colon-separated shorthand strings",
async () => {
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
index fcdda5a63..1f849e983 100644
--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
@@ -435,6 +435,24 @@ describe("V2 Types Validation", () => {
expect((result.parsers as any)[0].maxPages).toBe(100);
});
+ it("should accept physical page markdown for PDF parsers", () => {
+ const result = scrapeRequestSchema.parse({
+ url: "https://example.com/file.pdf",
+ parsers: [{ type: "pdf", mode: "auto", pageMarkdown: true }],
+ });
+
+ expect((result.parsers as any)[0].pageMarkdown).toBe(true);
+ });
+
+ it("should reject non-boolean physical page markdown", () => {
+ expect(() =>
+ scrapeRequestSchema.parse({
+ url: "https://example.com/file.pdf",
+ parsers: [{ type: "pdf", pageMarkdown: "yes" }],
+ }),
+ ).toThrow();
+ });
+
it("should reject PDF parser with maxPages exceeding limit", () => {
const input: ScrapeRequestInput = {
url: "https://example.com",
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index b27eccb10..460d19150 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1005,6 +1005,8 @@ export type Document = {
description?: string;
url?: string;
markdown?: string;
+ /** Physical PDF pages, populated by the v2 pageMarkdown parser option. */
+ pages?: Array<{ pageNumber: number; markdown: string }>;
html?: string;
rawHtml?: string;
links?: string[];
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index d6b2075fd..4a1dc62f5 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -482,6 +482,8 @@ const pdfParserWithOptions = z.strictObject({
type: z.literal("pdf"),
mode: pdfModeSchema.optional(),
maxPages: z.int().positive().finite().max(10000).optional(),
+ /** Include physical per-page markdown alongside document markdown. */
+ pageMarkdown: z.boolean().optional(),
// Experimental: route this request through the fire-pdf async pipeline
// (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back
// to sync on any async-path failure, so user-visible behavior is unchanged
@@ -531,6 +533,16 @@ export function getPDFMode(parsers?: Parsers): PDFMode {
return "auto";
}
+export function getPDFPageMarkdown(parsers?: Parsers): boolean {
+ if (!parsers) return false;
+ for (const parser of parsers) {
+ if (typeof parser === "object" && parser.type === "pdf") {
+ return parser.pageMarkdown === true;
+ }
+ }
+ return false;
+}
+
export function getFirePdfAsync(parsers?: Parsers): boolean {
if (!parsers) return false;
for (const parser of parsers) {
@@ -1226,6 +1238,8 @@ export type Document = {
description?: string;
url?: string;
markdown?: string;
+ /** Physical PDF pages, present only for `parsers[].pageMarkdown`. */
+ pages?: Array<{ pageNumber: number; markdown: string }>;
html?: string;
rawHtml?: string;
links?: string[];
diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.ts
index fd6fbbeb4..4f89de8c5 100644
--- a/apps/api/src/lib/gcs-pdf-cache.ts
+++ b/apps/api/src/lib/gcs-pdf-cache.ts
@@ -13,6 +13,8 @@ type CachedPdfResult = {
markdown: string;
html: string;
pagesProcessed?: number;
+ /** Physical page markdown; present only in page-capable cache variants. */
+ pageMarkdown?: Array<{ page: number; markdown: string }>;
};
const PROVIDER_PREFIXES: Record<PdfCacheProvider, string> = {
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 7160bfc4d..5d1563a59 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -27,7 +27,10 @@ import {
} from "./x-twitter";
import { queryEngpickerVerdict, useIndex } from "../../../services";
import { hasFormatOfType } from "../../../lib/format-utils";
-import { getPDFMaxPages } from "../../../controllers/v2/types";
+import {
+ getPDFMaxPages,
+ getPDFPageMarkdown,
+} from "../../../controllers/v2/types";
import type { PdfMetadata } from "./pdf/types";
import { BrandingProfile } from "../../../types/branding";
import { BrandingNotSupportedError } from "../error";
@@ -138,6 +141,7 @@ export type EngineScrapeResult = {
html: string;
markdown?: string;
+ pages?: Array<{ pageNumber: number; markdown: string }>;
json?: unknown;
statusCode: number;
error?: string;
@@ -563,6 +567,8 @@ export function shouldUseIndex(meta: Meta) {
!hasFormatOfType(meta.options.formats, "branding") &&
// Skip index if a non-default PDF maxPages is specified
getPDFMaxPages(meta.options.parsers) === undefined &&
+ // The URL index does not yet persist physical-page capability metadata.
+ !getPDFPageMarkdown(meta.options.parsers) &&
!hasCustomScreenshotSettings &&
meta.options.maxAge !== 0 &&
(meta.options.headers === undefined ||
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 3863f972e..f282f9f33 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -36,7 +36,10 @@ import {
IndexMissError,
NoCachedDataError,
} from "../../error";
-import { shouldParsePDF } from "../../../../controllers/v2/types";
+import {
+ getPDFPageMarkdown,
+ shouldParsePDF,
+} from "../../../../controllers/v2/types";
import { hasFormatOfType } from "../../../../lib/format-utils";
export async function sendDocumentToIndex(meta: Meta, document: Document) {
@@ -56,6 +59,10 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
// every access must go through the Exchange and its ledger.
meta.winnerEngine !== "exchange" &&
!(meta.winnerEngine === "pdf" && !shouldParsePDF(meta.options.parsers)) &&
+ // Page-aware results are capability-specific and are not represented in
+ // the URL index schema yet. Do not write an entry that could later be
+ // served without its required pages payload.
+ !getPDFPageMarkdown(meta.options.parsers) &&
!meta.options.parsers?.some(parser => {
if (
typeof parser === "object" &&
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
index f5068ca85..d41be0cfa 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
@@ -1,4 +1,40 @@
-import { reconcilePageCountWithFirePdf } from "../firePDF";
+import { robustFetch } from "../../../lib/fetch";
+import {
+ reconcilePageCountWithFirePdf,
+ scrapePDFWithFirePDF,
+} from "../firePDF";
+
+vi.mock("../../../lib/fetch", () => ({
+ robustFetch: vi.fn(),
+}));
+
+const mockedRobustFetch = vi.mocked(robustFetch);
+
+function makeMeta() {
+ const logger: any = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ debug: vi.fn(),
+ };
+
+ return {
+ id: "sync-page-markdown-test",
+ url: "https://example.com/file.pdf",
+ rewrittenUrl: undefined,
+ logger,
+ mock: null,
+ abort: {
+ throwIfAborted: vi.fn(),
+ asSignal: vi.fn(() => new AbortController().signal),
+ scrapeTimeout: vi.fn(() => 60_000),
+ },
+ internalOptions: {
+ zeroDataRetention: true,
+ teamId: "team-test",
+ },
+ } as any;
+}
describe("reconcilePageCountWithFirePdf", () => {
it("uses fire-pdf's count when the upstream pass left it at 0", () => {
@@ -41,3 +77,59 @@ describe("reconcilePageCountWithFirePdf", () => {
expect(reconcilePageCountWithFirePdf(5, { pagesProcessed: 0 })).toBe(5);
});
});
+
+describe("scrapePDFWithFirePDF page markdown", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it("requests and returns the physical page payload", async () => {
+ mockedRobustFetch.mockResolvedValue({
+ markdown: "continued paragraph",
+ failed_pages: null,
+ pages_processed: 2,
+ pages: [
+ { page: 1, markdown: "continued" },
+ { page: 2, markdown: "paragraph" },
+ ],
+ } as any);
+
+ const result = await scrapePDFWithFirePDF(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ "auto",
+ true,
+ );
+
+ expect(mockedRobustFetch).toHaveBeenCalledWith(
+ expect.objectContaining({
+ body: expect.objectContaining({ include_page_markdown: true }),
+ }),
+ );
+ expect(result.pageMarkdown).toEqual([
+ { page: 1, markdown: "continued" },
+ { page: 2, markdown: "paragraph" },
+ ]);
+ });
+
+ it("rejects document-only FirePDF responses for page-aware requests", async () => {
+ mockedRobustFetch.mockResolvedValue({
+ markdown: "document only",
+ failed_pages: null,
+ pages_processed: 1,
+ } as any);
+
+ await expect(
+ scrapePDFWithFirePDF(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ "auto",
+ true,
+ ),
+ ).rejects.toThrow(/did not include requested physical page markdown/);
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
index e3efdaaf4..5dbef323e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
@@ -7,7 +7,6 @@ vi.mock("../../../../../lib/gcs-pdf-cache", () => ({
savePdfResultToCache: vi.fn(async () => null),
}));
-
import {
FirePdfAsyncFailure,
scrapePDFWithFirePDFAsync,
@@ -145,7 +144,7 @@ describe("scrapePDFWithFirePDFAsync", () => {
internalOptions: {
zeroDataRetention: true,
teamId: "team-x",
- teamConcurrency: 12,
+ teamConcurrency: 12,
crawlId: undefined,
},
});
@@ -252,7 +251,101 @@ describe("scrapePDFWithFirePDFAsync", () => {
expect(fallback).not.toHaveBeenCalled();
expect(calls).toHaveLength(4);
// Account context rides the submit body (FirePDF ENG-5049).
- expect((calls[0].body as { team_concurrency?: number }).team_concurrency).toBe(12);
+ expect(
+ (calls[0].body as { team_concurrency?: number }).team_concurrency,
+ ).toBe(12);
+ });
+
+ it("requests and returns physical page markdown", async () => {
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: {
+ status: 200,
+ body: {
+ scrape_id: "scrape-id-test",
+ status: "done",
+ lane: "fast",
+ },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: {
+ schema_version: 3,
+ markdown: "continued paragraph",
+ pages: [
+ { page: 1, markdown: "continued" },
+ { page: 2, markdown: "paragraph" },
+ ],
+ pages_processed: 2,
+ },
+ },
+ },
+ ]);
+
+ const result = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ "auto",
+ { fetchImpl, sleepImpl: noopSleep },
+ true,
+ );
+
+ expect((calls[0].body as any).options.include_page_markdown).toBe(true);
+ expect(result.markdown).toBe("continued paragraph");
+ expect(result.pageMarkdown).toEqual([
+ { page: 1, markdown: "continued" },
+ { page: 2, markdown: "paragraph" },
+ ]);
+ });
+
+ it("fails a page-aware request when FirePDF omits the page payload", async () => {
+ const { fetchImpl } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: {
+ status: 200,
+ body: {
+ scrape_id: "scrape-id-test",
+ status: "done",
+ lane: "fast",
+ },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: {
+ schema_version: 1,
+ markdown: "document only",
+ pages_processed: 1,
+ },
+ },
+ },
+ ]);
+
+ const error = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ "auto",
+ { fetchImpl, sleepImpl: noopSleep },
+ true,
+ ).catch(error => error);
+
+ expect(error).toBeInstanceOf(FirePdfAsyncFailure);
+ expect(error.reason).toBe("http_5xx");
});
it("submits without team context when the snapshot is absent", async () => {
@@ -262,7 +355,12 @@ describe("scrapePDFWithFirePDFAsync", () => {
matchMethod: "POST",
response: {
status: 200,
- body: { scrape_id: "scrape-id-test", status: "done", lane: "fast", retry_after_ms: 0 },
+ body: {
+ scrape_id: "scrape-id-test",
+ status: "done",
+ lane: "fast",
+ retry_after_ms: 0,
+ },
},
},
{
@@ -284,15 +382,24 @@ describe("scrapePDFWithFirePDFAsync", () => {
const meta = makeMeta();
meta.internalOptions.teamConcurrency = null;
- const result = await scrapePDFWithFirePDFAsync(meta, "BASE64", undefined, undefined, undefined, {
- fetchImpl,
- fallbackImpl: fallback,
- sleepImpl: noopSleep,
- });
+ const result = await scrapePDFWithFirePDFAsync(
+ meta,
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ {
+ fetchImpl,
+ fallbackImpl: fallback,
+ sleepImpl: noopSleep,
+ },
+ );
// Missing snapshot must never block the scrape β€” field simply absent.
expect(result.markdown).toBe("# No context");
- expect((calls[0].body as { team_concurrency?: number }).team_concurrency).toBeUndefined();
+ expect(
+ (calls[0].body as { team_concurrency?: number }).team_concurrency,
+ ).toBeUndefined();
expect(fallback).not.toHaveBeenCalled();
});
@@ -335,7 +442,13 @@ describe("scrapePDFWithFirePDFAsync", () => {
expect(error).toBeInstanceOf(FirePdfAsyncFailure);
expect(error.reason).toBe("network_error");
- expect(calls.map(({ url, method, headers }) => ({ url, method, ...(headers !== undefined && { headers }) }))).toEqual([
+ expect(
+ calls.map(({ url, method, headers }) => ({
+ url,
+ method,
+ ...(headers !== undefined && { headers }),
+ })),
+ ).toEqual([
{
url: "http://fire-pdf.test/jobs",
method: "POST",
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
new file mode 100644
index 000000000..e741a6ac9
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
@@ -0,0 +1,299 @@
+import {
+ getPdfResultFromCache,
+ savePdfResultToCache,
+} from "../../../../../lib/gcs-pdf-cache";
+import {
+ cacheKeyShape,
+ maybeSaveResult,
+ tryGetCached,
+} from "../fire-pdf/cache";
+
+vi.mock("../../../../../lib/gcs-pdf-cache", () => ({
+ getPdfResultFromCache: vi.fn(),
+ savePdfResultToCache: vi.fn(),
+}));
+
+const getCached = vi.mocked(getPdfResultFromCache);
+const saveCached = vi.mocked(savePdfResultToCache);
+
+function makeMeta(zeroDataRetention = false) {
+ return {
+ id: "page-cache-test",
+ logger: {
+ info: vi.fn(),
+ warn: vi.fn(),
+ },
+ internalOptions: { zeroDataRetention },
+ } as any;
+}
+
+describe("FirePDF page-markdown cache capabilities", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ getCached.mockResolvedValue(null);
+ saveCached.mockResolvedValue(null);
+ });
+
+ it("uses versioned page-capable variants", () => {
+ expect(cacheKeyShape("auto", undefined, true)).toMatchObject({
+ cacheable: true,
+ ownVariant: "page-markdown-v1",
+ lookupVariants: ["page-markdown-v1", "ocr-page-markdown-v1"],
+ });
+ expect(cacheKeyShape("ocr", undefined, true)).toMatchObject({
+ cacheable: true,
+ ownVariant: "ocr-page-markdown-v1",
+ lookupVariants: ["ocr-page-markdown-v1"],
+ });
+ expect(cacheKeyShape("fast", undefined, true).cacheable).toBe(false);
+ expect(cacheKeyShape("auto", 5, true).cacheable).toBe(false);
+ });
+
+ it("never serves a document-only cache entry to a page-aware request", async () => {
+ getCached.mockResolvedValueOnce({
+ markdown: "legacy",
+ html: "<p>legacy</p>",
+ });
+
+ const result = await tryGetCached(
+ makeMeta(),
+ "BASE64",
+ "ocr",
+ undefined,
+ 2,
+ true,
+ );
+
+ expect(result).toBeNull();
+ expect(getCached).toHaveBeenCalledOnce();
+ expect(getCached).toHaveBeenCalledWith(
+ "BASE64",
+ "firepdf",
+ "ocr-page-markdown-v1",
+ );
+ });
+
+ it("rejects malformed page payloads in page-capable variants", async () => {
+ getCached
+ .mockResolvedValueOnce({
+ markdown: "wrong type",
+ html: "<p>wrong type</p>",
+ pageMarkdown: "not-an-array" as never,
+ })
+ .mockResolvedValueOnce({
+ markdown: "wrong item",
+ html: "<p>wrong item</p>",
+ pageMarkdown: [{ page: 0, markdown: 42 }] as never,
+ });
+
+ const result = await tryGetCached(
+ makeMeta(),
+ "BASE64",
+ "auto",
+ undefined,
+ 2,
+ true,
+ );
+
+ expect(result).toBeNull();
+ expect(getCached).toHaveBeenCalledTimes(2);
+ });
+
+ it("rejects page-capable variants missing required document fields", async () => {
+ getCached
+ .mockResolvedValueOnce({
+ html: "<p>missing markdown</p>",
+ pageMarkdown: [{ page: 1, markdown: "one" }],
+ } as never)
+ .mockResolvedValueOnce({
+ markdown: "missing html",
+ pageMarkdown: [{ page: 1, markdown: "one" }],
+ } as never);
+
+ const result = await tryGetCached(
+ makeMeta(),
+ "BASE64",
+ "auto",
+ undefined,
+ 1,
+ true,
+ );
+
+ expect(result).toBeNull();
+ expect(getCached).toHaveBeenCalledTimes(2);
+ });
+
+ it("accepts an explicitly empty cached markdown string", async () => {
+ getCached.mockResolvedValueOnce({
+ markdown: "",
+ html: "",
+ pageMarkdown: [{ page: 1, markdown: "" }],
+ });
+
+ const result = await tryGetCached(
+ makeMeta(),
+ "BASE64",
+ "auto",
+ undefined,
+ 1,
+ true,
+ );
+
+ expect(result).toMatchObject({ markdown: "", html: "" });
+ });
+
+ it("rejects invalid cached page counts", async () => {
+ getCached
+ .mockResolvedValueOnce({
+ markdown: "wrong type",
+ html: "<p>wrong type</p>",
+ pagesProcessed: "1",
+ pageMarkdown: [{ page: 1, markdown: "one" }],
+ } as never)
+ .mockResolvedValueOnce({
+ markdown: "negative",
+ html: "<p>negative</p>",
+ pagesProcessed: -1,
+ pageMarkdown: [{ page: 1, markdown: "one" }],
+ });
+
+ const result = await tryGetCached(
+ makeMeta(),
+ "BASE64",
+ "auto",
+ undefined,
+ 1,
+ true,
+ );
+
+ expect(result).toBeNull();
+ expect(getCached).toHaveBeenCalledTimes(2);
+ });
+
+ it("serves page-capable entries and preserves the page-count fallback", async () => {
+ getCached.mockResolvedValueOnce({
+ markdown: "whole",
+ html: "<p>whole</p>",
+ pageMarkdown: [
+ { page: 1, markdown: "one" },
+ { page: 2, markdown: "two" },
+ ],
+ });
+
+ const result = await tryGetCached(
+ makeMeta(),
+ "BASE64",
+ "auto",
+ undefined,
+ 2,
+ true,
+ );
+
+ expect(result?.pageMarkdown).toHaveLength(2);
+ expect(result?.pagesProcessed).toBe(2);
+ });
+
+ it("strips page payloads when a legacy request reuses an enriched sidecar", async () => {
+ getCached.mockResolvedValueOnce(null).mockResolvedValueOnce({
+ markdown: "whole",
+ html: "<p>whole</p>",
+ pageMarkdown: [
+ { page: 1, markdown: "one" },
+ { page: 2, markdown: "two" },
+ ],
+ });
+
+ const result = await tryGetCached(
+ makeMeta(),
+ "BASE64",
+ "auto",
+ undefined,
+ 2,
+ false,
+ );
+
+ expect(result).toMatchObject({
+ markdown: "whole",
+ html: "<p>whole</p>",
+ pagesProcessed: 2,
+ });
+ expect(result?.pageMarkdown).toBeUndefined();
+ expect(getCached).toHaveBeenNthCalledWith(
+ 2,
+ "BASE64",
+ "firepdf",
+ "page-markdown-v1",
+ );
+ });
+
+ it("writes an enriched sidecar plus a compact legacy entry", async () => {
+ const pageMarkdown = [
+ { page: 1, markdown: "one" },
+ { page: 2, markdown: "two" },
+ ];
+
+ await maybeSaveResult({
+ meta: makeMeta(),
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: true,
+ result: {
+ markdown: "whole",
+ html: "<p>whole</p>",
+ pagesProcessed: 2,
+ pageMarkdown,
+ },
+ });
+
+ expect(saveCached).toHaveBeenCalledTimes(2);
+ expect(saveCached).toHaveBeenNthCalledWith(
+ 1,
+ "BASE64",
+ expect.objectContaining({ pageMarkdown }),
+ "firepdf",
+ "page-markdown-v1",
+ );
+ expect(saveCached).toHaveBeenNthCalledWith(
+ 2,
+ "BASE64",
+ expect.not.objectContaining({ pageMarkdown: expect.anything() }),
+ "firepdf",
+ undefined,
+ );
+ });
+
+ it("does not rewrite an existing valid compact legacy entry", async () => {
+ getCached.mockResolvedValueOnce({
+ markdown: "existing",
+ html: "<p>existing</p>",
+ pagesProcessed: 2,
+ });
+
+ await maybeSaveResult({
+ meta: makeMeta(),
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: true,
+ result: {
+ markdown: "whole",
+ html: "<p>whole</p>",
+ pagesProcessed: 2,
+ pageMarkdown: [
+ { page: 1, markdown: "one" },
+ { page: 2, markdown: "two" },
+ ],
+ },
+ });
+
+ expect(getCached).toHaveBeenCalledWith("BASE64", "firepdf", undefined);
+ expect(saveCached).toHaveBeenCalledOnce();
+ expect(saveCached).toHaveBeenCalledWith(
+ "BASE64",
+ expect.objectContaining({ pageMarkdown: expect.any(Array) }),
+ "firepdf",
+ "page-markdown-v1",
+ );
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts
new file mode 100644
index 000000000..48bd86b6e
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts
@@ -0,0 +1,70 @@
+vi.mock("../../../../../services", async importOriginal => ({
+ ...(await importOriginal<typeof import("../../../../../services")>()),
+ useIndex: true,
+}));
+
+import { config } from "../../../../../config";
+import { shouldUseIndex } from "../../index";
+import { sendDocumentToIndex } from "../../index/index";
+
+describe("PDF page-markdown URL index policy", () => {
+ it("bypasses URL-index reads for page-aware requests", () => {
+ const originalWriteOnly = config.FIRECRAWL_INDEX_WRITE_ONLY;
+ (
+ config as { FIRECRAWL_INDEX_WRITE_ONLY?: boolean }
+ ).FIRECRAWL_INDEX_WRITE_ONLY = false;
+ const baseMeta = {
+ internalOptions: { isParse: false },
+ options: {
+ formats: ["markdown"],
+ parsers: [{ type: "pdf" }],
+ maxAge: 60_000,
+ },
+ } as any;
+
+ try {
+ expect(shouldUseIndex(baseMeta)).toBe(true);
+ expect(
+ shouldUseIndex({
+ ...baseMeta,
+ options: {
+ ...baseMeta.options,
+ parsers: [{ type: "pdf", pageMarkdown: true }],
+ },
+ }),
+ ).toBe(false);
+ } finally {
+ (
+ config as { FIRECRAWL_INDEX_WRITE_ONLY?: boolean }
+ ).FIRECRAWL_INDEX_WRITE_ONLY = originalWriteOnly;
+ }
+ });
+
+ it("does not write page-aware results to the document-only URL index", async () => {
+ const document = {
+ markdown: "whole document",
+ pages: [{ pageNumber: 1, markdown: "page one" }],
+ rawHtml: "<p>whole document</p>",
+ metadata: {
+ sourceURL: "https://example.com/file.pdf",
+ },
+ } as any;
+ const meta = {
+ url: "https://example.com/file.pdf",
+ winnerEngine: "pdf",
+ options: {
+ storeInCache: true,
+ parsers: [{ type: "pdf", pageMarkdown: true }],
+ },
+ internalOptions: {
+ isParse: false,
+ zeroDataRetention: false,
+ },
+ } as any;
+
+ const result = await sendDocumentToIndex(meta, document);
+
+ expect(result).toBe(document);
+ expect(result.metadata.indexId).toBeUndefined();
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
index 42cf54ffa..3df380609 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
@@ -37,6 +37,7 @@ export async function scrapePDFWithFirePDFAsync(
pagesProcessed?: number,
mode?: PDFMode,
deps: FirePdfAsyncDeps = {},
+ includePageMarkdown = false,
): Promise<PDFProcessorResult> {
const fetchImpl = deps.fetchImpl ?? undiciFetch;
const fallbackImpl = deps.fallbackImpl ?? scrapePDFWithFirePDF;
@@ -47,7 +48,14 @@ export async function scrapePDFWithFirePDFAsync(
// Async persists inputs and queue state, so ZDR is excluded until that
// lifecycle has an explicit delete-on-completion contract.
if (meta.internalOptions.zeroDataRetention) {
- return fallbackImpl(meta, base64Content, maxPages, pagesProcessed, mode);
+ return fallbackImpl(
+ meta,
+ base64Content,
+ maxPages,
+ pagesProcessed,
+ mode,
+ includePageMarkdown,
+ );
}
const cached = await tryGetCached(
@@ -56,6 +64,7 @@ export async function scrapePDFWithFirePDFAsync(
mode,
maxPages,
pagesProcessed,
+ includePageMarkdown,
);
if (cached) return cached;
@@ -73,7 +82,14 @@ export async function scrapePDFWithFirePDFAsync(
if (!baseUrl) {
// Should be unreachable β€” call site checks this β€” but fall back rather
// than crash if a route somehow bypasses the gate.
- return fallbackImpl(meta, base64Content, maxPages, pagesProcessed, mode);
+ return fallbackImpl(
+ meta,
+ base64Content,
+ maxPages,
+ pagesProcessed,
+ mode,
+ includePageMarkdown,
+ );
}
const overallStartedAt = now();
@@ -105,6 +121,7 @@ export async function scrapePDFWithFirePDFAsync(
maxPages,
pagesProcessed,
mode,
+ includePageMarkdown,
deadlineAt,
teamConcurrency,
fetchImpl,
@@ -160,6 +177,11 @@ export async function scrapePDFWithFirePDFAsync(
// ── Assemble + cache save ─────────────────────────────────────────────
const pages =
fetched.pages_processed ?? polled.poll.pages_processed ?? pagesProcessed;
+ if (includePageMarkdown && fetched.pages === undefined) {
+ failAsync(meta, "http_5xx", {
+ note: "FirePDF result omitted requested physical page markdown",
+ });
+ }
const durationMs = now() - overallStartedAt;
firePdfAsyncTotalDurationSeconds.observe(durationMs / 1000);
@@ -168,6 +190,7 @@ export async function scrapePDFWithFirePDFAsync(
durationMs,
markdownLength: fetched.markdown.length,
pagesProcessed: pages,
+ pageMarkdownPages: fetched.pages?.length,
failedPages: fetched.failed_pages,
partialPages: fetched.partial_pages,
pollCount: polled.pollCount,
@@ -177,6 +200,7 @@ export async function scrapePDFWithFirePDFAsync(
markdown: fetched.markdown,
html: await safeMarkdownToHtml(fetched.markdown, meta.logger, meta.id),
pagesProcessed: pages,
+ ...(fetched.pages ? { pageMarkdown: fetched.pages } : {}),
};
await maybeSaveResult({
@@ -184,6 +208,7 @@ export async function scrapePDFWithFirePDFAsync(
base64Content,
mode,
maxPages,
+ includePageMarkdown,
result: processorResult,
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
index c53dd93ab..494d044c1 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
@@ -10,15 +10,67 @@ import {
// entries. `fast` mode bypasses entirely (hard cost ceiling β€” must fail on
// scanned PDFs, not serve a cached OCR result), as does any call with
// `maxPages` (the cached entry may have been written with a different cap).
-function cacheKeyShape(
+const PAGE_MARKDOWN_VARIANT = "page-markdown-v1";
+const OCR_PAGE_MARKDOWN_VARIANT = "ocr-page-markdown-v1";
+
+function isValidCachedDocument(
+ value: unknown,
+): value is Pick<PDFProcessorResult, "html"> & { markdown: string } {
+ if (typeof value !== "object" || value === null) return false;
+ const cached = value as {
+ markdown?: unknown;
+ html?: unknown;
+ pagesProcessed?: unknown;
+ };
+ return (
+ typeof cached.markdown === "string" &&
+ typeof cached.html === "string" &&
+ (cached.pagesProcessed === undefined ||
+ (typeof cached.pagesProcessed === "number" &&
+ Number.isInteger(cached.pagesProcessed) &&
+ cached.pagesProcessed >= 0))
+ );
+}
+
+function isValidPageMarkdown(
+ value: unknown,
+): value is NonNullable<PDFProcessorResult["pageMarkdown"]> {
+ return (
+ Array.isArray(value) &&
+ value.every(
+ page =>
+ typeof page === "object" &&
+ page !== null &&
+ Number.isInteger((page as { page?: unknown }).page) &&
+ Number((page as { page: number }).page) > 0 &&
+ typeof (page as { markdown?: unknown }).markdown === "string",
+ )
+ );
+}
+
+export function cacheKeyShape(
mode: PDFMode | undefined,
maxPages: number | undefined,
+ includePageMarkdown: boolean,
) {
const cacheable = mode !== "fast" && !maxPages;
- const ownVariant: string | undefined = mode === "ocr" ? "ocr" : undefined;
- const lookupVariants: (string | undefined)[] =
- mode === "ocr" ? ["ocr"] : [undefined, "ocr"];
- return { cacheable, ownVariant, lookupVariants };
+ const baseVariant: string | undefined = mode === "ocr" ? "ocr" : undefined;
+ const ownVariant: string | undefined = includePageMarkdown
+ ? mode === "ocr"
+ ? OCR_PAGE_MARKDOWN_VARIANT
+ : PAGE_MARKDOWN_VARIANT
+ : baseVariant;
+
+ // Page-aware requests may only consume page-capable artifacts. Legacy
+ // requests prefer their compact entry but can reuse an enriched sidecar.
+ const lookupVariants: (string | undefined)[] = includePageMarkdown
+ ? mode === "ocr"
+ ? [OCR_PAGE_MARKDOWN_VARIANT]
+ : [PAGE_MARKDOWN_VARIANT, OCR_PAGE_MARKDOWN_VARIANT]
+ : mode === "ocr"
+ ? ["ocr", OCR_PAGE_MARKDOWN_VARIANT]
+ : [undefined, PAGE_MARKDOWN_VARIANT, "ocr", OCR_PAGE_MARKDOWN_VARIANT];
+ return { cacheable, ownVariant, baseVariant, lookupVariants };
}
export async function tryGetCached(
@@ -27,9 +79,14 @@ export async function tryGetCached(
mode: PDFMode | undefined,
maxPages: number | undefined,
pagesProcessed: number | undefined,
+ includePageMarkdown: boolean,
): Promise<PDFProcessorResult | null> {
if (meta.internalOptions.zeroDataRetention) return null;
- const { cacheable, lookupVariants } = cacheKeyShape(mode, maxPages);
+ const { cacheable, lookupVariants } = cacheKeyShape(
+ mode,
+ maxPages,
+ includePageMarkdown,
+ );
if (!cacheable) return null;
for (const variant of lookupVariants) {
@@ -40,21 +97,36 @@ export async function tryGetCached(
variant,
);
if (cached) {
- meta.logger.info("Using cached FirePDF result (async path)", {
+ if (
+ !isValidCachedDocument(cached) ||
+ (includePageMarkdown && !isValidPageMarkdown(cached.pageMarkdown))
+ ) {
+ // Defense in depth: variant names are the capability boundary, but
+ // never let a malformed/old artifact satisfy a cache lookup.
+ continue;
+ }
+ meta.logger.info("Using cached FirePDF result", {
scrapeId: meta.id,
requestedMode: mode,
cacheVariant: variant ?? "base",
});
+ if (!includePageMarkdown) {
+ const { pageMarkdown: _pageMarkdown, ...compactCached } = cached;
+ return {
+ ...compactCached,
+ pagesProcessed: cached.pagesProcessed ?? pagesProcessed,
+ };
+ }
return {
...cached,
pagesProcessed: cached.pagesProcessed ?? pagesProcessed,
};
}
} catch (error) {
- meta.logger.warn(
- "Error checking FirePDF cache (async path), proceeding",
- { error, cacheVariant: variant ?? "base" },
- );
+ meta.logger.warn("Error checking FirePDF cache, proceeding", {
+ error,
+ cacheVariant: variant ?? "base",
+ });
}
}
return null;
@@ -65,19 +137,45 @@ export async function maybeSaveResult(args: {
base64Content: string;
mode: PDFMode | undefined;
maxPages: number | undefined;
+ includePageMarkdown: boolean;
result: PDFProcessorResult & { markdown: string };
}): Promise<void> {
- const { meta, base64Content, mode, maxPages, result } = args;
+ const { meta, base64Content, mode, maxPages, includePageMarkdown, result } =
+ args;
if (meta.internalOptions.zeroDataRetention) return;
- const { cacheable, ownVariant } = cacheKeyShape(mode, maxPages);
+ const { cacheable, ownVariant, baseVariant } = cacheKeyShape(
+ mode,
+ maxPages,
+ includePageMarkdown,
+ );
if (!cacheable) return;
try {
await savePdfResultToCache(base64Content, result, "firepdf", ownVariant);
+ // A page-capable parse is also a valid legacy result. Populate the compact
+ // base key when it is missing so a later legacy request never repeats the
+ // conversion. A page-sidecar miss can coexist with a warm legacy key during
+ // rollout, so avoid rewriting that object. Strip the page payload to keep
+ // the hot-path cache object small.
+ if (includePageMarkdown && ownVariant !== baseVariant) {
+ const existingBase = await getPdfResultFromCache(
+ base64Content,
+ "firepdf",
+ baseVariant,
+ );
+ if (!existingBase || !isValidCachedDocument(existingBase)) {
+ const { pageMarkdown: _pageMarkdown, ...baseResult } = result;
+ await savePdfResultToCache(
+ base64Content,
+ baseResult,
+ "firepdf",
+ baseVariant,
+ );
+ }
+ }
} catch (error) {
- meta.logger.warn(
- "Error saving FirePDF async result to cache (continuing)",
- { error },
- );
+ meta.logger.warn("Error saving FirePDF result to cache (continuing)", {
+ error,
+ });
}
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
index d99b8f1fe..7f2f66170 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
@@ -48,8 +48,15 @@ export const pollResponseSchema = z.object({
});
export const resultResponseSchema = z.object({
- schema_version: z.literal(1).optional(),
+ schema_version: z
+ .union([z.literal(1), z.literal(2), z.literal(3)])
+ .optional(),
markdown: z.string(),
+ pages: z
+ .array(
+ z.object({ page: z.number().int().positive(), markdown: z.string() }),
+ )
+ .optional(),
pages_processed: z.number().optional(),
failed_pages: z.array(z.number()).nullable().optional(),
partial_pages: z.array(z.number()).nullable().optional(),
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
index 9d58ec993..e0dfa8d2c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
@@ -19,6 +19,7 @@ type SubmitArgs = {
maxPages: number | undefined;
pagesProcessed: number | undefined;
mode: PDFMode | undefined;
+ includePageMarkdown: boolean;
deadlineAt: string;
/** Team's sold concurrency from the ACUC (ENG-5049 account context).
* Optional: entitlement lookup must never block or fail a scrape. */
@@ -58,6 +59,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
maxPages,
pagesProcessed,
mode,
+ includePageMarkdown,
deadlineAt,
teamConcurrency,
fetchImpl,
@@ -86,6 +88,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
...(pagesProcessed !== undefined && { pages_estimate: pagesProcessed }),
...(maxPages !== undefined && { max_pages: maxPages }),
...(mode !== undefined && { mode }),
+ ...(includePageMarkdown && { include_page_markdown: true }),
},
};
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
index fa8dec3b2..f41998235 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
@@ -5,11 +5,8 @@ import { z } from "zod";
import type { PDFProcessorResult } from "./types";
import type { PDFMode } from "../../../../controllers/v2/types";
import { safeMarkdownToHtml } from "./markdownToHtml";
-import {
- createPdfCacheKey,
- getPdfResultFromCache,
- savePdfResultToCache,
-} from "../../../../lib/gcs-pdf-cache";
+import { createPdfCacheKey } from "../../../../lib/gcs-pdf-cache";
+import { maybeSaveResult, tryGetCached } from "./fire-pdf/cache";
/**
* Reconcile an existing page count with what fire-pdf reported.
@@ -47,6 +44,7 @@ export async function scrapePDFWithFirePDF(
maxPages?: number,
pagesProcessed?: number,
mode?: PDFMode,
+ includePageMarkdown = false,
): Promise<PDFProcessorResult> {
const logger = meta.logger;
@@ -64,40 +62,17 @@ export async function scrapePDFWithFirePDF(
// scanned PDFs, not serve a cached OCR result).
const cacheable =
mode !== "fast" && !maxPages && !meta.internalOptions.zeroDataRetention;
- const ownVariant: string | undefined = mode === "ocr" ? "ocr" : undefined;
- const lookupVariants: (string | undefined)[] =
- mode === "ocr" ? ["ocr"] : [undefined, "ocr"];
-
- if (cacheable) {
- for (const variant of lookupVariants) {
- try {
- const cached = await getPdfResultFromCache(
- base64Content,
- "firepdf",
- variant,
- );
- if (cached) {
- logger.info("Using cached FirePDF result", {
- scrapeId: meta.id,
- requestedMode: mode,
- cacheVariant: variant ?? "base",
- });
- // Cache entries written before pagesProcessed existed don't carry
- // the field. Fall back to the caller's pagesProcessed argument so
- // billing on a stale hit doesn't silently regress to 0.
- return {
- ...cached,
- pagesProcessed: cached.pagesProcessed ?? pagesProcessed,
- };
- }
- } catch (error) {
- logger.warn("Error checking FirePDF cache, proceeding", {
- error,
- cacheVariant: variant ?? "base",
- });
- }
- }
- }
+ const cached = cacheable
+ ? await tryGetCached(
+ meta,
+ base64Content,
+ mode,
+ maxPages,
+ pagesProcessed,
+ includePageMarkdown,
+ )
+ : null;
+ if (cached) return cached;
meta.abort.throwIfAborted();
@@ -144,6 +119,7 @@ export async function scrapePDFWithFirePDF(
scrape_id: meta.id,
...(maxPages !== undefined && { max_pages: maxPages }),
...(mode !== undefined && { mode }),
+ ...(includePageMarkdown && { include_page_markdown: true }),
// Enrichment for the fire-pdf jobs DB / dashboard. fire-pdf treats
// these as optional β€” older fire-pdf builds will ignore unknown fields.
team_id: meta.internalOptions.teamId,
@@ -161,12 +137,22 @@ export async function scrapePDFWithFirePDF(
markdown: z.string(),
failed_pages: z.array(z.number()).nullable(),
pages_processed: z.number().optional(),
+ pages: z
+ .array(
+ z.object({ page: z.number().int().positive(), markdown: z.string() }),
+ )
+ .optional(),
}),
mock: meta.mock,
abort: meta.abort.asSignal(),
});
const durationMs = Date.now() - startedAt;
+ if (includePageMarkdown && resp.pages === undefined) {
+ throw new Error(
+ "FirePDF response did not include requested physical page markdown",
+ );
+ }
const pages = resp.pages_processed ?? pagesProcessed;
logger.info("FirePDF completed", {
@@ -183,19 +169,18 @@ export async function scrapePDFWithFirePDF(
markdown: resp.markdown,
html: await safeMarkdownToHtml(resp.markdown, logger, meta.id),
pagesProcessed: pages,
+ ...(resp.pages ? { pageMarkdown: resp.pages } : {}),
};
if (cacheable) {
- try {
- await savePdfResultToCache(
- base64Content,
- processorResult,
- "firepdf",
- ownVariant,
- );
- } catch (error) {
- logger.warn("Error saving FirePDF result to cache", { error });
- }
+ await maybeSaveResult({
+ meta,
+ base64Content,
+ mode,
+ maxPages,
+ includePageMarkdown,
+ result: processorResult,
+ });
}
return processorResult;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 4ebf2bbfe..3ce5be08e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -18,6 +18,7 @@ import {
shouldParsePDF,
getPDFMaxPages,
getPDFMode,
+ getPDFPageMarkdown,
getFirePdfAsync,
} from "../../../../controllers/v2/types";
import type { PDFMode } from "../../../../controllers/v2/types";
@@ -59,6 +60,13 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
const shouldParse = shouldParsePDF(meta.options.parsers);
const maxPages = getPDFMaxPages(meta.options.parsers);
const mode: PDFMode = getPDFMode(meta.options.parsers);
+ const includePageMarkdown = getPDFPageMarkdown(meta.options.parsers);
+
+ if (includePageMarkdown && !config.FIRE_PDF_BASE_URL) {
+ throw new Error(
+ "Physical page markdown is unavailable because FirePDF is not configured",
+ );
+ }
if (!shouldParse) {
if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {
@@ -170,7 +178,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
let shadowPagesNeedingOcr: number[] | undefined;
const forceFirePDF =
- !!meta.options.__forceFirePDF && !!config.FIRE_PDF_BASE_URL;
+ (!!meta.options.__forceFirePDF || includePageMarkdown) &&
+ !!config.FIRE_PDF_BASE_URL;
const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;
const logger = meta.logger.child({ method: "scrapePDF/processPdf" });
@@ -374,7 +383,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
// OCR / MU fallback.
// Skipped only when Rust extraction is enabled AND mode is "fast",
// unless we explicitly routed to MinerU via MINERU_PERCENT.
- const skipOCR = rustEnabled && mode === "fast" && !routeToMinerU;
+ const skipOCR =
+ rustEnabled && mode === "fast" && !routeToMinerU && !forceFirePDF;
if (!result && !skipOCR) {
const pdfBuffer = await readFile(tempFilePath);
const fileSizeBytes = pdfBuffer.length;
@@ -435,22 +445,64 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
});
}
try {
- result = await (
- useAsync ? scrapePDFWithFirePDFAsync : scrapePDFWithFirePDF
- )(
- {
- ...meta,
- logger: meta.logger.child({
- method: useAsync
- ? "scrapePDF/firePDFAsync"
- : "scrapePDF/firePDF",
- }),
- },
- base64Content,
- maxPages,
- effectivePageCount,
- mode,
- );
+ const firePdfMeta = {
+ ...meta,
+ logger: meta.logger.child({
+ method: useAsync ? "scrapePDF/firePDFAsync" : "scrapePDF/firePDF",
+ }),
+ };
+ if (useAsync) {
+ try {
+ result = await scrapePDFWithFirePDFAsync(
+ firePdfMeta,
+ base64Content,
+ maxPages,
+ effectivePageCount,
+ mode,
+ undefined,
+ includePageMarkdown,
+ );
+ } catch (error) {
+ if (
+ !includePageMarkdown ||
+ error instanceof RemoveFeatureError ||
+ error instanceof AbortManagerThrownError
+ ) {
+ throw error;
+ }
+ meta.logger.warn(
+ "FirePDF async page markdown failed -- retrying synchronously",
+ {
+ method: "scrapePDF/firePDFFallback",
+ error,
+ scrape_id: meta.id,
+ team_id: meta.internalOptions.teamId,
+ },
+ );
+ result = await scrapePDFWithFirePDF(
+ {
+ ...firePdfMeta,
+ logger: meta.logger.child({
+ method: "scrapePDF/firePDFSyncFallback",
+ }),
+ },
+ base64Content,
+ maxPages,
+ effectivePageCount,
+ mode,
+ true,
+ );
+ }
+ } else {
+ result = await scrapePDFWithFirePDF(
+ firePdfMeta,
+ base64Content,
+ maxPages,
+ effectivePageCount,
+ mode,
+ includePageMarkdown,
+ );
+ }
effectivePageCount = reconcilePageCountWithFirePdf(
effectivePageCount,
result,
@@ -630,6 +682,14 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
statusCode: response.status,
html: result?.html ?? "",
markdown: result?.markdown ?? "",
+ ...(includePageMarkdown && result?.pageMarkdown
+ ? {
+ pages: result.pageMarkdown.map(page => ({
+ pageNumber: page.page,
+ markdown: page.markdown,
+ })),
+ }
+ : {}),
pdfMetadata: {
numPages: effectivePageCount,
totalPages: totalPageCount,
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts
index b04067b4c..dc5f739f0 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts
@@ -1,6 +1,13 @@
+export type PdfPageMarkdown = {
+ /** 1-based physical PDF page number returned by fire-pdf. */
+ page: number;
+ markdown: string;
+};
+
export type PDFProcessorResult = {
html: string;
markdown?: string;
+ pageMarkdown?: PdfPageMarkdown[];
/**
* Pages the underlying engine actually processed for this request.
* Currently populated only by fire-pdf (via OcrSuccessBody.pages_processed).
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 8d1a154e8..f959fc0fc 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -977,6 +977,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
let document: Document = {
markdown: engineResult.markdown,
+ pages: engineResult.pages,
rawHtml: engineResult.html,
json: engineResult.json,
screenshot: engineResult.screenshot,