Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): expose PDF page markdown (#4219)
Abimael Martellfd240843e19d
diff --git a/apps/api/src/__tests__/snips/v2/parse.test.ts b/apps/api/src/__tests__/snips/v2/parse.test.tsindex ba2586e41..ee8a3f07d 100644--- a/apps/api/src/__tests__/snips/v2/parse.test.ts+++ b/apps/api/src/__tests__/snips/v2/parse.test.ts@@ -606,6 +606,45 @@ describe("/v2/parse", () => {);});+ describeIf(ALLOW_TEST_SUITE_WEBSITE && !!config.FIRE_PDF_BASE_URL)(+ "PDF upload physical page markdown",+ () => {+ it(+ "returns physical pages for an uploaded PDF",+ async () => {+ expect(pdfFixture).not.toBeNull();++ const result = await parse(+ {+ options: {+ formats: ["markdown"],+ parsers: [{ type: "pdf", mode: "auto", pageMarkdown: true }],+ },+ file: {+ content: pdfFixture!,+ filename: "upload.pdf",+ contentType: "application/pdf",+ },+ },+ identity,+ );++ expect(result.markdown).toContain("PDF Test File");+ expect(result.pages).toHaveLength(result.metadata.numPages!);+ result.pages?.forEach((page, index) => {+ expect(page).toEqual(+ expect.objectContaining({+ pageNumber: index + 1,+ markdown: expect.any(String),+ }),+ );+ });+ },+ scrapeTimeout * 3,+ );+ },+ );+it("returns a validation error when file is missing",async () => {diff --git a/apps/api/src/__tests__/snips/v2/parsers-pdf-page-markdown.test.ts b/apps/api/src/__tests__/snips/v2/parsers-pdf-page-markdown.test.tsnew file mode 100644index 000000000..57b243457--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/parsers-pdf-page-markdown.test.ts@@ -0,0 +1,47 @@+import {+ ALLOW_TEST_SUITE_WEBSITE,+ describeIf,+ TEST_SUITE_WEBSITE,+} from "../lib";+import { config } from "../../../config";+import { idmux, Identity, scrape, scrapeTimeout } from "./lib";++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "parsers-pdf-page-markdown",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000 + scrapeTimeout);++const SHOULD_RUN = ALLOW_TEST_SUITE_WEBSITE && !!config.FIRE_PDF_BASE_URL;++describeIf(SHOULD_RUN)("PDF physical page markdown", () => {+ it(+ "returns additive pages without changing document markdown",+ async () => {+ const response = await scrape(+ {+ url: `${TEST_SUITE_WEBSITE}/example.pdf`,+ formats: ["markdown"],+ parsers: [{ type: "pdf", mode: "auto", pageMarkdown: true }],+ },+ identity,+ );++ expect(response.markdown).toContain("PDF Test File");+ expect(response.pages).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ pageNumber: 1,+ markdown: expect.any(String),+ }),+ ]),+ );+ expect(response.pages).toHaveLength(response.metadata.numPages!);+ },+ scrapeTimeout * 3,+ );+});diff --git a/apps/api/src/__tests__/snips/v2/parsers.test.ts b/apps/api/src/__tests__/snips/v2/parsers.test.tsindex bd7b20681..dc8392353 100644--- a/apps/api/src/__tests__/snips/v2/parsers.test.ts+++ b/apps/api/src/__tests__/snips/v2/parsers.test.ts@@ -387,6 +387,24 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("Parsers parameter tests", () => {scrapeTimeout,);+ it.concurrent(+ "rejects non-boolean pageMarkdown",+ async () => {+ const raw = await scrapeRaw(+ {+ url: pdfUrl,+ parsers: [{ type: "pdf", pageMarkdown: "yes" } as any],+ },+ identity,+ );++ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ expect(raw.body.error).toBe("Bad Request");+ },+ scrapeTimeout,+ );+it.concurrent("rejects colon-separated shorthand strings",async () => {diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex fcdda5a63..1f849e983 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -435,6 +435,24 @@ describe("V2 Types Validation", () => {expect((result.parsers as any)[0].maxPages).toBe(100);});+ it("should accept physical page markdown for PDF parsers", () => {+ const result = scrapeRequestSchema.parse({+ url: "https://example.com/file.pdf",+ parsers: [{ type: "pdf", mode: "auto", pageMarkdown: true }],+ });++ expect((result.parsers as any)[0].pageMarkdown).toBe(true);+ });++ it("should reject non-boolean physical page markdown", () => {+ expect(() =>+ scrapeRequestSchema.parse({+ url: "https://example.com/file.pdf",+ parsers: [{ type: "pdf", pageMarkdown: "yes" }],+ }),+ ).toThrow();+ });+it("should reject PDF parser with maxPages exceeding limit", () => {const input: ScrapeRequestInput = {url: "https://example.com",diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex b27eccb10..460d19150 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1005,6 +1005,8 @@ export type Document = {description?: string;url?: string;markdown?: string;+ /** Physical PDF pages, populated by the v2 pageMarkdown parser option. */+ pages?: Array<{ pageNumber: number; markdown: string }>;html?: string;rawHtml?: string;links?: string[];diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex d6b2075fd..4a1dc62f5 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -482,6 +482,8 @@ const pdfParserWithOptions = z.strictObject({type: z.literal("pdf"),mode: pdfModeSchema.optional(),maxPages: z.int().positive().finite().max(10000).optional(),+ /** Include physical per-page markdown alongside document markdown. */+ pageMarkdown: z.boolean().optional(),// Experimental: route this request through the fire-pdf async pipeline// (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back// to sync on any async-path failure, so user-visible behavior is unchanged@@ -531,6 +533,16 @@ export function getPDFMode(parsers?: Parsers): PDFMode {return "auto";}+export function getPDFPageMarkdown(parsers?: Parsers): boolean {+ if (!parsers) return false;+ for (const parser of parsers) {+ if (typeof parser === "object" && parser.type === "pdf") {+ return parser.pageMarkdown === true;+ }+ }+ return false;+}+export function getFirePdfAsync(parsers?: Parsers): boolean {if (!parsers) return false;for (const parser of parsers) {@@ -1226,6 +1238,8 @@ export type Document = {description?: string;url?: string;markdown?: string;+ /** Physical PDF pages, present only for `parsers[].pageMarkdown`. */+ pages?: Array<{ pageNumber: number; markdown: string }>;html?: string;rawHtml?: string;links?: string[];diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.tsindex fd6fbbeb4..4f89de8c5 100644--- a/apps/api/src/lib/gcs-pdf-cache.ts+++ b/apps/api/src/lib/gcs-pdf-cache.ts@@ -13,6 +13,8 @@ type CachedPdfResult = {markdown: string;html: string;pagesProcessed?: number;+ /** Physical page markdown; present only in page-capable cache variants. */+ pageMarkdown?: Array<{ page: number; markdown: string }>;};const PROVIDER_PREFIXES: Record<PdfCacheProvider, string> = {diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 7160bfc4d..5d1563a59 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -27,7 +27,10 @@ import {} from "./x-twitter";import { queryEngpickerVerdict, useIndex } from "../../../services";import { hasFormatOfType } from "../../../lib/format-utils";-import { getPDFMaxPages } from "../../../controllers/v2/types";+import {+ getPDFMaxPages,+ getPDFPageMarkdown,+} from "../../../controllers/v2/types";import type { PdfMetadata } from "./pdf/types";import { BrandingProfile } from "../../../types/branding";import { BrandingNotSupportedError } from "../error";@@ -138,6 +141,7 @@ export type EngineScrapeResult = {html: string;markdown?: string;+ pages?: Array<{ pageNumber: number; markdown: string }>;json?: unknown;statusCode: number;error?: string;@@ -563,6 +567,8 @@ export function shouldUseIndex(meta: Meta) {!hasFormatOfType(meta.options.formats, "branding") &&// Skip index if a non-default PDF maxPages is specifiedgetPDFMaxPages(meta.options.parsers) === undefined &&+ // The URL index does not yet persist physical-page capability metadata.+ !getPDFPageMarkdown(meta.options.parsers) &&!hasCustomScreenshotSettings &&meta.options.maxAge !== 0 &&(meta.options.headers === undefined ||diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 3863f972e..f282f9f33 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -36,7 +36,10 @@ import {IndexMissError,NoCachedDataError,} from "../../error";-import { shouldParsePDF } from "../../../../controllers/v2/types";+import {+ getPDFPageMarkdown,+ shouldParsePDF,+} from "../../../../controllers/v2/types";import { hasFormatOfType } from "../../../../lib/format-utils";export async function sendDocumentToIndex(meta: Meta, document: Document) {@@ -56,6 +59,10 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {// every access must go through the Exchange and its ledger.meta.winnerEngine !== "exchange" &&!(meta.winnerEngine === "pdf" && !shouldParsePDF(meta.options.parsers)) &&+ // Page-aware results are capability-specific and are not represented in+ // the URL index schema yet. Do not write an entry that could later be+ // served without its required pages payload.+ !getPDFPageMarkdown(meta.options.parsers) &&!meta.options.parsers?.some(parser => {if (typeof parser === "object" &&diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.tsindex f5068ca85..d41be0cfa 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts@@ -1,4 +1,40 @@-import { reconcilePageCountWithFirePdf } from "../firePDF";+import { robustFetch } from "../../../lib/fetch";+import {+ reconcilePageCountWithFirePdf,+ scrapePDFWithFirePDF,+} from "../firePDF";++vi.mock("../../../lib/fetch", () => ({+ robustFetch: vi.fn(),+}));++const mockedRobustFetch = vi.mocked(robustFetch);++function makeMeta() {+ const logger: any = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+ };++ return {+ id: "sync-page-markdown-test",+ url: "https://example.com/file.pdf",+ rewrittenUrl: undefined,+ logger,+ mock: null,+ abort: {+ throwIfAborted: vi.fn(),+ asSignal: vi.fn(() => new AbortController().signal),+ scrapeTimeout: vi.fn(() => 60_000),+ },+ internalOptions: {+ zeroDataRetention: true,+ teamId: "team-test",+ },+ } as any;+}describe("reconcilePageCountWithFirePdf", () => {it("uses fire-pdf's count when the upstream pass left it at 0", () => {@@ -41,3 +77,59 @@ describe("reconcilePageCountWithFirePdf", () => {expect(reconcilePageCountWithFirePdf(5, { pagesProcessed: 0 })).toBe(5);});});++describe("scrapePDFWithFirePDF page markdown", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("requests and returns the physical page payload", async () => {+ mockedRobustFetch.mockResolvedValue({+ markdown: "continued paragraph",+ failed_pages: null,+ pages_processed: 2,+ pages: [+ { page: 1, markdown: "continued" },+ { page: 2, markdown: "paragraph" },+ ],+ } as any);++ const result = await scrapePDFWithFirePDF(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ "auto",+ true,+ );++ expect(mockedRobustFetch).toHaveBeenCalledWith(+ expect.objectContaining({+ body: expect.objectContaining({ include_page_markdown: true }),+ }),+ );+ expect(result.pageMarkdown).toEqual([+ { page: 1, markdown: "continued" },+ { page: 2, markdown: "paragraph" },+ ]);+ });++ it("rejects document-only FirePDF responses for page-aware requests", async () => {+ mockedRobustFetch.mockResolvedValue({+ markdown: "document only",+ failed_pages: null,+ pages_processed: 1,+ } as any);++ await expect(+ scrapePDFWithFirePDF(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ "auto",+ true,+ ),+ ).rejects.toThrow(/did not include requested physical page markdown/);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.tsindex e3efdaaf4..5dbef323e 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts@@ -7,7 +7,6 @@ vi.mock("../../../../../lib/gcs-pdf-cache", () => ({savePdfResultToCache: vi.fn(async () => null),}));-import {FirePdfAsyncFailure,scrapePDFWithFirePDFAsync,@@ -145,7 +144,7 @@ describe("scrapePDFWithFirePDFAsync", () => {internalOptions: {zeroDataRetention: true,teamId: "team-x",- teamConcurrency: 12,+ teamConcurrency: 12,crawlId: undefined,},});@@ -252,7 +251,101 @@ describe("scrapePDFWithFirePDFAsync", () => {expect(fallback).not.toHaveBeenCalled();expect(calls).toHaveLength(4);// Account context rides the submit body (FirePDF ENG-5049).- expect((calls[0].body as { team_concurrency?: number }).team_concurrency).toBe(12);+ expect(+ (calls[0].body as { team_concurrency?: number }).team_concurrency,+ ).toBe(12);+ });++ it("requests and returns physical page markdown", async () => {+ const { fetchImpl, calls } = makeFetchFromSequence([+ {+ matchUrl: /\/jobs$/,+ matchMethod: "POST",+ response: {+ status: 200,+ body: {+ scrape_id: "scrape-id-test",+ status: "done",+ lane: "fast",+ },+ },+ },+ {+ matchUrl: /\/jobs\/scrape-id-test\/result$/,+ matchMethod: "GET",+ response: {+ status: 200,+ body: {+ schema_version: 3,+ markdown: "continued paragraph",+ pages: [+ { page: 1, markdown: "continued" },+ { page: 2, markdown: "paragraph" },+ ],+ pages_processed: 2,+ },+ },+ },+ ]);++ const result = await scrapePDFWithFirePDFAsync(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ "auto",+ { fetchImpl, sleepImpl: noopSleep },+ true,+ );++ expect((calls[0].body as any).options.include_page_markdown).toBe(true);+ expect(result.markdown).toBe("continued paragraph");+ expect(result.pageMarkdown).toEqual([+ { page: 1, markdown: "continued" },+ { page: 2, markdown: "paragraph" },+ ]);+ });++ it("fails a page-aware request when FirePDF omits the page payload", async () => {+ const { fetchImpl } = makeFetchFromSequence([+ {+ matchUrl: /\/jobs$/,+ matchMethod: "POST",+ response: {+ status: 200,+ body: {+ scrape_id: "scrape-id-test",+ status: "done",+ lane: "fast",+ },+ },+ },+ {+ matchUrl: /\/jobs\/scrape-id-test\/result$/,+ matchMethod: "GET",+ response: {+ status: 200,+ body: {+ schema_version: 1,+ markdown: "document only",+ pages_processed: 1,+ },+ },+ },+ ]);++ const error = await scrapePDFWithFirePDFAsync(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ "auto",+ { fetchImpl, sleepImpl: noopSleep },+ true,+ ).catch(error => error);++ expect(error).toBeInstanceOf(FirePdfAsyncFailure);+ expect(error.reason).toBe("http_5xx");});it("submits without team context when the snapshot is absent", async () => {@@ -262,7 +355,12 @@ describe("scrapePDFWithFirePDFAsync", () => {matchMethod: "POST",response: {status: 200,- body: { scrape_id: "scrape-id-test", status: "done", lane: "fast", retry_after_ms: 0 },+ body: {+ scrape_id: "scrape-id-test",+ status: "done",+ lane: "fast",+ retry_after_ms: 0,+ },},},{@@ -284,15 +382,24 @@ describe("scrapePDFWithFirePDFAsync", () => {const meta = makeMeta();meta.internalOptions.teamConcurrency = null;- const result = await scrapePDFWithFirePDFAsync(meta, "BASE64", undefined, undefined, undefined, {- fetchImpl,- fallbackImpl: fallback,- sleepImpl: noopSleep,- });+ const result = await scrapePDFWithFirePDFAsync(+ meta,+ "BASE64",+ undefined,+ undefined,+ undefined,+ {+ fetchImpl,+ fallbackImpl: fallback,+ sleepImpl: noopSleep,+ },+ );// Missing snapshot must never block the scrape β field simply absent.expect(result.markdown).toBe("# No context");- expect((calls[0].body as { team_concurrency?: number }).team_concurrency).toBeUndefined();+ expect(+ (calls[0].body as { team_concurrency?: number }).team_concurrency,+ ).toBeUndefined();expect(fallback).not.toHaveBeenCalled();});@@ -335,7 +442,13 @@ describe("scrapePDFWithFirePDFAsync", () => {expect(error).toBeInstanceOf(FirePdfAsyncFailure);expect(error.reason).toBe("network_error");- expect(calls.map(({ url, method, headers }) => ({ url, method, ...(headers !== undefined && { headers }) }))).toEqual([+ expect(+ calls.map(({ url, method, headers }) => ({+ url,+ method,+ ...(headers !== undefined && { headers }),+ })),+ ).toEqual([{url: "http://fire-pdf.test/jobs",method: "POST",diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.tsnew file mode 100644index 000000000..e741a6ac9--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts@@ -0,0 +1,299 @@+import {+ getPdfResultFromCache,+ savePdfResultToCache,+} from "../../../../../lib/gcs-pdf-cache";+import {+ cacheKeyShape,+ maybeSaveResult,+ tryGetCached,+} from "../fire-pdf/cache";++vi.mock("../../../../../lib/gcs-pdf-cache", () => ({+ getPdfResultFromCache: vi.fn(),+ savePdfResultToCache: vi.fn(),+}));++const getCached = vi.mocked(getPdfResultFromCache);+const saveCached = vi.mocked(savePdfResultToCache);++function makeMeta(zeroDataRetention = false) {+ return {+ id: "page-cache-test",+ logger: {+ info: vi.fn(),+ warn: vi.fn(),+ },+ internalOptions: { zeroDataRetention },+ } as any;+}++describe("FirePDF page-markdown cache capabilities", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ getCached.mockResolvedValue(null);+ saveCached.mockResolvedValue(null);+ });++ it("uses versioned page-capable variants", () => {+ expect(cacheKeyShape("auto", undefined, true)).toMatchObject({+ cacheable: true,+ ownVariant: "page-markdown-v1",+ lookupVariants: ["page-markdown-v1", "ocr-page-markdown-v1"],+ });+ expect(cacheKeyShape("ocr", undefined, true)).toMatchObject({+ cacheable: true,+ ownVariant: "ocr-page-markdown-v1",+ lookupVariants: ["ocr-page-markdown-v1"],+ });+ expect(cacheKeyShape("fast", undefined, true).cacheable).toBe(false);+ expect(cacheKeyShape("auto", 5, true).cacheable).toBe(false);+ });++ it("never serves a document-only cache entry to a page-aware request", async () => {+ getCached.mockResolvedValueOnce({+ markdown: "legacy",+ html: "<p>legacy</p>",+ });++ const result = await tryGetCached(+ makeMeta(),+ "BASE64",+ "ocr",+ undefined,+ 2,+ true,+ );++ expect(result).toBeNull();+ expect(getCached).toHaveBeenCalledOnce();+ expect(getCached).toHaveBeenCalledWith(+ "BASE64",+ "firepdf",+ "ocr-page-markdown-v1",+ );+ });++ it("rejects malformed page payloads in page-capable variants", async () => {+ getCached+ .mockResolvedValueOnce({+ markdown: "wrong type",+ html: "<p>wrong type</p>",+ pageMarkdown: "not-an-array" as never,+ })+ .mockResolvedValueOnce({+ markdown: "wrong item",+ html: "<p>wrong item</p>",+ pageMarkdown: [{ page: 0, markdown: 42 }] as never,+ });++ const result = await tryGetCached(+ makeMeta(),+ "BASE64",+ "auto",+ undefined,+ 2,+ true,+ );++ expect(result).toBeNull();+ expect(getCached).toHaveBeenCalledTimes(2);+ });++ it("rejects page-capable variants missing required document fields", async () => {+ getCached+ .mockResolvedValueOnce({+ html: "<p>missing markdown</p>",+ pageMarkdown: [{ page: 1, markdown: "one" }],+ } as never)+ .mockResolvedValueOnce({+ markdown: "missing html",+ pageMarkdown: [{ page: 1, markdown: "one" }],+ } as never);++ const result = await tryGetCached(+ makeMeta(),+ "BASE64",+ "auto",+ undefined,+ 1,+ true,+ );++ expect(result).toBeNull();+ expect(getCached).toHaveBeenCalledTimes(2);+ });++ it("accepts an explicitly empty cached markdown string", async () => {+ getCached.mockResolvedValueOnce({+ markdown: "",+ html: "",+ pageMarkdown: [{ page: 1, markdown: "" }],+ });++ const result = await tryGetCached(+ makeMeta(),+ "BASE64",+ "auto",+ undefined,+ 1,+ true,+ );++ expect(result).toMatchObject({ markdown: "", html: "" });+ });++ it("rejects invalid cached page counts", async () => {+ getCached+ .mockResolvedValueOnce({+ markdown: "wrong type",+ html: "<p>wrong type</p>",+ pagesProcessed: "1",+ pageMarkdown: [{ page: 1, markdown: "one" }],+ } as never)+ .mockResolvedValueOnce({+ markdown: "negative",+ html: "<p>negative</p>",+ pagesProcessed: -1,+ pageMarkdown: [{ page: 1, markdown: "one" }],+ });++ const result = await tryGetCached(+ makeMeta(),+ "BASE64",+ "auto",+ undefined,+ 1,+ true,+ );++ expect(result).toBeNull();+ expect(getCached).toHaveBeenCalledTimes(2);+ });++ it("serves page-capable entries and preserves the page-count fallback", async () => {+ getCached.mockResolvedValueOnce({+ markdown: "whole",+ html: "<p>whole</p>",+ pageMarkdown: [+ { page: 1, markdown: "one" },+ { page: 2, markdown: "two" },+ ],+ });++ const result = await tryGetCached(+ makeMeta(),+ "BASE64",+ "auto",+ undefined,+ 2,+ true,+ );++ expect(result?.pageMarkdown).toHaveLength(2);+ expect(result?.pagesProcessed).toBe(2);+ });++ it("strips page payloads when a legacy request reuses an enriched sidecar", async () => {+ getCached.mockResolvedValueOnce(null).mockResolvedValueOnce({+ markdown: "whole",+ html: "<p>whole</p>",+ pageMarkdown: [+ { page: 1, markdown: "one" },+ { page: 2, markdown: "two" },+ ],+ });++ const result = await tryGetCached(+ makeMeta(),+ "BASE64",+ "auto",+ undefined,+ 2,+ false,+ );++ expect(result).toMatchObject({+ markdown: "whole",+ html: "<p>whole</p>",+ pagesProcessed: 2,+ });+ expect(result?.pageMarkdown).toBeUndefined();+ expect(getCached).toHaveBeenNthCalledWith(+ 2,+ "BASE64",+ "firepdf",+ "page-markdown-v1",+ );+ });++ it("writes an enriched sidecar plus a compact legacy entry", async () => {+ const pageMarkdown = [+ { page: 1, markdown: "one" },+ { page: 2, markdown: "two" },+ ];++ await maybeSaveResult({+ meta: makeMeta(),+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: true,+ result: {+ markdown: "whole",+ html: "<p>whole</p>",+ pagesProcessed: 2,+ pageMarkdown,+ },+ });++ expect(saveCached).toHaveBeenCalledTimes(2);+ expect(saveCached).toHaveBeenNthCalledWith(+ 1,+ "BASE64",+ expect.objectContaining({ pageMarkdown }),+ "firepdf",+ "page-markdown-v1",+ );+ expect(saveCached).toHaveBeenNthCalledWith(+ 2,+ "BASE64",+ expect.not.objectContaining({ pageMarkdown: expect.anything() }),+ "firepdf",+ undefined,+ );+ });++ it("does not rewrite an existing valid compact legacy entry", async () => {+ getCached.mockResolvedValueOnce({+ markdown: "existing",+ html: "<p>existing</p>",+ pagesProcessed: 2,+ });++ await maybeSaveResult({+ meta: makeMeta(),+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: true,+ result: {+ markdown: "whole",+ html: "<p>whole</p>",+ pagesProcessed: 2,+ pageMarkdown: [+ { page: 1, markdown: "one" },+ { page: 2, markdown: "two" },+ ],+ },+ });++ expect(getCached).toHaveBeenCalledWith("BASE64", "firepdf", undefined);+ expect(saveCached).toHaveBeenCalledOnce();+ expect(saveCached).toHaveBeenCalledWith(+ "BASE64",+ expect.objectContaining({ pageMarkdown: expect.any(Array) }),+ "firepdf",+ "page-markdown-v1",+ );+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.tsnew file mode 100644index 000000000..48bd86b6e--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts@@ -0,0 +1,70 @@+vi.mock("../../../../../services", async importOriginal => ({+ ...(await importOriginal<typeof import("../../../../../services")>()),+ useIndex: true,+}));++import { config } from "../../../../../config";+import { shouldUseIndex } from "../../index";+import { sendDocumentToIndex } from "../../index/index";++describe("PDF page-markdown URL index policy", () => {+ it("bypasses URL-index reads for page-aware requests", () => {+ const originalWriteOnly = config.FIRECRAWL_INDEX_WRITE_ONLY;+ (+ config as { FIRECRAWL_INDEX_WRITE_ONLY?: boolean }+ ).FIRECRAWL_INDEX_WRITE_ONLY = false;+ const baseMeta = {+ internalOptions: { isParse: false },+ options: {+ formats: ["markdown"],+ parsers: [{ type: "pdf" }],+ maxAge: 60_000,+ },+ } as any;++ try {+ expect(shouldUseIndex(baseMeta)).toBe(true);+ expect(+ shouldUseIndex({+ ...baseMeta,+ options: {+ ...baseMeta.options,+ parsers: [{ type: "pdf", pageMarkdown: true }],+ },+ }),+ ).toBe(false);+ } finally {+ (+ config as { FIRECRAWL_INDEX_WRITE_ONLY?: boolean }+ ).FIRECRAWL_INDEX_WRITE_ONLY = originalWriteOnly;+ }+ });++ it("does not write page-aware results to the document-only URL index", async () => {+ const document = {+ markdown: "whole document",+ pages: [{ pageNumber: 1, markdown: "page one" }],+ rawHtml: "<p>whole document</p>",+ metadata: {+ sourceURL: "https://example.com/file.pdf",+ },+ } as any;+ const meta = {+ url: "https://example.com/file.pdf",+ winnerEngine: "pdf",+ options: {+ storeInCache: true,+ parsers: [{ type: "pdf", pageMarkdown: true }],+ },+ internalOptions: {+ isParse: false,+ zeroDataRetention: false,+ },+ } as any;++ const result = await sendDocumentToIndex(meta, document);++ expect(result).toBe(document);+ expect(result.metadata.indexId).toBeUndefined();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.tsindex 42cf54ffa..3df380609 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts@@ -37,6 +37,7 @@ export async function scrapePDFWithFirePDFAsync(pagesProcessed?: number,mode?: PDFMode,deps: FirePdfAsyncDeps = {},+ includePageMarkdown = false,): Promise<PDFProcessorResult> {const fetchImpl = deps.fetchImpl ?? undiciFetch;const fallbackImpl = deps.fallbackImpl ?? scrapePDFWithFirePDF;@@ -47,7 +48,14 @@ export async function scrapePDFWithFirePDFAsync(// Async persists inputs and queue state, so ZDR is excluded until that// lifecycle has an explicit delete-on-completion contract.if (meta.internalOptions.zeroDataRetention) {- return fallbackImpl(meta, base64Content, maxPages, pagesProcessed, mode);+ return fallbackImpl(+ meta,+ base64Content,+ maxPages,+ pagesProcessed,+ mode,+ includePageMarkdown,+ );}const cached = await tryGetCached(@@ -56,6 +64,7 @@ export async function scrapePDFWithFirePDFAsync(mode,maxPages,pagesProcessed,+ includePageMarkdown,);if (cached) return cached;@@ -73,7 +82,14 @@ export async function scrapePDFWithFirePDFAsync(if (!baseUrl) {// Should be unreachable β call site checks this β but fall back rather// than crash if a route somehow bypasses the gate.- return fallbackImpl(meta, base64Content, maxPages, pagesProcessed, mode);+ return fallbackImpl(+ meta,+ base64Content,+ maxPages,+ pagesProcessed,+ mode,+ includePageMarkdown,+ );}const overallStartedAt = now();@@ -105,6 +121,7 @@ export async function scrapePDFWithFirePDFAsync(maxPages,pagesProcessed,mode,+ includePageMarkdown,deadlineAt,teamConcurrency,fetchImpl,@@ -160,6 +177,11 @@ export async function scrapePDFWithFirePDFAsync(// ββ Assemble + cache save βββββββββββββββββββββββββββββββββββββββββββββconst pages =fetched.pages_processed ?? polled.poll.pages_processed ?? pagesProcessed;+ if (includePageMarkdown && fetched.pages === undefined) {+ failAsync(meta, "http_5xx", {+ note: "FirePDF result omitted requested physical page markdown",+ });+ }const durationMs = now() - overallStartedAt;firePdfAsyncTotalDurationSeconds.observe(durationMs / 1000);@@ -168,6 +190,7 @@ export async function scrapePDFWithFirePDFAsync(durationMs,markdownLength: fetched.markdown.length,pagesProcessed: pages,+ pageMarkdownPages: fetched.pages?.length,failedPages: fetched.failed_pages,partialPages: fetched.partial_pages,pollCount: polled.pollCount,@@ -177,6 +200,7 @@ export async function scrapePDFWithFirePDFAsync(markdown: fetched.markdown,html: await safeMarkdownToHtml(fetched.markdown, meta.logger, meta.id),pagesProcessed: pages,+ ...(fetched.pages ? { pageMarkdown: fetched.pages } : {}),};await maybeSaveResult({@@ -184,6 +208,7 @@ export async function scrapePDFWithFirePDFAsync(base64Content,mode,maxPages,+ includePageMarkdown,result: processorResult,});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.tsindex c53dd93ab..494d044c1 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts@@ -10,15 +10,67 @@ import {// entries. `fast` mode bypasses entirely (hard cost ceiling β must fail on// scanned PDFs, not serve a cached OCR result), as does any call with// `maxPages` (the cached entry may have been written with a different cap).-function cacheKeyShape(+const PAGE_MARKDOWN_VARIANT = "page-markdown-v1";+const OCR_PAGE_MARKDOWN_VARIANT = "ocr-page-markdown-v1";++function isValidCachedDocument(+ value: unknown,+): value is Pick<PDFProcessorResult, "html"> & { markdown: string } {+ if (typeof value !== "object" || value === null) return false;+ const cached = value as {+ markdown?: unknown;+ html?: unknown;+ pagesProcessed?: unknown;+ };+ return (+ typeof cached.markdown === "string" &&+ typeof cached.html === "string" &&+ (cached.pagesProcessed === undefined ||+ (typeof cached.pagesProcessed === "number" &&+ Number.isInteger(cached.pagesProcessed) &&+ cached.pagesProcessed >= 0))+ );+}++function isValidPageMarkdown(+ value: unknown,+): value is NonNullable<PDFProcessorResult["pageMarkdown"]> {+ return (+ Array.isArray(value) &&+ value.every(+ page =>+ typeof page === "object" &&+ page !== null &&+ Number.isInteger((page as { page?: unknown }).page) &&+ Number((page as { page: number }).page) > 0 &&+ typeof (page as { markdown?: unknown }).markdown === "string",+ )+ );+}++export function cacheKeyShape(mode: PDFMode | undefined,maxPages: number | undefined,+ includePageMarkdown: boolean,) {const cacheable = mode !== "fast" && !maxPages;- const ownVariant: string | undefined = mode === "ocr" ? "ocr" : undefined;- const lookupVariants: (string | undefined)[] =- mode === "ocr" ? ["ocr"] : [undefined, "ocr"];- return { cacheable, ownVariant, lookupVariants };+ const baseVariant: string | undefined = mode === "ocr" ? "ocr" : undefined;+ const ownVariant: string | undefined = includePageMarkdown+ ? mode === "ocr"+ ? OCR_PAGE_MARKDOWN_VARIANT+ : PAGE_MARKDOWN_VARIANT+ : baseVariant;++ // Page-aware requests may only consume page-capable artifacts. Legacy+ // requests prefer their compact entry but can reuse an enriched sidecar.+ const lookupVariants: (string | undefined)[] = includePageMarkdown+ ? mode === "ocr"+ ? [OCR_PAGE_MARKDOWN_VARIANT]+ : [PAGE_MARKDOWN_VARIANT, OCR_PAGE_MARKDOWN_VARIANT]+ : mode === "ocr"+ ? ["ocr", OCR_PAGE_MARKDOWN_VARIANT]+ : [undefined, PAGE_MARKDOWN_VARIANT, "ocr", OCR_PAGE_MARKDOWN_VARIANT];+ return { cacheable, ownVariant, baseVariant, lookupVariants };}export async function tryGetCached(@@ -27,9 +79,14 @@ export async function tryGetCached(mode: PDFMode | undefined,maxPages: number | undefined,pagesProcessed: number | undefined,+ includePageMarkdown: boolean,): Promise<PDFProcessorResult | null> {if (meta.internalOptions.zeroDataRetention) return null;- const { cacheable, lookupVariants } = cacheKeyShape(mode, maxPages);+ const { cacheable, lookupVariants } = cacheKeyShape(+ mode,+ maxPages,+ includePageMarkdown,+ );if (!cacheable) return null;for (const variant of lookupVariants) {@@ -40,21 +97,36 @@ export async function tryGetCached(variant,);if (cached) {- meta.logger.info("Using cached FirePDF result (async path)", {+ if (+ !isValidCachedDocument(cached) ||+ (includePageMarkdown && !isValidPageMarkdown(cached.pageMarkdown))+ ) {+ // Defense in depth: variant names are the capability boundary, but+ // never let a malformed/old artifact satisfy a cache lookup.+ continue;+ }+ meta.logger.info("Using cached FirePDF result", {scrapeId: meta.id,requestedMode: mode,cacheVariant: variant ?? "base",});+ if (!includePageMarkdown) {+ const { pageMarkdown: _pageMarkdown, ...compactCached } = cached;+ return {+ ...compactCached,+ pagesProcessed: cached.pagesProcessed ?? pagesProcessed,+ };+ }return {...cached,pagesProcessed: cached.pagesProcessed ?? pagesProcessed,};}} catch (error) {- meta.logger.warn(- "Error checking FirePDF cache (async path), proceeding",- { error, cacheVariant: variant ?? "base" },- );+ meta.logger.warn("Error checking FirePDF cache, proceeding", {+ error,+ cacheVariant: variant ?? "base",+ });}}return null;@@ -65,19 +137,45 @@ export async function maybeSaveResult(args: {base64Content: string;mode: PDFMode | undefined;maxPages: number | undefined;+ includePageMarkdown: boolean;result: PDFProcessorResult & { markdown: string };}): Promise<void> {- const { meta, base64Content, mode, maxPages, result } = args;+ const { meta, base64Content, mode, maxPages, includePageMarkdown, result } =+ args;if (meta.internalOptions.zeroDataRetention) return;- const { cacheable, ownVariant } = cacheKeyShape(mode, maxPages);+ const { cacheable, ownVariant, baseVariant } = cacheKeyShape(+ mode,+ maxPages,+ includePageMarkdown,+ );if (!cacheable) return;try {await savePdfResultToCache(base64Content, result, "firepdf", ownVariant);+ // A page-capable parse is also a valid legacy result. Populate the compact+ // base key when it is missing so a later legacy request never repeats the+ // conversion. A page-sidecar miss can coexist with a warm legacy key during+ // rollout, so avoid rewriting that object. Strip the page payload to keep+ // the hot-path cache object small.+ if (includePageMarkdown && ownVariant !== baseVariant) {+ const existingBase = await getPdfResultFromCache(+ base64Content,+ "firepdf",+ baseVariant,+ );+ if (!existingBase || !isValidCachedDocument(existingBase)) {+ const { pageMarkdown: _pageMarkdown, ...baseResult } = result;+ await savePdfResultToCache(+ base64Content,+ baseResult,+ "firepdf",+ baseVariant,+ );+ }+ }} catch (error) {- meta.logger.warn(- "Error saving FirePDF async result to cache (continuing)",- { error },- );+ meta.logger.warn("Error saving FirePDF result to cache (continuing)", {+ error,+ });}}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.tsindex d99b8f1fe..7f2f66170 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts@@ -48,8 +48,15 @@ export const pollResponseSchema = z.object({});export const resultResponseSchema = z.object({- schema_version: z.literal(1).optional(),+ schema_version: z+ .union([z.literal(1), z.literal(2), z.literal(3)])+ .optional(),markdown: z.string(),+ pages: z+ .array(+ z.object({ page: z.number().int().positive(), markdown: z.string() }),+ )+ .optional(),pages_processed: z.number().optional(),failed_pages: z.array(z.number()).nullable().optional(),partial_pages: z.array(z.number()).nullable().optional(),diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.tsindex 9d58ec993..e0dfa8d2c 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts@@ -19,6 +19,7 @@ type SubmitArgs = {maxPages: number | undefined;pagesProcessed: number | undefined;mode: PDFMode | undefined;+ includePageMarkdown: boolean;deadlineAt: string;/** Team's sold concurrency from the ACUC (ENG-5049 account context).* Optional: entitlement lookup must never block or fail a scrape. */@@ -58,6 +59,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {maxPages,pagesProcessed,mode,+ includePageMarkdown,deadlineAt,teamConcurrency,fetchImpl,@@ -86,6 +88,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {...(pagesProcessed !== undefined && { pages_estimate: pagesProcessed }),...(maxPages !== undefined && { max_pages: maxPages }),...(mode !== undefined && { mode }),+ ...(includePageMarkdown && { include_page_markdown: true }),},};diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsindex fa8dec3b2..f41998235 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -5,11 +5,8 @@ import { z } from "zod";import type { PDFProcessorResult } from "./types";import type { PDFMode } from "../../../../controllers/v2/types";import { safeMarkdownToHtml } from "./markdownToHtml";-import {- createPdfCacheKey,- getPdfResultFromCache,- savePdfResultToCache,-} from "../../../../lib/gcs-pdf-cache";+import { createPdfCacheKey } from "../../../../lib/gcs-pdf-cache";+import { maybeSaveResult, tryGetCached } from "./fire-pdf/cache";/*** Reconcile an existing page count with what fire-pdf reported.@@ -47,6 +44,7 @@ export async function scrapePDFWithFirePDF(maxPages?: number,pagesProcessed?: number,mode?: PDFMode,+ includePageMarkdown = false,): Promise<PDFProcessorResult> {const logger = meta.logger;@@ -64,40 +62,17 @@ export async function scrapePDFWithFirePDF(// scanned PDFs, not serve a cached OCR result).const cacheable =mode !== "fast" && !maxPages && !meta.internalOptions.zeroDataRetention;- const ownVariant: string | undefined = mode === "ocr" ? "ocr" : undefined;- const lookupVariants: (string | undefined)[] =- mode === "ocr" ? ["ocr"] : [undefined, "ocr"];-- if (cacheable) {- for (const variant of lookupVariants) {- try {- const cached = await getPdfResultFromCache(- base64Content,- "firepdf",- variant,- );- if (cached) {- logger.info("Using cached FirePDF result", {- scrapeId: meta.id,- requestedMode: mode,- cacheVariant: variant ?? "base",- });- // Cache entries written before pagesProcessed existed don't carry- // the field. Fall back to the caller's pagesProcessed argument so- // billing on a stale hit doesn't silently regress to 0.- return {- ...cached,- pagesProcessed: cached.pagesProcessed ?? pagesProcessed,- };- }- } catch (error) {- logger.warn("Error checking FirePDF cache, proceeding", {- error,- cacheVariant: variant ?? "base",- });- }- }- }+ const cached = cacheable+ ? await tryGetCached(+ meta,+ base64Content,+ mode,+ maxPages,+ pagesProcessed,+ includePageMarkdown,+ )+ : null;+ if (cached) return cached;meta.abort.throwIfAborted();@@ -144,6 +119,7 @@ export async function scrapePDFWithFirePDF(scrape_id: meta.id,...(maxPages !== undefined && { max_pages: maxPages }),...(mode !== undefined && { mode }),+ ...(includePageMarkdown && { include_page_markdown: true }),// Enrichment for the fire-pdf jobs DB / dashboard. fire-pdf treats// these as optional β older fire-pdf builds will ignore unknown fields.team_id: meta.internalOptions.teamId,@@ -161,12 +137,22 @@ export async function scrapePDFWithFirePDF(markdown: z.string(),failed_pages: z.array(z.number()).nullable(),pages_processed: z.number().optional(),+ pages: z+ .array(+ z.object({ page: z.number().int().positive(), markdown: z.string() }),+ )+ .optional(),}),mock: meta.mock,abort: meta.abort.asSignal(),});const durationMs = Date.now() - startedAt;+ if (includePageMarkdown && resp.pages === undefined) {+ throw new Error(+ "FirePDF response did not include requested physical page markdown",+ );+ }const pages = resp.pages_processed ?? pagesProcessed;logger.info("FirePDF completed", {@@ -183,19 +169,18 @@ export async function scrapePDFWithFirePDF(markdown: resp.markdown,html: await safeMarkdownToHtml(resp.markdown, logger, meta.id),pagesProcessed: pages,+ ...(resp.pages ? { pageMarkdown: resp.pages } : {}),};if (cacheable) {- try {- await savePdfResultToCache(- base64Content,- processorResult,- "firepdf",- ownVariant,- );- } catch (error) {- logger.warn("Error saving FirePDF result to cache", { error });- }+ await maybeSaveResult({+ meta,+ base64Content,+ mode,+ maxPages,+ includePageMarkdown,+ result: processorResult,+ });}return processorResult;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 4ebf2bbfe..3ce5be08e 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -18,6 +18,7 @@ import {shouldParsePDF,getPDFMaxPages,getPDFMode,+ getPDFPageMarkdown,getFirePdfAsync,} from "../../../../controllers/v2/types";import type { PDFMode } from "../../../../controllers/v2/types";@@ -59,6 +60,13 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {const shouldParse = shouldParsePDF(meta.options.parsers);const maxPages = getPDFMaxPages(meta.options.parsers);const mode: PDFMode = getPDFMode(meta.options.parsers);+ const includePageMarkdown = getPDFPageMarkdown(meta.options.parsers);++ if (includePageMarkdown && !config.FIRE_PDF_BASE_URL) {+ throw new Error(+ "Physical page markdown is unavailable because FirePDF is not configured",+ );+ }if (!shouldParse) {if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {@@ -170,7 +178,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {let shadowPagesNeedingOcr: number[] | undefined;const forceFirePDF =- !!meta.options.__forceFirePDF && !!config.FIRE_PDF_BASE_URL;+ (!!meta.options.__forceFirePDF || includePageMarkdown) &&+ !!config.FIRE_PDF_BASE_URL;const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;const logger = meta.logger.child({ method: "scrapePDF/processPdf" });@@ -374,7 +383,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {// OCR / MU fallback.// Skipped only when Rust extraction is enabled AND mode is "fast",// unless we explicitly routed to MinerU via MINERU_PERCENT.- const skipOCR = rustEnabled && mode === "fast" && !routeToMinerU;+ const skipOCR =+ rustEnabled && mode === "fast" && !routeToMinerU && !forceFirePDF;if (!result && !skipOCR) {const pdfBuffer = await readFile(tempFilePath);const fileSizeBytes = pdfBuffer.length;@@ -435,22 +445,64 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {});}try {- result = await (- useAsync ? scrapePDFWithFirePDFAsync : scrapePDFWithFirePDF- )(- {- ...meta,- logger: meta.logger.child({- method: useAsync- ? "scrapePDF/firePDFAsync"- : "scrapePDF/firePDF",- }),- },- base64Content,- maxPages,- effectivePageCount,- mode,- );+ const firePdfMeta = {+ ...meta,+ logger: meta.logger.child({+ method: useAsync ? "scrapePDF/firePDFAsync" : "scrapePDF/firePDF",+ }),+ };+ if (useAsync) {+ try {+ result = await scrapePDFWithFirePDFAsync(+ firePdfMeta,+ base64Content,+ maxPages,+ effectivePageCount,+ mode,+ undefined,+ includePageMarkdown,+ );+ } catch (error) {+ if (+ !includePageMarkdown ||+ error instanceof RemoveFeatureError ||+ error instanceof AbortManagerThrownError+ ) {+ throw error;+ }+ meta.logger.warn(+ "FirePDF async page markdown failed -- retrying synchronously",+ {+ method: "scrapePDF/firePDFFallback",+ error,+ scrape_id: meta.id,+ team_id: meta.internalOptions.teamId,+ },+ );+ result = await scrapePDFWithFirePDF(+ {+ ...firePdfMeta,+ logger: meta.logger.child({+ method: "scrapePDF/firePDFSyncFallback",+ }),+ },+ base64Content,+ maxPages,+ effectivePageCount,+ mode,+ true,+ );+ }+ } else {+ result = await scrapePDFWithFirePDF(+ firePdfMeta,+ base64Content,+ maxPages,+ effectivePageCount,+ mode,+ includePageMarkdown,+ );+ }effectivePageCount = reconcilePageCountWithFirePdf(effectivePageCount,result,@@ -630,6 +682,14 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {statusCode: response.status,html: result?.html ?? "",markdown: result?.markdown ?? "",+ ...(includePageMarkdown && result?.pageMarkdown+ ? {+ pages: result.pageMarkdown.map(page => ({+ pageNumber: page.page,+ markdown: page.markdown,+ })),+ }+ : {}),pdfMetadata: {numPages: effectivePageCount,totalPages: totalPageCount,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/types.tsindex b04067b4c..dc5f739f0 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts@@ -1,6 +1,13 @@+export type PdfPageMarkdown = {+ /** 1-based physical PDF page number returned by fire-pdf. */+ page: number;+ markdown: string;+};+export type PDFProcessorResult = {html: string;markdown?: string;+ pageMarkdown?: PdfPageMarkdown[];/*** Pages the underlying engine actually processed for this request.* Currently populated only by fire-pdf (via OcrSuccessBody.pages_processed).diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 8d1a154e8..f959fc0fc 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -977,6 +977,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {let document: Document = {markdown: engineResult.markdown,+ pages: engineResult.pages,rawHtml: engineResult.html,json: engineResult.json,screenshot: engineResult.screenshot,