Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(v2): pageMarkers PDF parser option β€” per-page page attribution in document.markdown (#4365)

Abimael Martell24b811264004
diff --git a/apps/api/src/__tests__/pdf-parser-options.test.ts b/apps/api/src/__tests__/pdf-parser-options.test.ts
index f891efe6e..d3f783f71 100644
--- a/apps/api/src/__tests__/pdf-parser-options.test.ts
+++ b/apps/api/src/__tests__/pdf-parser-options.test.ts
@@ -1,6 +1,7 @@
import {
getPDFBlocks,
getPDFPageMarkdown,
+ getPDFPageMarkers,
scrapeOptions,
} from "../controllers/v2/types";
@@ -23,6 +24,27 @@ describe("PDF parser option getters", () => {
expect(getPDFBlocks([{ type: "pdf" }])).toBe(false);
expect(getPDFBlocks(undefined)).toBe(false);
});
+
+ it("reads the `pageMarkers` option", () => {
+ expect(getPDFPageMarkers([{ type: "pdf", pageMarkers: true }])).toBe(true);
+ expect(getPDFPageMarkers([{ type: "pdf", pageMarkers: false }])).toBe(
+ false,
+ );
+ expect(getPDFPageMarkers([{ type: "pdf" }])).toBe(false);
+ expect(getPDFPageMarkers(["pdf"])).toBe(false);
+ expect(getPDFPageMarkers(undefined)).toBe(false);
+ });
+
+ it("keeps `pageMarkers` flowing through the alias-normalizing transform", () => {
+ const parser = parsePdfParser({
+ type: "pdf",
+ pageMarkers: true,
+ pageMarkdown: true,
+ });
+ expect(parser.pageMarkers).toBe(true);
+ expect(parser.pages).toBe(true);
+ expect(getPDFPageMarkers([parser as any])).toBe(true);
+ });
});
describe("deprecated pageMarkdown alias", () => {
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 54233c799..496d00384 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -497,6 +497,13 @@ const pdfParserWithOptions = z
* reading order) alongside document markdown β€” populates
* `document.blocks`. */
blocks: z.boolean().optional(),
+ /** Join PDF pages in `document.markdown` with
+ * `\n\n---\n\n<!-- page N -->\n\n` where N is the 1-based physical page
+ * of the content that follows. Markers appear between pages only (no
+ * leading marker for page 1), and numbering may skip pages merged by
+ * cross-page stitching β€” callers that need every physical page should
+ * use `pages: true` instead. No new response field. */
+ pageMarkers: z.boolean().optional(),
// Experimental: route this request through the fire-pdf async pipeline
// (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back
// to sync on any async-path failure, so user-visible behavior is unchanged
@@ -583,6 +590,16 @@ export function getPDFBlocks(parsers?: Parsers): boolean {
return false;
}
+export function getPDFPageMarkers(parsers?: Parsers): boolean {
+ if (!parsers) return false;
+ for (const parser of parsers) {
+ if (typeof parser === "object" && parser.type === "pdf") {
+ return parser.pageMarkers === true;
+ }
+ }
+ return false;
+}
+
export function getFirePdfAsync(parsers?: Parsers): boolean {
if (!parsers) return false;
for (const parser of parsers) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 315e68eda..040180633 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -30,6 +30,7 @@ import { hasFormatOfType } from "../../../lib/format-utils";
import {
getPDFBlocks,
getPDFPageMarkdown,
+ getPDFPageMarkers,
} from "../../../controllers/v2/types";
import type { PdfMetadata, PdfPageBlocks } from "./pdf/types";
import { BrandingProfile } from "../../../types/branding";
@@ -567,9 +568,10 @@ export function shouldUseIndex(meta: Meta) {
!hasFormatOfType(meta.options.formats, "changeTracking") &&
!hasFormatOfType(meta.options.formats, "branding") &&
// The URL index does not yet persist physical-page or typed-block
- // capability metadata.
+ // capability metadata, and its markdown never carries page markers.
!getPDFPageMarkdown(meta.options.parsers) &&
!getPDFBlocks(meta.options.parsers) &&
+ !getPDFPageMarkers(meta.options.parsers) &&
!hasCustomScreenshotSettings &&
meta.options.maxAge !== 0 &&
(meta.options.headers === undefined ||
@@ -680,17 +682,19 @@ export async function buildFallbackList(meta: Meta): Promise<
_engines.push(...indexEngines);
meta.internalOptions.forceEngine = indexEngines;
} else if (meta.internalOptions.agentIndexOnly) {
- // Index documents carry no physical-page or typed-block payloads, so an
- // index-only request that demands them can only be answered wrong. Fail
- // loud with the canonical index-only error (maps to a clean 4xx and
- // tells the caller how to unlock live scraping) instead of silently
- // serving a document without the capability.
+ // Index documents carry no physical-page or typed-block payloads, and
+ // their markdown never carries page markers, so an index-only request
+ // that demands them can only be answered wrong. Fail loud with the
+ // canonical index-only error (maps to a clean 4xx and tells the caller
+ // how to unlock live scraping) instead of silently serving a document
+ // without the capability.
if (
getPDFPageMarkdown(meta.options.parsers) ||
- getPDFBlocks(meta.options.parsers)
+ getPDFBlocks(meta.options.parsers) ||
+ getPDFPageMarkers(meta.options.parsers)
) {
meta.logger.warn(
- "agentIndexOnly request demands pageMarkdown/blocks, which the URL index cannot serve",
+ "agentIndexOnly request demands pageMarkdown/blocks/pageMarkers, which the URL index cannot serve",
{ parsers: meta.options.parsers },
);
throw new AgentIndexOnlyError();
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index e2f9596b4..223a297c8 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -40,6 +40,7 @@ import {
getPDFBlocks,
getPDFMaxPages,
getPDFPageMarkdown,
+ getPDFPageMarkers,
shouldParsePDF,
} from "../../../../controllers/v2/types";
import { hasFormatOfType } from "../../../../lib/format-utils";
@@ -64,8 +65,10 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
// Page-aware and block-aware results are capability-specific and are not
// represented in the URL index schema yet. Do not write an entry that
// could later be served without its required pages/blocks payload.
+ // Marker-bearing markdown is mutated output β€” never index it either.
!getPDFPageMarkdown(meta.options.parsers) &&
!getPDFBlocks(meta.options.parsers) &&
+ !getPDFPageMarkers(meta.options.parsers) &&
!meta.options.parsers?.some(parser => {
if (
typeof parser === "object" &&
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
index 6ad00666c..4acd9f504 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
@@ -211,3 +211,105 @@ describe("scrapePDFWithFirePDF typed blocks", () => {
).rejects.toThrow(/did not include requested typed blocks/);
});
});
+
+describe("scrapePDFWithFirePDF page markers", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it("requests marker-joined markdown and returns it verbatim", async () => {
+ const marked = "Page 1\n\n---\n\n<!-- page 2 -->\n\nPage 2";
+ mockedRobustFetch.mockResolvedValue({
+ markdown: marked,
+ failed_pages: null,
+ pages_processed: 2,
+ page_markers: true,
+ } as any);
+
+ const result = await scrapePDFWithFirePDF(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ "auto",
+ false,
+ false,
+ true,
+ );
+
+ expect(mockedRobustFetch).toHaveBeenCalledWith(
+ expect.objectContaining({
+ body: expect.objectContaining({ page_markers: true }),
+ }),
+ );
+ expect(result.markdown).toBe(marked);
+ });
+
+ it("does not send page_markers for plain requests", async () => {
+ mockedRobustFetch.mockResolvedValue({
+ markdown: "plain",
+ failed_pages: null,
+ pages_processed: 1,
+ } as any);
+
+ await scrapePDFWithFirePDF(makeMeta(), "BASE64");
+
+ expect(
+ ((mockedRobustFetch.mock.calls[0][0] as any).body as any).page_markers,
+ ).toBeUndefined();
+ });
+
+ it("rejects FirePDF responses that do not acknowledge page markers", async () => {
+ // An old fire-pdf build ignores the unknown `page_markers` request field
+ // and returns ordinary markdown with no echo β€” indistinguishable from
+ // marked output by content alone, so the missing echo must fail loud.
+ mockedRobustFetch.mockResolvedValue({
+ markdown: "Page 1\n\n---\n\nPage 2",
+ failed_pages: null,
+ pages_processed: 2,
+ } as any);
+
+ await expect(
+ scrapePDFWithFirePDF(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ "auto",
+ false,
+ false,
+ true,
+ ),
+ ).rejects.toThrow(/did not acknowledge requested page markers/);
+ });
+
+ it("composes page_markers with include_blocks on the wire", async () => {
+ mockedRobustFetch.mockResolvedValue({
+ markdown: "Page 1\n\n---\n\n<!-- page 2 -->\n\nPage 2",
+ failed_pages: null,
+ pages_processed: 2,
+ blocks: [],
+ page_markers: true,
+ } as any);
+
+ await scrapePDFWithFirePDF(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ "auto",
+ false,
+ true,
+ true,
+ );
+
+ expect(mockedRobustFetch).toHaveBeenCalledWith(
+ expect.objectContaining({
+ body: expect.objectContaining({
+ include_blocks: true,
+ page_markers: true,
+ }),
+ }),
+ );
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
index 842d0d035..03d351129 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
@@ -348,6 +348,99 @@ describe("scrapePDFWithFirePDFAsync", () => {
expect(error.reason).toBe("http_5xx");
});
+ it("requests page markers and returns the acknowledged marked markdown", async () => {
+ const marked = "Page 1\n\n---\n\n<!-- page 2 -->\n\nPage 2";
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: {
+ status: 200,
+ body: {
+ scrape_id: "scrape-id-test",
+ status: "done",
+ lane: "fast",
+ },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: {
+ schema_version: 1,
+ markdown: marked,
+ pages_processed: 2,
+ page_markers: true,
+ },
+ },
+ },
+ ]);
+
+ const result = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ "auto",
+ { fetchImpl, sleepImpl: noopSleep },
+ false,
+ false,
+ true,
+ );
+
+ expect((calls[0].body as any).options.pageMarkers).toBe(true);
+ expect(result.markdown).toBe(marked);
+ });
+
+ it("fails a marker request when the result lacks the page_markers echo", async () => {
+ // An older worker ignores the unknown pageMarkers option and persists
+ // ordinary markdown; without the echo this must fail (and fall back to
+ // the sync path) rather than cache unmarked markdown as marked output.
+ const { fetchImpl } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: {
+ status: 200,
+ body: {
+ scrape_id: "scrape-id-test",
+ status: "done",
+ lane: "fast",
+ },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: {
+ schema_version: 1,
+ markdown: "Page 1\n\n---\n\nPage 2",
+ pages_processed: 2,
+ },
+ },
+ },
+ ]);
+
+ const error = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ "auto",
+ { fetchImpl, sleepImpl: noopSleep },
+ false,
+ false,
+ true,
+ ).catch(error => error);
+
+ expect(error).toBeInstanceOf(FirePdfAsyncFailure);
+ expect(error.reason).toBe("http_5xx");
+ });
+
it("requests and returns typed blocks, tolerating the legacy pages alias", async () => {
const blocks = [
{
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
index d697b3831..24c88fcd7 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
@@ -82,6 +82,43 @@ describe("FirePDF page-markdown cache capabilities", () => {
expect(cacheKeyShape("auto", 5, false, true).cacheable).toBe(false);
});
+ it("maps page-marker requests into a fully disjoint variant family", () => {
+ // pageMarkers rewrites the markdown itself, so no marker lookup may ever
+ // name a non-marker variant (and vice versa β€” see the plain-request test:
+ // its probe list is unchanged and marker-free).
+ expect(cacheKeyShape("auto", undefined, false, false, true)).toMatchObject({
+ cacheable: true,
+ ownVariant: "markers-v1",
+ baseVariant: "markers-v1",
+ lookupVariants: [
+ "markers-v1",
+ "page-markdown-markers-v1",
+ "ocr-markers-v1",
+ "ocr-page-markdown-markers-v1",
+ ],
+ });
+ expect(cacheKeyShape("ocr", undefined, false, false, true)).toMatchObject({
+ cacheable: true,
+ ownVariant: "ocr-markers-v1",
+ lookupVariants: ["ocr-markers-v1", "ocr-page-markdown-markers-v1"],
+ });
+ expect(cacheKeyShape("auto", undefined, false, true, true)).toMatchObject({
+ cacheable: true,
+ ownVariant: "blocks-markers-v1",
+ baseVariant: "markers-v1",
+ lookupVariants: [
+ "blocks-markers-v1",
+ "page-markdown-blocks-markers-v1",
+ "ocr-blocks-markers-v1",
+ "ocr-page-markdown-blocks-markers-v1",
+ ],
+ });
+ expect(cacheKeyShape("fast", undefined, false, false, true).cacheable).toBe(
+ false,
+ );
+ expect(cacheKeyShape("auto", 5, false, false, true).cacheable).toBe(false);
+ });
+
it("keeps the historical probe list for plain requests", () => {
expect(cacheKeyShape("auto", undefined, false, false)).toMatchObject({
cacheable: true,
@@ -504,4 +541,73 @@ describe("FirePDF page-markdown cache capabilities", () => {
undefined,
);
});
+
+ it("saves marker results only under marker variants (never the base key)", async () => {
+ const marked = "Page 1\n\n---\n\n<!-- page 2 -->\n\nPage 2";
+
+ await maybeSaveResult({
+ meta: makeMeta(),
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ pageMarkers: true,
+ result: {
+ markdown: marked,
+ html: "<p>marked</p>",
+ pagesProcessed: 2,
+ },
+ });
+
+ // Marker markdown must never back-fill the non-marker base key: a later
+ // plain request would silently receive marker-mutated markdown.
+ expect(saveCached).toHaveBeenCalledOnce();
+ expect(saveCached).toHaveBeenCalledWith(
+ "BASE64",
+ expect.objectContaining({ markdown: marked }),
+ "firepdf",
+ "markers-v1",
+ );
+ });
+
+ it("back-fills enriched marker results within the marker family only", async () => {
+ const marked = "Page 1\n\n---\n\n<!-- page 2 -->\n\nPage 2";
+ const blocks = [
+ { page: 1, width: 800, height: 1100, status: "ok", items: [] },
+ ];
+
+ await maybeSaveResult({
+ meta: makeMeta(),
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: true,
+ pageMarkers: true,
+ result: {
+ markdown: marked,
+ html: "<p>marked</p>",
+ pagesProcessed: 2,
+ blocks,
+ },
+ });
+
+ expect(getCached).toHaveBeenCalledWith("BASE64", "firepdf", "markers-v1");
+ expect(saveCached).toHaveBeenCalledTimes(2);
+ expect(saveCached).toHaveBeenNthCalledWith(
+ 1,
+ "BASE64",
+ expect.objectContaining({ blocks }),
+ "firepdf",
+ "blocks-markers-v1",
+ );
+ expect(saveCached).toHaveBeenNthCalledWith(
+ 2,
+ "BASE64",
+ expect.not.objectContaining({ blocks: expect.anything() }),
+ "firepdf",
+ "markers-v1",
+ );
+ });
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts
index 5a12a4204..6e80a2c5f 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts
@@ -52,6 +52,15 @@ describe("PDF page-markdown URL index policy", () => {
},
}),
).toBe(false);
+ expect(
+ shouldUseIndex({
+ ...baseMeta,
+ options: {
+ ...baseMeta.options,
+ parsers: [{ type: "pdf", pageMarkers: true }],
+ },
+ }),
+ ).toBe(false);
} finally {
(
config as { FIRECRAWL_INDEX_WRITE_ONLY?: boolean }
@@ -116,4 +125,31 @@ describe("PDF page-markdown URL index policy", () => {
expect(result).toBe(document);
expect(result.metadata.indexId).toBeUndefined();
});
+
+ it("does not write marker-bearing markdown to the URL index", async () => {
+ const document = {
+ markdown: "Page 1\n\n---\n\n<!-- page 2 -->\n\nPage 2",
+ rawHtml: "<p>whole document</p>",
+ metadata: {
+ sourceURL: "https://example.com/file.pdf",
+ },
+ } as any;
+ const meta = {
+ url: "https://example.com/file.pdf",
+ winnerEngine: "pdf",
+ options: {
+ storeInCache: true,
+ parsers: [{ type: "pdf", pageMarkers: true }],
+ },
+ internalOptions: {
+ isParse: false,
+ zeroDataRetention: false,
+ },
+ } as any;
+
+ const result = await sendDocumentToIndex(meta, document);
+
+ expect(result).toBe(document);
+ expect(result.metadata.indexId).toBeUndefined();
+ });
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
index 3174dc88c..53e135bd6 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
@@ -39,6 +39,7 @@ export async function scrapePDFWithFirePDFAsync(
deps: FirePdfAsyncDeps = {},
includePageMarkdown = false,
includeBlocks = false,
+ pageMarkers = false,
): Promise<PDFProcessorResult> {
const fetchImpl = deps.fetchImpl ?? undiciFetch;
const fallbackImpl = deps.fallbackImpl ?? scrapePDFWithFirePDF;
@@ -57,6 +58,7 @@ export async function scrapePDFWithFirePDFAsync(
mode,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
}
@@ -68,6 +70,7 @@ export async function scrapePDFWithFirePDFAsync(
pagesProcessed,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
if (cached) return cached;
@@ -93,6 +96,7 @@ export async function scrapePDFWithFirePDFAsync(
mode,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
}
@@ -127,6 +131,7 @@ export async function scrapePDFWithFirePDFAsync(
mode,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
deadlineAt,
teamConcurrency,
fetchImpl,
@@ -192,6 +197,16 @@ export async function scrapePDFWithFirePDFAsync(
note: "FirePDF result omitted requested typed blocks",
});
}
+ if (pageMarkers && fetched.page_markers !== true) {
+ // Markers are baked into the markdown, so the missing echo is the only
+ // signal the worker build ignored the option; accepting the result
+ // would cache unmarked markdown under a marker cache variant. Fail the
+ // async attempt β€” the caller retries synchronously, where the same
+ // echo contract applies.
+ failAsync(meta, "http_5xx", {
+ note: "FirePDF result did not acknowledge requested page markers",
+ });
+ }
const durationMs = now() - overallStartedAt;
firePdfAsyncTotalDurationSeconds.observe(durationMs / 1000);
@@ -222,6 +237,7 @@ export async function scrapePDFWithFirePDFAsync(
maxPages,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
result: processorResult,
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
index 647aaa3ae..d62fc723e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
@@ -18,6 +18,19 @@ const OCR_BLOCKS_VARIANT = "ocr-blocks-v1";
const PAGE_MARKDOWN_BLOCKS_VARIANT = "page-markdown-blocks-v1";
const OCR_PAGE_MARKDOWN_BLOCKS_VARIANT = "ocr-page-markdown-blocks-v1";
+// `page_markers` rewrites the document markdown itself (inter-page
+// `<!-- page N -->` separators), unlike pages/blocks which are extra
+// payloads beside unchanged markdown. Marker and non-marker artifacts can
+// therefore never serve each other. Follow the `mode: ocr` dedicated-variant
+// precedent: map every variant name into a disjoint `…markers…` family.
+// Within that family the ocr/pages/blocks capability lattice applies
+// unchanged, because those artifacts differ only in sidecars again.
+function withPageMarkers(variant: string | undefined): string {
+ if (variant === undefined) return "markers-v1";
+ if (variant === "ocr") return "ocr-markers-v1";
+ return variant.replace(/-v1$/, "-markers-v1");
+}
+
function isValidCachedDocument(
value: unknown,
): value is Pick<PDFProcessorResult, "html"> & { markdown: string } {
@@ -67,6 +80,7 @@ export function cacheKeyShape(
maxPages: number | undefined,
includePageMarkdown: boolean,
includeBlocks: boolean,
+ pageMarkers = false,
) {
const cacheable = mode !== "fast" && !maxPages;
const isOcr = mode === "ocr";
@@ -116,6 +130,14 @@ export function cacheKeyShape(
: isOcr
? ["ocr", OCR_PAGE_MARKDOWN_VARIANT]
: [undefined, PAGE_MARKDOWN_VARIANT, "ocr", OCR_PAGE_MARKDOWN_VARIANT];
+ if (pageMarkers) {
+ return {
+ cacheable,
+ ownVariant: withPageMarkers(ownVariant),
+ baseVariant: withPageMarkers(baseVariant),
+ lookupVariants: lookupVariants.map(withPageMarkers),
+ };
+ }
return { cacheable, ownVariant, baseVariant, lookupVariants };
}
@@ -127,6 +149,7 @@ export async function tryGetCached(
pagesProcessed: number | undefined,
includePageMarkdown: boolean,
includeBlocks: boolean,
+ pageMarkers = false,
): Promise<PDFProcessorResult | null> {
if (meta.internalOptions.zeroDataRetention) return null;
const { cacheable, lookupVariants } = cacheKeyShape(
@@ -134,6 +157,7 @@ export async function tryGetCached(
maxPages,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
if (!cacheable) return null;
@@ -186,6 +210,7 @@ export async function maybeSaveResult(args: {
maxPages: number | undefined;
includePageMarkdown: boolean;
includeBlocks: boolean;
+ pageMarkers?: boolean;
result: PDFProcessorResult & { markdown: string };
}): Promise<void> {
const {
@@ -195,6 +220,7 @@ export async function maybeSaveResult(args: {
maxPages,
includePageMarkdown,
includeBlocks,
+ pageMarkers = false,
result,
} = args;
if (meta.internalOptions.zeroDataRetention) return;
@@ -203,6 +229,7 @@ export async function maybeSaveResult(args: {
maxPages,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
if (!cacheable) return;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
index 47f197e19..7e5bbf734 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
@@ -125,6 +125,11 @@ export const resultResponseSchema = z.object({
pages_processed: z.number().optional(),
failed_pages: z.array(z.number()).nullable().optional(),
partial_pages: z.array(z.number()).nullable().optional(),
+ // Echo of an honored pageMarkers job option. Markers are baked into
+ // `markdown` and their absence is not detectable by content, so the echo
+ // is the only proof the fire-pdf worker build understood the option β€”
+ // older workers ignore unknown option keys and omit it.
+ page_markers: z.literal(true).optional(),
});
export type PollResponse = z.infer<typeof pollResponseSchema>;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
index 968e0ad25..716cc4791 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
@@ -21,6 +21,7 @@ type SubmitArgs = {
mode: PDFMode | undefined;
includePageMarkdown: boolean;
includeBlocks: boolean;
+ pageMarkers: boolean;
deadlineAt: string;
/** Team's sold concurrency from the ACUC (ENG-5049 account context).
* Optional: entitlement lookup must never block or fail a scrape. */
@@ -62,6 +63,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
mode,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
deadlineAt,
teamConcurrency,
fetchImpl,
@@ -92,6 +94,12 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
...(mode !== undefined && { mode }),
...(includePageMarkdown && { include_page_markdown: true }),
...(includeBlocks && { include_blocks: true }),
+ // Intentionally camelCase, unlike its siblings: the fire-pdf async
+ // /jobs options schema named this key `pageMarkers` (fire-pdf
+ // api/src/http/schemas/jobs.ts) while the sync /ocr path uses
+ // `page_markers`. Sending snake_case here would be rejected as an
+ // unknown option.
+ ...(pageMarkers && { pageMarkers: true }),
},
};
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
index 3d9b11311..de9d45779 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
@@ -47,6 +47,7 @@ export async function scrapePDFWithFirePDF(
mode?: PDFMode,
includePageMarkdown = false,
includeBlocks = false,
+ pageMarkers = false,
): Promise<PDFProcessorResult> {
const logger = meta.logger;
@@ -62,6 +63,11 @@ export async function scrapePDFWithFirePDF(
// running fire-pdf again.
// - `fast` is bypassed entirely (hard cost ceiling β€” must fail on
// scanned PDFs, not serve a cached OCR result).
+ // - `page_markers` rewrites the document markdown itself (inter-page
+ // `<!-- page N -->` separators), so marker requests read/write a
+ // fully disjoint `…markers…` variant family β€” a base-variant entry
+ // must never be served for a marker request and vice versa. See
+ // cacheKeyShape in fire-pdf/cache.ts.
const cacheable =
mode !== "fast" && !maxPages && !meta.internalOptions.zeroDataRetention;
const cached = cacheable
@@ -73,6 +79,7 @@ export async function scrapePDFWithFirePDF(
pagesProcessed,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
)
: null;
if (cached) return cached;
@@ -124,6 +131,7 @@ export async function scrapePDFWithFirePDF(
...(mode !== undefined && { mode }),
...(includePageMarkdown && { include_page_markdown: true }),
...(includeBlocks && { include_blocks: true }),
+ ...(pageMarkers && { page_markers: true }),
// Enrichment for the fire-pdf jobs DB / dashboard. fire-pdf treats
// these as optional β€” older fire-pdf builds will ignore unknown fields.
team_id: meta.internalOptions.teamId,
@@ -143,6 +151,12 @@ export async function scrapePDFWithFirePDF(
pages_processed: z.number().optional(),
pages: firePdfPagesSchema,
blocks: firePdfBlocksSchema,
+ // Echo of an honored page_markers request. Markers are baked into
+ // `markdown` and their absence is not reliably detectable there (a
+ // single-page or fully-stitched document legitimately has none), so
+ // the echo is the only proof the fire-pdf build understood the
+ // option β€” older builds ignore unknown request fields and omit it.
+ page_markers: z.literal(true).optional(),
}),
mock: meta.mock,
abort: meta.abort.asSignal(),
@@ -157,6 +171,13 @@ export async function scrapePDFWithFirePDF(
if (includeBlocks && resp.blocks === undefined) {
throw new Error("FirePDF response did not include requested typed blocks");
}
+ if (pageMarkers && resp.page_markers !== true) {
+ // Without the echo, the markdown is ordinary unmarked output; caching
+ // it under a marker variant would silently poison the marker cache.
+ throw new Error(
+ "FirePDF response did not acknowledge requested page markers",
+ );
+ }
const pages = resp.pages_processed ?? pagesProcessed;
logger.info("FirePDF completed", {
@@ -185,6 +206,7 @@ export async function scrapePDFWithFirePDF(
maxPages,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
result: processorResult,
});
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index e40c2ae19..bc67579ea 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -20,6 +20,7 @@ import {
getPDFMode,
getPDFPageMarkdown,
getPDFBlocks,
+ getPDFPageMarkers,
getFirePdfAsync,
} from "../../../../controllers/v2/types";
import type { PDFMode } from "../../../../controllers/v2/types";
@@ -64,6 +65,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
const mode: PDFMode = getPDFMode(meta.options.parsers);
const includePageMarkdown = getPDFPageMarkdown(meta.options.parsers);
const includeBlocks = getPDFBlocks(meta.options.parsers);
+ const pageMarkers = getPDFPageMarkers(meta.options.parsers);
if (includePageMarkdown && !config.FIRE_PDF_BASE_URL) {
throw new Error(
@@ -77,6 +79,12 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
);
}
+ if (pageMarkers && !config.FIRE_PDF_BASE_URL) {
+ throw new Error(
+ "Page markers are unavailable because FirePDF is not configured",
+ );
+ }
+
if (!shouldParse) {
if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {
const content = (await readFile(meta.pdfPrefetch.filePath)).toString(
@@ -187,7 +195,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
let shadowPagesNeedingOcr: number[] | undefined;
const forceFirePDF =
- (!!meta.options.__forceFirePDF || includePageMarkdown || includeBlocks) &&
+ (!!meta.options.__forceFirePDF ||
+ includePageMarkdown ||
+ includeBlocks ||
+ pageMarkers) &&
!!config.FIRE_PDF_BASE_URL;
const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;
const logger = meta.logger.child({ method: "scrapePDF/processPdf" });
@@ -471,17 +482,18 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
undefined,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
} catch (error) {
if (
- (!includePageMarkdown && !includeBlocks) ||
+ (!includePageMarkdown && !includeBlocks && !pageMarkers) ||
error instanceof RemoveFeatureError ||
error instanceof AbortManagerThrownError
) {
throw error;
}
meta.logger.warn(
- "FirePDF async page markdown/blocks failed -- retrying synchronously",
+ "FirePDF async page markdown/blocks/markers failed -- retrying synchronously",
{
method: "scrapePDF/firePDFFallback",
error,
@@ -502,6 +514,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
mode,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
}
} else {
@@ -513,6 +526,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
mode,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
}
effectivePageCount = reconcilePageCountWithFirePdf(