Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v2): pageMarkers PDF parser option β per-page page attribution in document.markdown (#4365)
Abimael Martell24b811264004
diff --git a/apps/api/src/__tests__/pdf-parser-options.test.ts b/apps/api/src/__tests__/pdf-parser-options.test.tsindex f891efe6e..d3f783f71 100644--- a/apps/api/src/__tests__/pdf-parser-options.test.ts+++ b/apps/api/src/__tests__/pdf-parser-options.test.ts@@ -1,6 +1,7 @@import {getPDFBlocks,getPDFPageMarkdown,+ getPDFPageMarkers,scrapeOptions,} from "../controllers/v2/types";@@ -23,6 +24,27 @@ describe("PDF parser option getters", () => {expect(getPDFBlocks([{ type: "pdf" }])).toBe(false);expect(getPDFBlocks(undefined)).toBe(false);});++ it("reads the `pageMarkers` option", () => {+ expect(getPDFPageMarkers([{ type: "pdf", pageMarkers: true }])).toBe(true);+ expect(getPDFPageMarkers([{ type: "pdf", pageMarkers: false }])).toBe(+ false,+ );+ expect(getPDFPageMarkers([{ type: "pdf" }])).toBe(false);+ expect(getPDFPageMarkers(["pdf"])).toBe(false);+ expect(getPDFPageMarkers(undefined)).toBe(false);+ });++ it("keeps `pageMarkers` flowing through the alias-normalizing transform", () => {+ const parser = parsePdfParser({+ type: "pdf",+ pageMarkers: true,+ pageMarkdown: true,+ });+ expect(parser.pageMarkers).toBe(true);+ expect(parser.pages).toBe(true);+ expect(getPDFPageMarkers([parser as any])).toBe(true);+ });});describe("deprecated pageMarkdown alias", () => {diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 54233c799..496d00384 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -497,6 +497,13 @@ const pdfParserWithOptions = z* reading order) alongside document markdown β populates* `document.blocks`. */blocks: z.boolean().optional(),+ /** Join PDF pages in `document.markdown` with+ * `\n\n---\n\n<!-- page N -->\n\n` where N is the 1-based physical page+ * of the content that follows. Markers appear between pages only (no+ * leading marker for page 1), and numbering may skip pages merged by+ * cross-page stitching β callers that need every physical page should+ * use `pages: true` instead. No new response field. */+ pageMarkers: z.boolean().optional(),// Experimental: route this request through the fire-pdf async pipeline// (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back// to sync on any async-path failure, so user-visible behavior is unchanged@@ -583,6 +590,16 @@ export function getPDFBlocks(parsers?: Parsers): boolean {return false;}+export function getPDFPageMarkers(parsers?: Parsers): boolean {+ if (!parsers) return false;+ for (const parser of parsers) {+ if (typeof parser === "object" && parser.type === "pdf") {+ return parser.pageMarkers === true;+ }+ }+ return false;+}+export function getFirePdfAsync(parsers?: Parsers): boolean {if (!parsers) return false;for (const parser of parsers) {diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 315e68eda..040180633 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -30,6 +30,7 @@ import { hasFormatOfType } from "../../../lib/format-utils";import {getPDFBlocks,getPDFPageMarkdown,+ getPDFPageMarkers,} from "../../../controllers/v2/types";import type { PdfMetadata, PdfPageBlocks } from "./pdf/types";import { BrandingProfile } from "../../../types/branding";@@ -567,9 +568,10 @@ export function shouldUseIndex(meta: Meta) {!hasFormatOfType(meta.options.formats, "changeTracking") &&!hasFormatOfType(meta.options.formats, "branding") &&// The URL index does not yet persist physical-page or typed-block- // capability metadata.+ // capability metadata, and its markdown never carries page markers.!getPDFPageMarkdown(meta.options.parsers) &&!getPDFBlocks(meta.options.parsers) &&+ !getPDFPageMarkers(meta.options.parsers) &&!hasCustomScreenshotSettings &&meta.options.maxAge !== 0 &&(meta.options.headers === undefined ||@@ -680,17 +682,19 @@ export async function buildFallbackList(meta: Meta): Promise<_engines.push(...indexEngines);meta.internalOptions.forceEngine = indexEngines;} else if (meta.internalOptions.agentIndexOnly) {- // Index documents carry no physical-page or typed-block payloads, so an- // index-only request that demands them can only be answered wrong. Fail- // loud with the canonical index-only error (maps to a clean 4xx and- // tells the caller how to unlock live scraping) instead of silently- // serving a document without the capability.+ // Index documents carry no physical-page or typed-block payloads, and+ // their markdown never carries page markers, so an index-only request+ // that demands them can only be answered wrong. Fail loud with the+ // canonical index-only error (maps to a clean 4xx and tells the caller+ // how to unlock live scraping) instead of silently serving a document+ // without the capability.if (getPDFPageMarkdown(meta.options.parsers) ||- getPDFBlocks(meta.options.parsers)+ getPDFBlocks(meta.options.parsers) ||+ getPDFPageMarkers(meta.options.parsers)) {meta.logger.warn(- "agentIndexOnly request demands pageMarkdown/blocks, which the URL index cannot serve",+ "agentIndexOnly request demands pageMarkdown/blocks/pageMarkers, which the URL index cannot serve",{ parsers: meta.options.parsers },);throw new AgentIndexOnlyError();diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex e2f9596b4..223a297c8 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -40,6 +40,7 @@ import {getPDFBlocks,getPDFMaxPages,getPDFPageMarkdown,+ getPDFPageMarkers,shouldParsePDF,} from "../../../../controllers/v2/types";import { hasFormatOfType } from "../../../../lib/format-utils";@@ -64,8 +65,10 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {// Page-aware and block-aware results are capability-specific and are not// represented in the URL index schema yet. Do not write an entry that// could later be served without its required pages/blocks payload.+ // Marker-bearing markdown is mutated output β never index it either.!getPDFPageMarkdown(meta.options.parsers) &&!getPDFBlocks(meta.options.parsers) &&+ !getPDFPageMarkers(meta.options.parsers) &&!meta.options.parsers?.some(parser => {if (typeof parser === "object" &&diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.tsindex 6ad00666c..4acd9f504 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts@@ -211,3 +211,105 @@ describe("scrapePDFWithFirePDF typed blocks", () => {).rejects.toThrow(/did not include requested typed blocks/);});});++describe("scrapePDFWithFirePDF page markers", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("requests marker-joined markdown and returns it verbatim", async () => {+ const marked = "Page 1\n\n---\n\n<!-- page 2 -->\n\nPage 2";+ mockedRobustFetch.mockResolvedValue({+ markdown: marked,+ failed_pages: null,+ pages_processed: 2,+ page_markers: true,+ } as any);++ const result = await scrapePDFWithFirePDF(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ "auto",+ false,+ false,+ true,+ );++ expect(mockedRobustFetch).toHaveBeenCalledWith(+ expect.objectContaining({+ body: expect.objectContaining({ page_markers: true }),+ }),+ );+ expect(result.markdown).toBe(marked);+ });++ it("does not send page_markers for plain requests", async () => {+ mockedRobustFetch.mockResolvedValue({+ markdown: "plain",+ failed_pages: null,+ pages_processed: 1,+ } as any);++ await scrapePDFWithFirePDF(makeMeta(), "BASE64");++ expect(+ ((mockedRobustFetch.mock.calls[0][0] as any).body as any).page_markers,+ ).toBeUndefined();+ });++ it("rejects FirePDF responses that do not acknowledge page markers", async () => {+ // An old fire-pdf build ignores the unknown `page_markers` request field+ // and returns ordinary markdown with no echo β indistinguishable from+ // marked output by content alone, so the missing echo must fail loud.+ mockedRobustFetch.mockResolvedValue({+ markdown: "Page 1\n\n---\n\nPage 2",+ failed_pages: null,+ pages_processed: 2,+ } as any);++ await expect(+ scrapePDFWithFirePDF(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ "auto",+ false,+ false,+ true,+ ),+ ).rejects.toThrow(/did not acknowledge requested page markers/);+ });++ it("composes page_markers with include_blocks on the wire", async () => {+ mockedRobustFetch.mockResolvedValue({+ markdown: "Page 1\n\n---\n\n<!-- page 2 -->\n\nPage 2",+ failed_pages: null,+ pages_processed: 2,+ blocks: [],+ page_markers: true,+ } as any);++ await scrapePDFWithFirePDF(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ "auto",+ false,+ true,+ true,+ );++ expect(mockedRobustFetch).toHaveBeenCalledWith(+ expect.objectContaining({+ body: expect.objectContaining({+ include_blocks: true,+ page_markers: true,+ }),+ }),+ );+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.tsindex 842d0d035..03d351129 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts@@ -348,6 +348,99 @@ describe("scrapePDFWithFirePDFAsync", () => {expect(error.reason).toBe("http_5xx");});+ it("requests page markers and returns the acknowledged marked markdown", async () => {+ const marked = "Page 1\n\n---\n\n<!-- page 2 -->\n\nPage 2";+ const { fetchImpl, calls } = makeFetchFromSequence([+ {+ matchUrl: /\/jobs$/,+ matchMethod: "POST",+ response: {+ status: 200,+ body: {+ scrape_id: "scrape-id-test",+ status: "done",+ lane: "fast",+ },+ },+ },+ {+ matchUrl: /\/jobs\/scrape-id-test\/result$/,+ matchMethod: "GET",+ response: {+ status: 200,+ body: {+ schema_version: 1,+ markdown: marked,+ pages_processed: 2,+ page_markers: true,+ },+ },+ },+ ]);++ const result = await scrapePDFWithFirePDFAsync(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ "auto",+ { fetchImpl, sleepImpl: noopSleep },+ false,+ false,+ true,+ );++ expect((calls[0].body as any).options.pageMarkers).toBe(true);+ expect(result.markdown).toBe(marked);+ });++ it("fails a marker request when the result lacks the page_markers echo", async () => {+ // An older worker ignores the unknown pageMarkers option and persists+ // ordinary markdown; without the echo this must fail (and fall back to+ // the sync path) rather than cache unmarked markdown as marked output.+ const { fetchImpl } = makeFetchFromSequence([+ {+ matchUrl: /\/jobs$/,+ matchMethod: "POST",+ response: {+ status: 200,+ body: {+ scrape_id: "scrape-id-test",+ status: "done",+ lane: "fast",+ },+ },+ },+ {+ matchUrl: /\/jobs\/scrape-id-test\/result$/,+ matchMethod: "GET",+ response: {+ status: 200,+ body: {+ schema_version: 1,+ markdown: "Page 1\n\n---\n\nPage 2",+ pages_processed: 2,+ },+ },+ },+ ]);++ const error = await scrapePDFWithFirePDFAsync(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ "auto",+ { fetchImpl, sleepImpl: noopSleep },+ false,+ false,+ true,+ ).catch(error => error);++ expect(error).toBeInstanceOf(FirePdfAsyncFailure);+ expect(error.reason).toBe("http_5xx");+ });+it("requests and returns typed blocks, tolerating the legacy pages alias", async () => {const blocks = [{diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.tsindex d697b3831..24c88fcd7 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts@@ -82,6 +82,43 @@ describe("FirePDF page-markdown cache capabilities", () => {expect(cacheKeyShape("auto", 5, false, true).cacheable).toBe(false);});+ it("maps page-marker requests into a fully disjoint variant family", () => {+ // pageMarkers rewrites the markdown itself, so no marker lookup may ever+ // name a non-marker variant (and vice versa β see the plain-request test:+ // its probe list is unchanged and marker-free).+ expect(cacheKeyShape("auto", undefined, false, false, true)).toMatchObject({+ cacheable: true,+ ownVariant: "markers-v1",+ baseVariant: "markers-v1",+ lookupVariants: [+ "markers-v1",+ "page-markdown-markers-v1",+ "ocr-markers-v1",+ "ocr-page-markdown-markers-v1",+ ],+ });+ expect(cacheKeyShape("ocr", undefined, false, false, true)).toMatchObject({+ cacheable: true,+ ownVariant: "ocr-markers-v1",+ lookupVariants: ["ocr-markers-v1", "ocr-page-markdown-markers-v1"],+ });+ expect(cacheKeyShape("auto", undefined, false, true, true)).toMatchObject({+ cacheable: true,+ ownVariant: "blocks-markers-v1",+ baseVariant: "markers-v1",+ lookupVariants: [+ "blocks-markers-v1",+ "page-markdown-blocks-markers-v1",+ "ocr-blocks-markers-v1",+ "ocr-page-markdown-blocks-markers-v1",+ ],+ });+ expect(cacheKeyShape("fast", undefined, false, false, true).cacheable).toBe(+ false,+ );+ expect(cacheKeyShape("auto", 5, false, false, true).cacheable).toBe(false);+ });+it("keeps the historical probe list for plain requests", () => {expect(cacheKeyShape("auto", undefined, false, false)).toMatchObject({cacheable: true,@@ -504,4 +541,73 @@ describe("FirePDF page-markdown cache capabilities", () => {undefined,);});++ it("saves marker results only under marker variants (never the base key)", async () => {+ const marked = "Page 1\n\n---\n\n<!-- page 2 -->\n\nPage 2";++ await maybeSaveResult({+ meta: makeMeta(),+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ pageMarkers: true,+ result: {+ markdown: marked,+ html: "<p>marked</p>",+ pagesProcessed: 2,+ },+ });++ // Marker markdown must never back-fill the non-marker base key: a later+ // plain request would silently receive marker-mutated markdown.+ expect(saveCached).toHaveBeenCalledOnce();+ expect(saveCached).toHaveBeenCalledWith(+ "BASE64",+ expect.objectContaining({ markdown: marked }),+ "firepdf",+ "markers-v1",+ );+ });++ it("back-fills enriched marker results within the marker family only", async () => {+ const marked = "Page 1\n\n---\n\n<!-- page 2 -->\n\nPage 2";+ const blocks = [+ { page: 1, width: 800, height: 1100, status: "ok", items: [] },+ ];++ await maybeSaveResult({+ meta: makeMeta(),+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: true,+ pageMarkers: true,+ result: {+ markdown: marked,+ html: "<p>marked</p>",+ pagesProcessed: 2,+ blocks,+ },+ });++ expect(getCached).toHaveBeenCalledWith("BASE64", "firepdf", "markers-v1");+ expect(saveCached).toHaveBeenCalledTimes(2);+ expect(saveCached).toHaveBeenNthCalledWith(+ 1,+ "BASE64",+ expect.objectContaining({ blocks }),+ "firepdf",+ "blocks-markers-v1",+ );+ expect(saveCached).toHaveBeenNthCalledWith(+ 2,+ "BASE64",+ expect.not.objectContaining({ blocks: expect.anything() }),+ "firepdf",+ "markers-v1",+ );+ });});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.tsindex 5a12a4204..6e80a2c5f 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts@@ -52,6 +52,15 @@ describe("PDF page-markdown URL index policy", () => {},}),).toBe(false);+ expect(+ shouldUseIndex({+ ...baseMeta,+ options: {+ ...baseMeta.options,+ parsers: [{ type: "pdf", pageMarkers: true }],+ },+ }),+ ).toBe(false);} finally {(config as { FIRECRAWL_INDEX_WRITE_ONLY?: boolean }@@ -116,4 +125,31 @@ describe("PDF page-markdown URL index policy", () => {expect(result).toBe(document);expect(result.metadata.indexId).toBeUndefined();});++ it("does not write marker-bearing markdown to the URL index", async () => {+ const document = {+ markdown: "Page 1\n\n---\n\n<!-- page 2 -->\n\nPage 2",+ rawHtml: "<p>whole document</p>",+ metadata: {+ sourceURL: "https://example.com/file.pdf",+ },+ } as any;+ const meta = {+ url: "https://example.com/file.pdf",+ winnerEngine: "pdf",+ options: {+ storeInCache: true,+ parsers: [{ type: "pdf", pageMarkers: true }],+ },+ internalOptions: {+ isParse: false,+ zeroDataRetention: false,+ },+ } as any;++ const result = await sendDocumentToIndex(meta, document);++ expect(result).toBe(document);+ expect(result.metadata.indexId).toBeUndefined();+ });});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.tsindex 3174dc88c..53e135bd6 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts@@ -39,6 +39,7 @@ export async function scrapePDFWithFirePDFAsync(deps: FirePdfAsyncDeps = {},includePageMarkdown = false,includeBlocks = false,+ pageMarkers = false,): Promise<PDFProcessorResult> {const fetchImpl = deps.fetchImpl ?? undiciFetch;const fallbackImpl = deps.fallbackImpl ?? scrapePDFWithFirePDF;@@ -57,6 +58,7 @@ export async function scrapePDFWithFirePDFAsync(mode,includePageMarkdown,includeBlocks,+ pageMarkers,);}@@ -68,6 +70,7 @@ export async function scrapePDFWithFirePDFAsync(pagesProcessed,includePageMarkdown,includeBlocks,+ pageMarkers,);if (cached) return cached;@@ -93,6 +96,7 @@ export async function scrapePDFWithFirePDFAsync(mode,includePageMarkdown,includeBlocks,+ pageMarkers,);}@@ -127,6 +131,7 @@ export async function scrapePDFWithFirePDFAsync(mode,includePageMarkdown,includeBlocks,+ pageMarkers,deadlineAt,teamConcurrency,fetchImpl,@@ -192,6 +197,16 @@ export async function scrapePDFWithFirePDFAsync(note: "FirePDF result omitted requested typed blocks",});}+ if (pageMarkers && fetched.page_markers !== true) {+ // Markers are baked into the markdown, so the missing echo is the only+ // signal the worker build ignored the option; accepting the result+ // would cache unmarked markdown under a marker cache variant. Fail the+ // async attempt β the caller retries synchronously, where the same+ // echo contract applies.+ failAsync(meta, "http_5xx", {+ note: "FirePDF result did not acknowledge requested page markers",+ });+ }const durationMs = now() - overallStartedAt;firePdfAsyncTotalDurationSeconds.observe(durationMs / 1000);@@ -222,6 +237,7 @@ export async function scrapePDFWithFirePDFAsync(maxPages,includePageMarkdown,includeBlocks,+ pageMarkers,result: processorResult,});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.tsindex 647aaa3ae..d62fc723e 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts@@ -18,6 +18,19 @@ const OCR_BLOCKS_VARIANT = "ocr-blocks-v1";const PAGE_MARKDOWN_BLOCKS_VARIANT = "page-markdown-blocks-v1";const OCR_PAGE_MARKDOWN_BLOCKS_VARIANT = "ocr-page-markdown-blocks-v1";+// `page_markers` rewrites the document markdown itself (inter-page+// `<!-- page N -->` separators), unlike pages/blocks which are extra+// payloads beside unchanged markdown. Marker and non-marker artifacts can+// therefore never serve each other. Follow the `mode: ocr` dedicated-variant+// precedent: map every variant name into a disjoint `β¦markersβ¦` family.+// Within that family the ocr/pages/blocks capability lattice applies+// unchanged, because those artifacts differ only in sidecars again.+function withPageMarkers(variant: string | undefined): string {+ if (variant === undefined) return "markers-v1";+ if (variant === "ocr") return "ocr-markers-v1";+ return variant.replace(/-v1$/, "-markers-v1");+}+function isValidCachedDocument(value: unknown,): value is Pick<PDFProcessorResult, "html"> & { markdown: string } {@@ -67,6 +80,7 @@ export function cacheKeyShape(maxPages: number | undefined,includePageMarkdown: boolean,includeBlocks: boolean,+ pageMarkers = false,) {const cacheable = mode !== "fast" && !maxPages;const isOcr = mode === "ocr";@@ -116,6 +130,14 @@ export function cacheKeyShape(: isOcr? ["ocr", OCR_PAGE_MARKDOWN_VARIANT]: [undefined, PAGE_MARKDOWN_VARIANT, "ocr", OCR_PAGE_MARKDOWN_VARIANT];+ if (pageMarkers) {+ return {+ cacheable,+ ownVariant: withPageMarkers(ownVariant),+ baseVariant: withPageMarkers(baseVariant),+ lookupVariants: lookupVariants.map(withPageMarkers),+ };+ }return { cacheable, ownVariant, baseVariant, lookupVariants };}@@ -127,6 +149,7 @@ export async function tryGetCached(pagesProcessed: number | undefined,includePageMarkdown: boolean,includeBlocks: boolean,+ pageMarkers = false,): Promise<PDFProcessorResult | null> {if (meta.internalOptions.zeroDataRetention) return null;const { cacheable, lookupVariants } = cacheKeyShape(@@ -134,6 +157,7 @@ export async function tryGetCached(maxPages,includePageMarkdown,includeBlocks,+ pageMarkers,);if (!cacheable) return null;@@ -186,6 +210,7 @@ export async function maybeSaveResult(args: {maxPages: number | undefined;includePageMarkdown: boolean;includeBlocks: boolean;+ pageMarkers?: boolean;result: PDFProcessorResult & { markdown: string };}): Promise<void> {const {@@ -195,6 +220,7 @@ export async function maybeSaveResult(args: {maxPages,includePageMarkdown,includeBlocks,+ pageMarkers = false,result,} = args;if (meta.internalOptions.zeroDataRetention) return;@@ -203,6 +229,7 @@ export async function maybeSaveResult(args: {maxPages,includePageMarkdown,includeBlocks,+ pageMarkers,);if (!cacheable) return;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.tsindex 47f197e19..7e5bbf734 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts@@ -125,6 +125,11 @@ export const resultResponseSchema = z.object({pages_processed: z.number().optional(),failed_pages: z.array(z.number()).nullable().optional(),partial_pages: z.array(z.number()).nullable().optional(),+ // Echo of an honored pageMarkers job option. Markers are baked into+ // `markdown` and their absence is not detectable by content, so the echo+ // is the only proof the fire-pdf worker build understood the option β+ // older workers ignore unknown option keys and omit it.+ page_markers: z.literal(true).optional(),});export type PollResponse = z.infer<typeof pollResponseSchema>;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.tsindex 968e0ad25..716cc4791 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts@@ -21,6 +21,7 @@ type SubmitArgs = {mode: PDFMode | undefined;includePageMarkdown: boolean;includeBlocks: boolean;+ pageMarkers: boolean;deadlineAt: string;/** Team's sold concurrency from the ACUC (ENG-5049 account context).* Optional: entitlement lookup must never block or fail a scrape. */@@ -62,6 +63,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {mode,includePageMarkdown,includeBlocks,+ pageMarkers,deadlineAt,teamConcurrency,fetchImpl,@@ -92,6 +94,12 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {...(mode !== undefined && { mode }),...(includePageMarkdown && { include_page_markdown: true }),...(includeBlocks && { include_blocks: true }),+ // Intentionally camelCase, unlike its siblings: the fire-pdf async+ // /jobs options schema named this key `pageMarkers` (fire-pdf+ // api/src/http/schemas/jobs.ts) while the sync /ocr path uses+ // `page_markers`. Sending snake_case here would be rejected as an+ // unknown option.+ ...(pageMarkers && { pageMarkers: true }),},};diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsindex 3d9b11311..de9d45779 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -47,6 +47,7 @@ export async function scrapePDFWithFirePDF(mode?: PDFMode,includePageMarkdown = false,includeBlocks = false,+ pageMarkers = false,): Promise<PDFProcessorResult> {const logger = meta.logger;@@ -62,6 +63,11 @@ export async function scrapePDFWithFirePDF(// running fire-pdf again.// - `fast` is bypassed entirely (hard cost ceiling β must fail on// scanned PDFs, not serve a cached OCR result).+ // - `page_markers` rewrites the document markdown itself (inter-page+ // `<!-- page N -->` separators), so marker requests read/write a+ // fully disjoint `β¦markersβ¦` variant family β a base-variant entry+ // must never be served for a marker request and vice versa. See+ // cacheKeyShape in fire-pdf/cache.ts.const cacheable =mode !== "fast" && !maxPages && !meta.internalOptions.zeroDataRetention;const cached = cacheable@@ -73,6 +79,7 @@ export async function scrapePDFWithFirePDF(pagesProcessed,includePageMarkdown,includeBlocks,+ pageMarkers,): null;if (cached) return cached;@@ -124,6 +131,7 @@ export async function scrapePDFWithFirePDF(...(mode !== undefined && { mode }),...(includePageMarkdown && { include_page_markdown: true }),...(includeBlocks && { include_blocks: true }),+ ...(pageMarkers && { page_markers: true }),// Enrichment for the fire-pdf jobs DB / dashboard. fire-pdf treats// these as optional β older fire-pdf builds will ignore unknown fields.team_id: meta.internalOptions.teamId,@@ -143,6 +151,12 @@ export async function scrapePDFWithFirePDF(pages_processed: z.number().optional(),pages: firePdfPagesSchema,blocks: firePdfBlocksSchema,+ // Echo of an honored page_markers request. Markers are baked into+ // `markdown` and their absence is not reliably detectable there (a+ // single-page or fully-stitched document legitimately has none), so+ // the echo is the only proof the fire-pdf build understood the+ // option β older builds ignore unknown request fields and omit it.+ page_markers: z.literal(true).optional(),}),mock: meta.mock,abort: meta.abort.asSignal(),@@ -157,6 +171,13 @@ export async function scrapePDFWithFirePDF(if (includeBlocks && resp.blocks === undefined) {throw new Error("FirePDF response did not include requested typed blocks");}+ if (pageMarkers && resp.page_markers !== true) {+ // Without the echo, the markdown is ordinary unmarked output; caching+ // it under a marker variant would silently poison the marker cache.+ throw new Error(+ "FirePDF response did not acknowledge requested page markers",+ );+ }const pages = resp.pages_processed ?? pagesProcessed;logger.info("FirePDF completed", {@@ -185,6 +206,7 @@ export async function scrapePDFWithFirePDF(maxPages,includePageMarkdown,includeBlocks,+ pageMarkers,result: processorResult,});}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex e40c2ae19..bc67579ea 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -20,6 +20,7 @@ import {getPDFMode,getPDFPageMarkdown,getPDFBlocks,+ getPDFPageMarkers,getFirePdfAsync,} from "../../../../controllers/v2/types";import type { PDFMode } from "../../../../controllers/v2/types";@@ -64,6 +65,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {const mode: PDFMode = getPDFMode(meta.options.parsers);const includePageMarkdown = getPDFPageMarkdown(meta.options.parsers);const includeBlocks = getPDFBlocks(meta.options.parsers);+ const pageMarkers = getPDFPageMarkers(meta.options.parsers);if (includePageMarkdown && !config.FIRE_PDF_BASE_URL) {throw new Error(@@ -77,6 +79,12 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {);}+ if (pageMarkers && !config.FIRE_PDF_BASE_URL) {+ throw new Error(+ "Page markers are unavailable because FirePDF is not configured",+ );+ }+if (!shouldParse) {if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {const content = (await readFile(meta.pdfPrefetch.filePath)).toString(@@ -187,7 +195,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {let shadowPagesNeedingOcr: number[] | undefined;const forceFirePDF =- (!!meta.options.__forceFirePDF || includePageMarkdown || includeBlocks) &&+ (!!meta.options.__forceFirePDF ||+ includePageMarkdown ||+ includeBlocks ||+ pageMarkers) &&!!config.FIRE_PDF_BASE_URL;const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;const logger = meta.logger.child({ method: "scrapePDF/processPdf" });@@ -471,17 +482,18 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {undefined,includePageMarkdown,includeBlocks,+ pageMarkers,);} catch (error) {if (- (!includePageMarkdown && !includeBlocks) ||+ (!includePageMarkdown && !includeBlocks && !pageMarkers) ||error instanceof RemoveFeatureError ||error instanceof AbortManagerThrownError) {throw error;}meta.logger.warn(- "FirePDF async page markdown/blocks failed -- retrying synchronously",+ "FirePDF async page markdown/blocks/markers failed -- retrying synchronously",{method: "scrapePDF/firePDFFallback",error,@@ -502,6 +514,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {mode,includePageMarkdown,includeBlocks,+ pageMarkers,);}} else {@@ -513,6 +526,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {mode,includePageMarkdown,includeBlocks,+ pageMarkers,);}effectivePageCount = reconcilePageCountWithFirePdf(