Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): expose fire-pdf typed blocks via parsers[].blocks (#4347)
Abimael Martell64eb4e1415a1
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex f5821ed7c..b8844ece7 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -4,6 +4,7 @@ import { z } from "zod";import { protocolIncluded, checkUrl } from "../../lib/validateUrl";import { hasReachableHost } from "../../lib/url-utils";import { countries } from "../../lib/validate-country";+import type { PdfPageBlocks } from "../../scraper/scrapeURL/engines/pdf/types";import {ExtractorOptions,PageOptions,@@ -1007,6 +1008,9 @@ export type Document = {markdown?: string;/** Physical PDF pages, populated by the v2 pageMarkdown parser option. */pages?: Array<{ pageNumber: number; markdown: string }>;+ /** Typed PDF layout blocks with bounding boxes, populated by the v2+ * `blocks` parser option. */+ blocks?: PdfPageBlocks[];html?: string;rawHtml?: string;links?: string[];diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 49503bc06..7eba70eb3 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -18,6 +18,7 @@ import {ScrapeOptions as V1ScrapeOptions,} from "../v1/types";import type { InternalOptions } from "../../scraper/scrapeURL";+import type { PdfPageBlocks } from "../../scraper/scrapeURL/engines/pdf/types";import { ErrorCodes } from "../../lib/error";import Ajv from "ajv";import addFormats from "ajv-formats";@@ -484,6 +485,9 @@ const pdfParserWithOptions = z.strictObject({maxPages: z.int().positive().finite().max(10000).optional(),/** Include physical per-page markdown alongside document markdown. */pageMarkdown: z.boolean().optional(),+ /** Include per-page typed layout blocks (bounding boxes, block types,+ * reading order) alongside document markdown. */+ blocks: z.boolean().optional(),// Experimental: route this request through the fire-pdf async pipeline// (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back// to sync on any async-path failure, so user-visible behavior is unchanged@@ -543,6 +547,16 @@ export function getPDFPageMarkdown(parsers?: Parsers): boolean {return false;}+export function getPDFBlocks(parsers?: Parsers): boolean {+ if (!parsers) return false;+ for (const parser of parsers) {+ if (typeof parser === "object" && parser.type === "pdf") {+ return parser.blocks === true;+ }+ }+ return false;+}+export function getFirePdfAsync(parsers?: Parsers): boolean {if (!parsers) return false;for (const parser of parsers) {@@ -1242,6 +1256,9 @@ export type Document = {markdown?: string;/** Physical PDF pages, present only for `parsers[].pageMarkdown`. */pages?: Array<{ pageNumber: number; markdown: string }>;+ /** Typed PDF layout blocks with bounding boxes, present only for+ * `parsers[].blocks`. */+ blocks?: PdfPageBlocks[];html?: string;rawHtml?: string;links?: string[];diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.tsindex 4f89de8c5..6cb865109 100644--- a/apps/api/src/lib/gcs-pdf-cache.ts+++ b/apps/api/src/lib/gcs-pdf-cache.ts@@ -1,4 +1,5 @@import { ApiError } from "@google-cloud/storage";+import type { FirePdfPageBlocks } from "../scraper/scrapeURL/engines/pdf/types";import { logger } from "./logger";import { config } from "../config";import crypto from "crypto";@@ -15,6 +16,9 @@ type CachedPdfResult = {pagesProcessed?: number;/** Physical page markdown; present only in page-capable cache variants. */pageMarkdown?: Array<{ page: number; markdown: string }>;+ /** Typed layout blocks (fire-pdf wire shape); present only in+ * block-capable cache variants. */+ blocks?: FirePdfPageBlocks[];};const PROVIDER_PREFIXES: Record<PdfCacheProvider, string> = {diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 224efaf76..315e68eda 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -27,10 +27,13 @@ import {} from "./x-twitter";import { queryEngpickerVerdict, useIndex } from "../../../services";import { hasFormatOfType } from "../../../lib/format-utils";-import { getPDFPageMarkdown } from "../../../controllers/v2/types";-import type { PdfMetadata } from "./pdf/types";+import {+ getPDFBlocks,+ getPDFPageMarkdown,+} from "../../../controllers/v2/types";+import type { PdfMetadata, PdfPageBlocks } from "./pdf/types";import { BrandingProfile } from "../../../types/branding";-import { BrandingNotSupportedError } from "../error";+import { AgentIndexOnlyError, BrandingNotSupportedError } from "../error";import { isUrlBlocked } from "../../WebScraper/utils/blocklist";import {canUseExchangeForRequest,@@ -139,6 +142,7 @@ export type EngineScrapeResult = {html: string;markdown?: string;pages?: Array<{ pageNumber: number; markdown: string }>;+ blocks?: PdfPageBlocks[];json?: unknown;statusCode: number;error?: string;@@ -562,8 +566,10 @@ export function shouldUseIndex(meta: Meta) {config.FIRECRAWL_INDEX_WRITE_ONLY !== true &&!hasFormatOfType(meta.options.formats, "changeTracking") &&!hasFormatOfType(meta.options.formats, "branding") &&- // The URL index does not yet persist physical-page capability metadata.+ // The URL index does not yet persist physical-page or typed-block+ // capability metadata.!getPDFPageMarkdown(meta.options.parsers) &&+ !getPDFBlocks(meta.options.parsers) &&!hasCustomScreenshotSettings &&meta.options.maxAge !== 0 &&(meta.options.headers === undefined ||@@ -674,6 +680,21 @@ export async function buildFallbackList(meta: Meta): Promise<_engines.push(...indexEngines);meta.internalOptions.forceEngine = indexEngines;} else if (meta.internalOptions.agentIndexOnly) {+ // Index documents carry no physical-page or typed-block payloads, so an+ // index-only request that demands them can only be answered wrong. Fail+ // loud with the canonical index-only error (maps to a clean 4xx and+ // tells the caller how to unlock live scraping) instead of silently+ // serving a document without the capability.+ if (+ getPDFPageMarkdown(meta.options.parsers) ||+ getPDFBlocks(meta.options.parsers)+ ) {+ meta.logger.warn(+ "agentIndexOnly request demands pageMarkdown/blocks, which the URL index cannot serve",+ { parsers: meta.options.parsers },+ );+ throw new AgentIndexOnlyError();+ }const indexEngines: Engine[] = useIndex ? ["index", "index;documents"] : [];_engines.length = 0;_engines.push(...indexEngines);diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 08f2ba413..e2f9596b4 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -37,6 +37,7 @@ import {NoCachedDataError,} from "../../error";import {+ getPDFBlocks,getPDFMaxPages,getPDFPageMarkdown,shouldParsePDF,@@ -60,10 +61,11 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {// every access must go through the Exchange and its ledger.meta.winnerEngine !== "exchange" &&!(meta.winnerEngine === "pdf" && !shouldParsePDF(meta.options.parsers)) &&- // Page-aware results are capability-specific and are not represented in- // the URL index schema yet. Do not write an entry that could later be- // served without its required pages payload.+ // Page-aware and block-aware results are capability-specific and are not+ // represented in the URL index schema yet. Do not write an entry that+ // could later be served without its required pages/blocks payload.!getPDFPageMarkdown(meta.options.parsers) &&+ !getPDFBlocks(meta.options.parsers) &&!meta.options.parsers?.some(parser => {if (typeof parser === "object" &&diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.tsindex d41be0cfa..6ad00666c 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts@@ -133,3 +133,81 @@ describe("scrapePDFWithFirePDF page markdown", () => {).rejects.toThrow(/did not include requested physical page markdown/);});});++describe("scrapePDFWithFirePDF typed blocks", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("requests and returns the block payload", async () => {+ const blocks = [+ {+ page: 1,+ width: 800,+ height: 1100,+ status: "ok",+ items: [+ {+ id: "p1.b0",+ type: "title",+ label: "doc_title",+ bbox: [0.1, 0.05, 0.9, 0.1],+ content: "# Annual Report",+ markdown_span: [0, 15],+ reading_order: 0,+ source: "native_text",+ confidence: { layout: 0.97, ocr: null },+ },+ ],+ },+ ];+ mockedRobustFetch.mockResolvedValue({+ markdown: "# Annual Report",+ failed_pages: null,+ pages_processed: 1,+ blocks,+ } as any);++ const result = await scrapePDFWithFirePDF(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ "auto",+ false,+ true,+ );++ expect(mockedRobustFetch).toHaveBeenCalledWith(+ expect.objectContaining({+ body: expect.objectContaining({ include_blocks: true }),+ }),+ );+ expect(+ ((mockedRobustFetch.mock.calls[0][0] as any).body as any)+ .include_page_markdown,+ ).toBeUndefined();+ expect(result.blocks).toEqual(blocks);+ expect(result.pageMarkdown).toBeUndefined();+ });++ it("rejects block-less FirePDF responses for block-aware requests", async () => {+ mockedRobustFetch.mockResolvedValue({+ markdown: "document only",+ failed_pages: null,+ pages_processed: 1,+ } as any);++ await expect(+ scrapePDFWithFirePDF(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ "auto",+ false,+ true,+ ),+ ).rejects.toThrow(/did not include requested typed blocks/);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.tsindex 5dbef323e..842d0d035 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts@@ -348,6 +348,125 @@ describe("scrapePDFWithFirePDFAsync", () => {expect(error.reason).toBe("http_5xx");});+ it("requests and returns typed blocks, tolerating the legacy pages alias", async () => {+ const blocks = [+ {+ page: 1,+ width: 800,+ height: 1100,+ status: "ok",+ items: [+ {+ id: "p1.b0",+ type: "text",+ label: "text",+ bbox: [0.1, 0.1, 0.9, 0.2],+ content: "hello",+ markdown_span: [0, 5],+ reading_order: 0,+ source: "native_text",+ confidence: { layout: 0.97, ocr: null },+ },+ ],+ },+ ];+ const { fetchImpl, calls } = makeFetchFromSequence([+ {+ matchUrl: /\/jobs$/,+ matchMethod: "POST",+ response: {+ status: 200,+ body: {+ scrape_id: "scrape-id-test",+ status: "done",+ lane: "fast",+ },+ },+ },+ {+ matchUrl: /\/jobs\/scrape-id-test\/result$/,+ matchMethod: "GET",+ response: {+ status: 200,+ body: {+ schema_version: 2,+ markdown: "hello",+ blocks,+ // Block-only jobs return `pages` as the legacy block alias+ // (no per-page markdown). The result parser must drop it+ // rather than fail the whole response.+ pages: [+ { page: 1, width: 800, height: 1100, status: "ok", blocks: [] },+ ],+ pages_processed: 1,+ },+ },+ },+ ]);++ const result = await scrapePDFWithFirePDFAsync(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ "auto",+ { fetchImpl, sleepImpl: noopSleep },+ false,+ true,+ );++ expect((calls[0].body as any).options.include_blocks).toBe(true);+ expect((calls[0].body as any).options.include_page_markdown).toBe(+ undefined,+ );+ expect(result.markdown).toBe("hello");+ expect(result.blocks).toEqual(blocks);+ expect(result.pageMarkdown).toBeUndefined();+ });++ it("fails a block-aware request when FirePDF omits the block payload", async () => {+ const { fetchImpl } = makeFetchFromSequence([+ {+ matchUrl: /\/jobs$/,+ matchMethod: "POST",+ response: {+ status: 200,+ body: {+ scrape_id: "scrape-id-test",+ status: "done",+ lane: "fast",+ },+ },+ },+ {+ matchUrl: /\/jobs\/scrape-id-test\/result$/,+ matchMethod: "GET",+ response: {+ status: 200,+ body: {+ schema_version: 1,+ markdown: "document only",+ pages_processed: 1,+ },+ },+ },+ ]);++ const error = await scrapePDFWithFirePDFAsync(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ "auto",+ { fetchImpl, sleepImpl: noopSleep },+ false,+ true,+ ).catch(error => error);++ expect(error).toBeInstanceOf(FirePdfAsyncFailure);+ expect(error.reason).toBe("http_5xx");+ });+it("submits without team context when the snapshot is absent", async () => {const { fetchImpl, calls } = makeFetchFromSequence([{diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.tsindex e741a6ac9..d697b3831 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts@@ -35,18 +35,69 @@ describe("FirePDF page-markdown cache capabilities", () => {});it("uses versioned page-capable variants", () => {- expect(cacheKeyShape("auto", undefined, true)).toMatchObject({+ expect(cacheKeyShape("auto", undefined, true, false)).toMatchObject({cacheable: true,ownVariant: "page-markdown-v1",- lookupVariants: ["page-markdown-v1", "ocr-page-markdown-v1"],+ lookupVariants: [+ "page-markdown-v1",+ "page-markdown-blocks-v1",+ "ocr-page-markdown-v1",+ "ocr-page-markdown-blocks-v1",+ ],});- expect(cacheKeyShape("ocr", undefined, true)).toMatchObject({+ expect(cacheKeyShape("ocr", undefined, true, false)).toMatchObject({cacheable: true,ownVariant: "ocr-page-markdown-v1",- lookupVariants: ["ocr-page-markdown-v1"],+ lookupVariants: ["ocr-page-markdown-v1", "ocr-page-markdown-blocks-v1"],+ });+ expect(cacheKeyShape("fast", undefined, true, false).cacheable).toBe(false);+ expect(cacheKeyShape("auto", 5, true, false).cacheable).toBe(false);+ });++ it("uses versioned block-capable variants", () => {+ expect(cacheKeyShape("auto", undefined, false, true)).toMatchObject({+ cacheable: true,+ ownVariant: "blocks-v1",+ lookupVariants: [+ "blocks-v1",+ "page-markdown-blocks-v1",+ "ocr-blocks-v1",+ "ocr-page-markdown-blocks-v1",+ ],+ });+ expect(cacheKeyShape("ocr", undefined, false, true)).toMatchObject({+ cacheable: true,+ ownVariant: "ocr-blocks-v1",+ lookupVariants: ["ocr-blocks-v1", "ocr-page-markdown-blocks-v1"],+ });+ expect(cacheKeyShape("auto", undefined, true, true)).toMatchObject({+ cacheable: true,+ ownVariant: "page-markdown-blocks-v1",+ lookupVariants: [+ "page-markdown-blocks-v1",+ "ocr-page-markdown-blocks-v1",+ ],+ });+ expect(cacheKeyShape("fast", undefined, false, true).cacheable).toBe(false);+ expect(cacheKeyShape("auto", 5, false, true).cacheable).toBe(false);+ });++ it("keeps the historical probe list for plain requests", () => {+ expect(cacheKeyShape("auto", undefined, false, false)).toMatchObject({+ cacheable: true,+ ownVariant: undefined,+ lookupVariants: [+ undefined,+ "page-markdown-v1",+ "ocr",+ "ocr-page-markdown-v1",+ ],+ });+ expect(cacheKeyShape("ocr", undefined, false, false)).toMatchObject({+ cacheable: true,+ ownVariant: "ocr",+ lookupVariants: ["ocr", "ocr-page-markdown-v1"],});- expect(cacheKeyShape("fast", undefined, true).cacheable).toBe(false);- expect(cacheKeyShape("auto", 5, true).cacheable).toBe(false);});it("never serves a document-only cache entry to a page-aware request", async () => {@@ -62,15 +113,23 @@ describe("FirePDF page-markdown cache capabilities", () => {undefined,2,true,+ false,);expect(result).toBeNull();- expect(getCached).toHaveBeenCalledOnce();- expect(getCached).toHaveBeenCalledWith(+ expect(getCached).toHaveBeenCalledTimes(2);+ expect(getCached).toHaveBeenNthCalledWith(+ 1,"BASE64","firepdf","ocr-page-markdown-v1",);+ expect(getCached).toHaveBeenNthCalledWith(+ 2,+ "BASE64",+ "firepdf",+ "ocr-page-markdown-blocks-v1",+ );});it("rejects malformed page payloads in page-capable variants", async () => {@@ -93,10 +152,11 @@ describe("FirePDF page-markdown cache capabilities", () => {undefined,2,true,+ false,);expect(result).toBeNull();- expect(getCached).toHaveBeenCalledTimes(2);+ expect(getCached).toHaveBeenCalledTimes(4);});it("rejects page-capable variants missing required document fields", async () => {@@ -117,10 +177,11 @@ describe("FirePDF page-markdown cache capabilities", () => {undefined,1,true,+ false,);expect(result).toBeNull();- expect(getCached).toHaveBeenCalledTimes(2);+ expect(getCached).toHaveBeenCalledTimes(4);});it("accepts an explicitly empty cached markdown string", async () => {@@ -137,6 +198,7 @@ describe("FirePDF page-markdown cache capabilities", () => {undefined,1,true,+ false,);expect(result).toMatchObject({ markdown: "", html: "" });@@ -164,10 +226,11 @@ describe("FirePDF page-markdown cache capabilities", () => {undefined,1,true,+ false,);expect(result).toBeNull();- expect(getCached).toHaveBeenCalledTimes(2);+ expect(getCached).toHaveBeenCalledTimes(4);});it("serves page-capable entries and preserves the page-count fallback", async () => {@@ -187,6 +250,7 @@ describe("FirePDF page-markdown cache capabilities", () => {undefined,2,true,+ false,);expect(result?.pageMarkdown).toHaveLength(2);@@ -210,6 +274,7 @@ describe("FirePDF page-markdown cache capabilities", () => {undefined,2,false,+ false,);expect(result).toMatchObject({@@ -226,6 +291,104 @@ describe("FirePDF page-markdown cache capabilities", () => {);});+ it("never serves a block-less cache entry to a block-aware request", async () => {+ getCached.mockResolvedValueOnce({+ markdown: "no blocks",+ html: "<p>no blocks</p>",+ });++ const result = await tryGetCached(+ makeMeta(),+ "BASE64",+ "ocr",+ undefined,+ 1,+ false,+ true,+ );++ expect(result).toBeNull();+ expect(getCached).toHaveBeenCalledTimes(2);+ expect(getCached).toHaveBeenNthCalledWith(+ 1,+ "BASE64",+ "firepdf",+ "ocr-blocks-v1",+ );+ expect(getCached).toHaveBeenNthCalledWith(+ 2,+ "BASE64",+ "firepdf",+ "ocr-page-markdown-blocks-v1",+ );+ });++ it("serves block-capable entries and strips payloads the request skipped", async () => {+ const blocks = [+ { page: 1, width: 800, height: 1100, status: "ok", items: [] },+ ];+ getCached.mockResolvedValueOnce(null).mockResolvedValueOnce({+ markdown: "whole",+ html: "<p>whole</p>",+ pagesProcessed: 1,+ pageMarkdown: [{ page: 1, markdown: "one" }],+ blocks,+ });++ const result = await tryGetCached(+ makeMeta(),+ "BASE64",+ "auto",+ undefined,+ 1,+ false,+ true,+ );++ expect(getCached).toHaveBeenNthCalledWith(+ 2,+ "BASE64",+ "firepdf",+ "page-markdown-blocks-v1",+ );+ expect(result?.blocks).toEqual(blocks);+ expect(result?.pageMarkdown).toBeUndefined();+ });++ it("rejects malformed block payloads in block-capable variants", async () => {+ getCached+ .mockResolvedValueOnce({+ markdown: "wrong type",+ html: "<p>wrong type</p>",+ blocks: "not-an-array" as never,+ })+ .mockResolvedValueOnce({+ markdown: "wrong page",+ html: "<p>wrong page</p>",+ blocks: [{ page: 0, items: null }] as never,+ })+ .mockResolvedValueOnce({+ markdown: "wrong item",+ html: "<p>wrong item</p>",+ blocks: [+ { page: 1, width: 800, height: 1100, status: "ok", items: [null] },+ ] as never,+ });++ const result = await tryGetCached(+ makeMeta(),+ "BASE64",+ "auto",+ undefined,+ 1,+ false,+ true,+ );++ expect(result).toBeNull();+ expect(getCached).toHaveBeenCalledTimes(4);+ });+it("writes an enriched sidecar plus a compact legacy entry", async () => {const pageMarkdown = [{ page: 1, markdown: "one" },@@ -238,6 +401,7 @@ describe("FirePDF page-markdown cache capabilities", () => {mode: "auto",maxPages: undefined,includePageMarkdown: true,+ includeBlocks: false,result: {markdown: "whole",html: "<p>whole</p>",@@ -276,6 +440,7 @@ describe("FirePDF page-markdown cache capabilities", () => {mode: "auto",maxPages: undefined,includePageMarkdown: true,+ includeBlocks: false,result: {markdown: "whole",html: "<p>whole</p>",@@ -296,4 +461,47 @@ describe("FirePDF page-markdown cache capabilities", () => {"page-markdown-v1",);});++ it("writes a block sidecar plus a compact legacy entry", async () => {+ const blocks = [+ {+ page: 1,+ width: 800,+ height: 1100,+ status: "ok",+ items: [],+ },+ ];++ await maybeSaveResult({+ meta: makeMeta(),+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: true,+ result: {+ markdown: "whole",+ html: "<p>whole</p>",+ pagesProcessed: 1,+ blocks,+ },+ });++ expect(saveCached).toHaveBeenCalledTimes(2);+ expect(saveCached).toHaveBeenNthCalledWith(+ 1,+ "BASE64",+ expect.objectContaining({ blocks }),+ "firepdf",+ "blocks-v1",+ );+ expect(saveCached).toHaveBeenNthCalledWith(+ 2,+ "BASE64",+ expect.not.objectContaining({ blocks: expect.anything() }),+ "firepdf",+ undefined,+ );+ });});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.tsindex 48bd86b6e..69b8bd569 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts@@ -33,6 +33,15 @@ describe("PDF page-markdown URL index policy", () => {},}),).toBe(false);+ expect(+ shouldUseIndex({+ ...baseMeta,+ options: {+ ...baseMeta.options,+ parsers: [{ type: "pdf", blocks: true }],+ },+ }),+ ).toBe(false);} finally {(config as { FIRECRAWL_INDEX_WRITE_ONLY?: boolean }@@ -67,4 +76,34 @@ describe("PDF page-markdown URL index policy", () => {expect(result).toBe(document);expect(result.metadata.indexId).toBeUndefined();});++ it("does not write block-aware results to the document-only URL index", async () => {+ const document = {+ markdown: "whole document",+ blocks: [+ { pageNumber: 1, width: 800, height: 1100, status: "ok", items: [] },+ ],+ rawHtml: "<p>whole document</p>",+ metadata: {+ sourceURL: "https://example.com/file.pdf",+ },+ } as any;+ const meta = {+ url: "https://example.com/file.pdf",+ winnerEngine: "pdf",+ options: {+ storeInCache: true,+ parsers: [{ type: "pdf", blocks: true }],+ },+ internalOptions: {+ isParse: false,+ zeroDataRetention: false,+ },+ } as any;++ const result = await sendDocumentToIndex(meta, document);++ expect(result).toBe(document);+ expect(result.metadata.indexId).toBeUndefined();+ });});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/blocks.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/blocks.tsnew file mode 100644index 000000000..f1d707cbf--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/blocks.ts@@ -0,0 +1,27 @@+import type { FirePdfPageBlocks, PdfPageBlocks } from "./types";++/**+ * Project fire-pdf's snake_case block wire shape onto the camelCase shape+ * surfaced on `Document.blocks`. Pure adapter β keeps the provider contract+ * translation out of the engine orchestration and next to the wire schema+ * it mirrors (fire-pdf/schema.ts).+ */+export function toPublicBlocks(blocks: FirePdfPageBlocks[]): PdfPageBlocks[] {+ return blocks.map(page => ({+ pageNumber: page.page,+ width: page.width,+ height: page.height,+ status: page.status,+ items: page.items.map(item => ({+ id: item.id,+ type: item.type,+ label: item.label,+ bbox: item.bbox,+ content: item.content,+ markdownSpan: item.markdown_span,+ readingOrder: item.reading_order,+ source: item.source,+ confidence: item.confidence,+ })),+ }));+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.tsindex 3df380609..3174dc88c 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts@@ -38,6 +38,7 @@ export async function scrapePDFWithFirePDFAsync(mode?: PDFMode,deps: FirePdfAsyncDeps = {},includePageMarkdown = false,+ includeBlocks = false,): Promise<PDFProcessorResult> {const fetchImpl = deps.fetchImpl ?? undiciFetch;const fallbackImpl = deps.fallbackImpl ?? scrapePDFWithFirePDF;@@ -55,6 +56,7 @@ export async function scrapePDFWithFirePDFAsync(pagesProcessed,mode,includePageMarkdown,+ includeBlocks,);}@@ -65,6 +67,7 @@ export async function scrapePDFWithFirePDFAsync(maxPages,pagesProcessed,includePageMarkdown,+ includeBlocks,);if (cached) return cached;@@ -89,6 +92,7 @@ export async function scrapePDFWithFirePDFAsync(pagesProcessed,mode,includePageMarkdown,+ includeBlocks,);}@@ -122,6 +126,7 @@ export async function scrapePDFWithFirePDFAsync(pagesProcessed,mode,includePageMarkdown,+ includeBlocks,deadlineAt,teamConcurrency,fetchImpl,@@ -182,6 +187,11 @@ export async function scrapePDFWithFirePDFAsync(note: "FirePDF result omitted requested physical page markdown",});}+ if (includeBlocks && fetched.blocks === undefined) {+ failAsync(meta, "http_5xx", {+ note: "FirePDF result omitted requested typed blocks",+ });+ }const durationMs = now() - overallStartedAt;firePdfAsyncTotalDurationSeconds.observe(durationMs / 1000);@@ -191,6 +201,7 @@ export async function scrapePDFWithFirePDFAsync(markdownLength: fetched.markdown.length,pagesProcessed: pages,pageMarkdownPages: fetched.pages?.length,+ blockPages: fetched.blocks?.length,failedPages: fetched.failed_pages,partialPages: fetched.partial_pages,pollCount: polled.pollCount,@@ -201,6 +212,7 @@ export async function scrapePDFWithFirePDFAsync(html: await safeMarkdownToHtml(fetched.markdown, meta.logger, meta.id),pagesProcessed: pages,...(fetched.pages ? { pageMarkdown: fetched.pages } : {}),+ ...(fetched.blocks ? { blocks: fetched.blocks } : {}),};await maybeSaveResult({@@ -209,6 +221,7 @@ export async function scrapePDFWithFirePDFAsync(mode,maxPages,includePageMarkdown,+ includeBlocks,result: processorResult,});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.tsindex 494d044c1..647aaa3ae 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts@@ -5,6 +5,7 @@ import {getPdfResultFromCache,savePdfResultToCache,} from "../../../../../lib/gcs-pdf-cache";+import { firePdfBlockPagesSchema } from "./schema";// Cache layout mirrors the sync `scrapePDFWithFirePDF` so async/sync share// entries. `fast` mode bypasses entirely (hard cost ceiling β must fail on@@ -12,6 +13,10 @@ import {// `maxPages` (the cached entry may have been written with a different cap).const PAGE_MARKDOWN_VARIANT = "page-markdown-v1";const OCR_PAGE_MARKDOWN_VARIANT = "ocr-page-markdown-v1";+const BLOCKS_VARIANT = "blocks-v1";+const OCR_BLOCKS_VARIANT = "ocr-blocks-v1";+const PAGE_MARKDOWN_BLOCKS_VARIANT = "page-markdown-blocks-v1";+const OCR_PAGE_MARKDOWN_BLOCKS_VARIANT = "ocr-page-markdown-blocks-v1";function isValidCachedDocument(value: unknown,@@ -48,28 +53,69 @@ function isValidPageMarkdown();}+// Cached block sidecars must satisfy the full wire contract before being+// served β a malformed or stale GCS artifact is skipped (and regenerated)+// rather than surfaced as invalid public block data.+function isValidBlocks(+ value: unknown,+): value is NonNullable<PDFProcessorResult["blocks"]> {+ return firePdfBlockPagesSchema.safeParse(value).success;+}+export function cacheKeyShape(mode: PDFMode | undefined,maxPages: number | undefined,includePageMarkdown: boolean,+ includeBlocks: boolean,) {const cacheable = mode !== "fast" && !maxPages;- const baseVariant: string | undefined = mode === "ocr" ? "ocr" : undefined;- const ownVariant: string | undefined = includePageMarkdown- ? mode === "ocr"- ? OCR_PAGE_MARKDOWN_VARIANT- : PAGE_MARKDOWN_VARIANT- : baseVariant;+ const isOcr = mode === "ocr";+ const baseVariant: string | undefined = isOcr ? "ocr" : undefined;+ const ownVariant: string | undefined =+ includePageMarkdown && includeBlocks+ ? isOcr+ ? OCR_PAGE_MARKDOWN_BLOCKS_VARIANT+ : PAGE_MARKDOWN_BLOCKS_VARIANT+ : includeBlocks+ ? isOcr+ ? OCR_BLOCKS_VARIANT+ : BLOCKS_VARIANT+ : includePageMarkdown+ ? isOcr+ ? OCR_PAGE_MARKDOWN_VARIANT+ : PAGE_MARKDOWN_VARIANT+ : baseVariant;- // Page-aware requests may only consume page-capable artifacts. Legacy- // requests prefer their compact entry but can reuse an enriched sidecar.- const lookupVariants: (string | undefined)[] = includePageMarkdown- ? mode === "ocr"- ? [OCR_PAGE_MARKDOWN_VARIANT]- : [PAGE_MARKDOWN_VARIANT, OCR_PAGE_MARKDOWN_VARIANT]- : mode === "ocr"- ? ["ocr", OCR_PAGE_MARKDOWN_VARIANT]- : [undefined, PAGE_MARKDOWN_VARIANT, "ocr", OCR_PAGE_MARKDOWN_VARIANT];+ // Capability rule: a request may only consume artifacts carrying every+ // capability it asked for (pages/blocks), but can reuse a richer sidecar.+ // Compact entries are preferred, and `auto` may fall back to ocr-written+ // artifacts. Plain requests keep the historical 4-variant probe list β+ // the hot path is not taxed with block-sidecar lookups.+ const lookupVariants: (string | undefined)[] = includeBlocks+ ? includePageMarkdown+ ? isOcr+ ? [OCR_PAGE_MARKDOWN_BLOCKS_VARIANT]+ : [PAGE_MARKDOWN_BLOCKS_VARIANT, OCR_PAGE_MARKDOWN_BLOCKS_VARIANT]+ : isOcr+ ? [OCR_BLOCKS_VARIANT, OCR_PAGE_MARKDOWN_BLOCKS_VARIANT]+ : [+ BLOCKS_VARIANT,+ PAGE_MARKDOWN_BLOCKS_VARIANT,+ OCR_BLOCKS_VARIANT,+ OCR_PAGE_MARKDOWN_BLOCKS_VARIANT,+ ]+ : includePageMarkdown+ ? isOcr+ ? [OCR_PAGE_MARKDOWN_VARIANT, OCR_PAGE_MARKDOWN_BLOCKS_VARIANT]+ : [+ PAGE_MARKDOWN_VARIANT,+ PAGE_MARKDOWN_BLOCKS_VARIANT,+ OCR_PAGE_MARKDOWN_VARIANT,+ OCR_PAGE_MARKDOWN_BLOCKS_VARIANT,+ ]+ : isOcr+ ? ["ocr", OCR_PAGE_MARKDOWN_VARIANT]+ : [undefined, PAGE_MARKDOWN_VARIANT, "ocr", OCR_PAGE_MARKDOWN_VARIANT];return { cacheable, ownVariant, baseVariant, lookupVariants };}@@ -80,12 +126,14 @@ export async function tryGetCached(maxPages: number | undefined,pagesProcessed: number | undefined,includePageMarkdown: boolean,+ includeBlocks: boolean,): Promise<PDFProcessorResult | null> {if (meta.internalOptions.zeroDataRetention) return null;const { cacheable, lookupVariants } = cacheKeyShape(mode,maxPages,includePageMarkdown,+ includeBlocks,);if (!cacheable) return null;@@ -99,7 +147,8 @@ export async function tryGetCached(if (cached) {if (!isValidCachedDocument(cached) ||- (includePageMarkdown && !isValidPageMarkdown(cached.pageMarkdown))+ (includePageMarkdown && !isValidPageMarkdown(cached.pageMarkdown)) ||+ (includeBlocks && !isValidBlocks(cached.blocks))) {// Defense in depth: variant names are the capability boundary, but// never let a malformed/old artifact satisfy a cache lookup.@@ -110,15 +159,13 @@ export async function tryGetCached(requestedMode: mode,cacheVariant: variant ?? "base",});- if (!includePageMarkdown) {- const { pageMarkdown: _pageMarkdown, ...compactCached } = cached;- return {- ...compactCached,- pagesProcessed: cached.pagesProcessed ?? pagesProcessed,- };- }+ // Strip payloads the request didn't ask for so a richer sidecar+ // serves a poorer request without leaking extra capabilities.+ const { pageMarkdown, blocks, ...compactCached } = cached;return {- ...cached,+ ...compactCached,+ ...(includePageMarkdown ? { pageMarkdown } : {}),+ ...(includeBlocks ? { blocks } : {}),pagesProcessed: cached.pagesProcessed ?? pagesProcessed,};}@@ -138,33 +185,46 @@ export async function maybeSaveResult(args: {mode: PDFMode | undefined;maxPages: number | undefined;includePageMarkdown: boolean;+ includeBlocks: boolean;result: PDFProcessorResult & { markdown: string };}): Promise<void> {- const { meta, base64Content, mode, maxPages, includePageMarkdown, result } =- args;+ const {+ meta,+ base64Content,+ mode,+ maxPages,+ includePageMarkdown,+ includeBlocks,+ result,+ } = args;if (meta.internalOptions.zeroDataRetention) return;const { cacheable, ownVariant, baseVariant } = cacheKeyShape(mode,maxPages,includePageMarkdown,+ includeBlocks,);if (!cacheable) return;try {await savePdfResultToCache(base64Content, result, "firepdf", ownVariant);- // A page-capable parse is also a valid legacy result. Populate the compact- // base key when it is missing so a later legacy request never repeats the- // conversion. A page-sidecar miss can coexist with a warm legacy key during- // rollout, so avoid rewriting that object. Strip the page payload to keep- // the hot-path cache object small.- if (includePageMarkdown && ownVariant !== baseVariant) {+ // An enriched (page/block-capable) parse is also a valid legacy result.+ // Populate the compact base key when it is missing so a later legacy+ // request never repeats the conversion. A sidecar miss can coexist with+ // a warm legacy key during rollout, so avoid rewriting that object.+ // Strip the enriched payloads to keep the hot-path cache object small.+ if ((includePageMarkdown || includeBlocks) && ownVariant !== baseVariant) {const existingBase = await getPdfResultFromCache(base64Content,"firepdf",baseVariant,);if (!existingBase || !isValidCachedDocument(existingBase)) {- const { pageMarkdown: _pageMarkdown, ...baseResult } = result;+ const {+ pageMarkdown: _pageMarkdown,+ blocks: _blocks,+ ...baseResult+ } = result;await savePdfResultToCache(base64Content,baseResult,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.tsindex 7f2f66170..47f197e19 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts@@ -47,16 +47,81 @@ export const pollResponseSchema = z.object({error_message: z.string().optional(),});+/**+ * Physical page markdown payload, shared by the sync `/ocr` and async+ * `/jobs/:id/result` response schemas. When `include_blocks` is requested+ * without `include_page_markdown`, fire-pdf returns `pages` in a legacy+ * block-alias shape (`{page, width, height, status, blocks}` β no markdown).+ * The union matches that alias explicitly and drops it to `undefined`, so+ * genuine protocol corruption still fails the response parse, and the+ * callers' "requested page markdown missing" checks still fire when page+ * markdown was requested but absent.+ */+export const firePdfPagesSchema = z+ .union([+ z.array(+ z.object({ page: z.number().int().positive(), markdown: z.string() }),+ ),+ z+ .array(+ // Full documented alias shape (LegacyPageBlocks) β requiring every+ // field keeps the union discriminating: a payload that is neither+ // valid page markdown nor a complete alias fails the parse.+ z.object({+ page: z.number().int().positive(),+ width: z.number().nullable(),+ height: z.number().nullable(),+ status: z.string(),+ blocks: z.array(z.unknown()),+ }),+ )+ .transform((): undefined => undefined),+ ])+ .optional();++/** Typed layout blocks (fire-pdf docs/blocks-schema.md), present only when+ * the request set `include_blocks`. Wire shape β snake_case passthrough.+ * Single source of truth for the block contract: the response parsers and+ * the GCS cache validator both use it, and the wire TS types are inferred+ * from it. */+export const firePdfBlockPagesSchema = z.array(+ z.object({+ page: z.number().int().positive(),+ width: z.number().nullable(),+ height: z.number().nullable(),+ // Documented values: ok | partial | failed. Kept open so a new+ // page status never fails an otherwise-valid scrape.+ status: z.string(),+ items: z.array(+ z.object({+ id: z.string(),+ type: z.string(),+ label: z.string().nullable(),+ bbox: z+ .tuple([z.number(), z.number(), z.number(), z.number()])+ .nullable(),+ content: z.string(),+ markdown_span: z.tuple([z.number(), z.number()]).nullable(),+ reading_order: z.number(),+ source: z.string().nullable(),+ confidence: z.object({+ layout: z.number().nullable(),+ ocr: z.number().nullable(),+ }),+ }),+ ),+ }),+);++export const firePdfBlocksSchema = firePdfBlockPagesSchema.optional();+export const resultResponseSchema = z.object({schema_version: z.union([z.literal(1), z.literal(2), z.literal(3)]).optional(),markdown: z.string(),- pages: z- .array(- z.object({ page: z.number().int().positive(), markdown: z.string() }),- )- .optional(),+ pages: firePdfPagesSchema,+ blocks: firePdfBlocksSchema,pages_processed: z.number().optional(),failed_pages: z.array(z.number()).nullable().optional(),partial_pages: z.array(z.number()).nullable().optional(),diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.tsindex e0dfa8d2c..968e0ad25 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts@@ -20,6 +20,7 @@ type SubmitArgs = {pagesProcessed: number | undefined;mode: PDFMode | undefined;includePageMarkdown: boolean;+ includeBlocks: boolean;deadlineAt: string;/** Team's sold concurrency from the ACUC (ENG-5049 account context).* Optional: entitlement lookup must never block or fail a scrape. */@@ -60,6 +61,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {pagesProcessed,mode,includePageMarkdown,+ includeBlocks,deadlineAt,teamConcurrency,fetchImpl,@@ -89,6 +91,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {...(maxPages !== undefined && { max_pages: maxPages }),...(mode !== undefined && { mode }),...(includePageMarkdown && { include_page_markdown: true }),+ ...(includeBlocks && { include_blocks: true }),},};diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsindex f41998235..3d9b11311 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -7,6 +7,7 @@ import type { PDFMode } from "../../../../controllers/v2/types";import { safeMarkdownToHtml } from "./markdownToHtml";import { createPdfCacheKey } from "../../../../lib/gcs-pdf-cache";import { maybeSaveResult, tryGetCached } from "./fire-pdf/cache";+import { firePdfBlocksSchema, firePdfPagesSchema } from "./fire-pdf/schema";/*** Reconcile an existing page count with what fire-pdf reported.@@ -45,6 +46,7 @@ export async function scrapePDFWithFirePDF(pagesProcessed?: number,mode?: PDFMode,includePageMarkdown = false,+ includeBlocks = false,): Promise<PDFProcessorResult> {const logger = meta.logger;@@ -70,6 +72,7 @@ export async function scrapePDFWithFirePDF(maxPages,pagesProcessed,includePageMarkdown,+ includeBlocks,): null;if (cached) return cached;@@ -120,6 +123,7 @@ export async function scrapePDFWithFirePDF(...(maxPages !== undefined && { max_pages: maxPages }),...(mode !== undefined && { mode }),...(includePageMarkdown && { include_page_markdown: true }),+ ...(includeBlocks && { include_blocks: true }),// Enrichment for the fire-pdf jobs DB / dashboard. fire-pdf treats// these as optional β older fire-pdf builds will ignore unknown fields.team_id: meta.internalOptions.teamId,@@ -137,11 +141,8 @@ export async function scrapePDFWithFirePDF(markdown: z.string(),failed_pages: z.array(z.number()).nullable(),pages_processed: z.number().optional(),- pages: z- .array(- z.object({ page: z.number().int().positive(), markdown: z.string() }),- )- .optional(),+ pages: firePdfPagesSchema,+ blocks: firePdfBlocksSchema,}),mock: meta.mock,abort: meta.abort.asSignal(),@@ -153,6 +154,9 @@ export async function scrapePDFWithFirePDF("FirePDF response did not include requested physical page markdown",);}+ if (includeBlocks && resp.blocks === undefined) {+ throw new Error("FirePDF response did not include requested typed blocks");+ }const pages = resp.pages_processed ?? pagesProcessed;logger.info("FirePDF completed", {@@ -170,6 +174,7 @@ export async function scrapePDFWithFirePDF(html: await safeMarkdownToHtml(resp.markdown, logger, meta.id),pagesProcessed: pages,...(resp.pages ? { pageMarkdown: resp.pages } : {}),+ ...(resp.blocks ? { blocks: resp.blocks } : {}),};if (cacheable) {@@ -179,6 +184,7 @@ export async function scrapePDFWithFirePDF(mode,maxPages,includePageMarkdown,+ includeBlocks,result: processorResult,});}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 3ce5be08e..e40c2ae19 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -19,6 +19,7 @@ import {getPDFMaxPages,getPDFMode,getPDFPageMarkdown,+ getPDFBlocks,getFirePdfAsync,} from "../../../../controllers/v2/types";import type { PDFMode } from "../../../../controllers/v2/types";@@ -40,6 +41,7 @@ import { reconcilePageCountWithFirePdf, scrapePDFWithFirePDF } from "./firePDF";import { scrapePDFWithFirePDFAsync } from "./fire-pdf/async";import { decideFirePdfAsyncRoute } from "./fire-pdf/routing";import { scrapePDFWithParsePDF } from "./pdfParse";+import { toPublicBlocks } from "./blocks";import { captureExceptionWithZdrCheck } from "../../../../services/sentry";import { isPdfBuffer, PDF_SNIFF_WINDOW } from "./pdfUtils";import { comparePdfOutputs } from "./shadowComparison";@@ -61,6 +63,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {const maxPages = getPDFMaxPages(meta.options.parsers);const mode: PDFMode = getPDFMode(meta.options.parsers);const includePageMarkdown = getPDFPageMarkdown(meta.options.parsers);+ const includeBlocks = getPDFBlocks(meta.options.parsers);if (includePageMarkdown && !config.FIRE_PDF_BASE_URL) {throw new Error(@@ -68,6 +71,12 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {);}+ if (includeBlocks && !config.FIRE_PDF_BASE_URL) {+ throw new Error(+ "Typed blocks are unavailable because FirePDF is not configured",+ );+ }+if (!shouldParse) {if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {const content = (await readFile(meta.pdfPrefetch.filePath)).toString(@@ -178,7 +187,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {let shadowPagesNeedingOcr: number[] | undefined;const forceFirePDF =- (!!meta.options.__forceFirePDF || includePageMarkdown) &&+ (!!meta.options.__forceFirePDF || includePageMarkdown || includeBlocks) &&!!config.FIRE_PDF_BASE_URL;const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;const logger = meta.logger.child({ method: "scrapePDF/processPdf" });@@ -461,17 +470,18 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {mode,undefined,includePageMarkdown,+ includeBlocks,);} catch (error) {if (- !includePageMarkdown ||+ (!includePageMarkdown && !includeBlocks) ||error instanceof RemoveFeatureError ||error instanceof AbortManagerThrownError) {throw error;}meta.logger.warn(- "FirePDF async page markdown failed -- retrying synchronously",+ "FirePDF async page markdown/blocks failed -- retrying synchronously",{method: "scrapePDF/firePDFFallback",error,@@ -490,7 +500,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {maxPages,effectivePageCount,mode,- true,+ includePageMarkdown,+ includeBlocks,);}} else {@@ -501,6 +512,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {effectivePageCount,mode,includePageMarkdown,+ includeBlocks,);}effectivePageCount = reconcilePageCountWithFirePdf(@@ -690,6 +702,9 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {})),}: {}),+ ...(includeBlocks && result?.blocks+ ? { blocks: toPublicBlocks(result.blocks) }+ : {}),pdfMetadata: {numPages: effectivePageCount,totalPages: totalPageCount,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/types.tsindex dc5f739f0..c021e134f 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts@@ -1,13 +1,45 @@+import type { z } from "zod";+import type { firePdfBlockPagesSchema } from "./fire-pdf/schema";+export type PdfPageMarkdown = {/** 1-based physical PDF page number returned by fire-pdf. */page: number;markdown: string;};+/** Typed layout blocks as fire-pdf returns them (snake_case wire shape,+ * fire-pdf docs/blocks-schema.md). Inferred from the zod wire schema so the+ * contract lives in exactly one place. Cached artifacts store this shape. */+export type FirePdfPageBlocks = z.infer<typeof firePdfBlockPagesSchema>[number];++/** Public camelCase shape surfaced on `Document.blocks`. */+export type PdfBlockItem = {+ id: string;+ type: string;+ label: string | null;+ bbox: [number, number, number, number] | null;+ content: string;+ markdownSpan: [number, number] | null;+ readingOrder: number;+ source: string | null;+ confidence: { layout: number | null; ocr: number | null };+};++export type PdfPageBlocks = {+ pageNumber: number;+ width: number | null;+ height: number | null;+ status: string;+ items: PdfBlockItem[];+};+export type PDFProcessorResult = {html: string;markdown?: string;pageMarkdown?: PdfPageMarkdown[];+ /** Typed layout blocks (fire-pdf wire shape); present only when the+ * request set the `blocks` parser option. */+ blocks?: FirePdfPageBlocks[];/*** Pages the underlying engine actually processed for this request.* Currently populated only by fire-pdf (via OcrSuccessBody.pages_processed).diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex cc53ac83b..26e8b3cc3 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -985,6 +985,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {let document: Document = {markdown: engineResult.markdown,pages: engineResult.pages,+ blocks: engineResult.blocks,rawHtml: engineResult.html,json: engineResult.json,screenshot: engineResult.screenshot,