Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(api): rename PDF parser option pageMarkdown -> pages (#4352)

Abimael Martell2014d38d9c18
diff --git a/apps/api/src/__tests__/pdf-parser-options.test.ts b/apps/api/src/__tests__/pdf-parser-options.test.ts
new file mode 100644
index 000000000..f891efe6e
--- /dev/null
+++ b/apps/api/src/__tests__/pdf-parser-options.test.ts
@@ -0,0 +1,43 @@
+import {
+ getPDFBlocks,
+ getPDFPageMarkdown,
+ scrapeOptions,
+} from "../controllers/v2/types";
+
+function parsePdfParser(parser: Record<string, unknown>) {
+ const parsed = scrapeOptions.parse({ parsers: [parser] });
+ return parsed.parsers![0] as Record<string, unknown>;
+}
+
+describe("PDF parser option getters", () => {
+ it("reads the public `pages` option", () => {
+ expect(getPDFPageMarkdown([{ type: "pdf", pages: true }])).toBe(true);
+ expect(getPDFPageMarkdown([{ type: "pdf", pages: false }])).toBe(false);
+ expect(getPDFPageMarkdown([{ type: "pdf" }])).toBe(false);
+ expect(getPDFPageMarkdown(["pdf"])).toBe(false);
+ expect(getPDFPageMarkdown(undefined)).toBe(false);
+ });
+
+ it("reads the `blocks` option", () => {
+ expect(getPDFBlocks([{ type: "pdf", blocks: true }])).toBe(true);
+ expect(getPDFBlocks([{ type: "pdf" }])).toBe(false);
+ expect(getPDFBlocks(undefined)).toBe(false);
+ });
+});
+
+describe("deprecated pageMarkdown alias", () => {
+ it("is folded into `pages` at the schema boundary", () => {
+ const parser = parsePdfParser({ type: "pdf", pageMarkdown: true });
+ expect(parser.pages).toBe(true);
+ expect("pageMarkdown" in parser).toBe(false);
+ });
+
+ it("loses to an explicit `pages` when both are set", () => {
+ expect(
+ parsePdfParser({ type: "pdf", pages: false, pageMarkdown: true }).pages,
+ ).toBe(false);
+ expect(
+ parsePdfParser({ type: "pdf", pages: true, pageMarkdown: false }).pages,
+ ).toBe(true);
+ });
+});
diff --git a/apps/api/src/__tests__/snips/v2/parse.test.ts b/apps/api/src/__tests__/snips/v2/parse.test.ts
index ee8a3f07d..b6426b275 100644
--- a/apps/api/src/__tests__/snips/v2/parse.test.ts
+++ b/apps/api/src/__tests__/snips/v2/parse.test.ts
@@ -618,7 +618,7 @@ describe("/v2/parse", () => {
{
options: {
formats: ["markdown"],
- parsers: [{ type: "pdf", mode: "auto", pageMarkdown: true }],
+ parsers: [{ type: "pdf", mode: "auto", pages: true }],
},
file: {
content: pdfFixture!,
diff --git a/apps/api/src/__tests__/snips/v2/parsers-pdf-page-markdown.test.ts b/apps/api/src/__tests__/snips/v2/parsers-pdf-page-markdown.test.ts
index 57b243457..3d816f6b6 100644
--- a/apps/api/src/__tests__/snips/v2/parsers-pdf-page-markdown.test.ts
+++ b/apps/api/src/__tests__/snips/v2/parsers-pdf-page-markdown.test.ts
@@ -26,7 +26,7 @@ describeIf(SHOULD_RUN)("PDF physical page markdown", () => {
{
url: `${TEST_SUITE_WEBSITE}/example.pdf`,
formats: ["markdown"],
- parsers: [{ type: "pdf", mode: "auto", pageMarkdown: true }],
+ parsers: [{ type: "pdf", mode: "auto", pages: true }],
},
identity,
);
diff --git a/apps/api/src/__tests__/snips/v2/parsers.test.ts b/apps/api/src/__tests__/snips/v2/parsers.test.ts
index dc8392353..b265ce77f 100644
--- a/apps/api/src/__tests__/snips/v2/parsers.test.ts
+++ b/apps/api/src/__tests__/snips/v2/parsers.test.ts
@@ -388,12 +388,12 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("Parsers parameter tests", () => {
);
it.concurrent(
- "rejects non-boolean pageMarkdown",
+ "rejects non-boolean pages",
async () => {
const raw = await scrapeRaw(
{
url: pdfUrl,
- parsers: [{ type: "pdf", pageMarkdown: "yes" } as any],
+ parsers: [{ type: "pdf", pages: "yes" } as any],
},
identity,
);
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
index 1f849e983..dbc267172 100644
--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
@@ -436,15 +436,31 @@ describe("V2 Types Validation", () => {
});
it("should accept physical page markdown for PDF parsers", () => {
+ const result = scrapeRequestSchema.parse({
+ url: "https://example.com/file.pdf",
+ parsers: [{ type: "pdf", mode: "auto", pages: true }],
+ });
+
+ expect((result.parsers as any)[0].pages).toBe(true);
+ });
+
+ it("should fold the deprecated pageMarkdown alias into pages", () => {
const result = scrapeRequestSchema.parse({
url: "https://example.com/file.pdf",
parsers: [{ type: "pdf", mode: "auto", pageMarkdown: true }],
});
- expect((result.parsers as any)[0].pageMarkdown).toBe(true);
+ expect((result.parsers as any)[0].pages).toBe(true);
+ expect("pageMarkdown" in (result.parsers as any)[0]).toBe(false);
});
it("should reject non-boolean physical page markdown", () => {
+ expect(() =>
+ scrapeRequestSchema.parse({
+ url: "https://example.com/file.pdf",
+ parsers: [{ type: "pdf", pages: "yes" }],
+ }),
+ ).toThrow();
expect(() =>
scrapeRequestSchema.parse({
url: "https://example.com/file.pdf",
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index b8844ece7..64f8a8e3d 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1006,7 +1006,7 @@ export type Document = {
description?: string;
url?: string;
markdown?: string;
- /** Physical PDF pages, populated by the v2 pageMarkdown parser option. */
+ /** Physical PDF pages, populated by the v2 `pages` parser option. */
pages?: Array<{ pageNumber: number; markdown: string }>;
/** Typed PDF layout blocks with bounding boxes, populated by the v2
* `blocks` parser option. */
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 7eba70eb3..2dd98a199 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -479,21 +479,39 @@ const pdfModeSchema = z.enum(["fast", "auto", "ocr"]);
export type PDFMode = z.infer<typeof pdfModeSchema>;
-const pdfParserWithOptions = z.strictObject({
- type: z.literal("pdf"),
- mode: pdfModeSchema.optional(),
- maxPages: z.int().positive().finite().max(10000).optional(),
- /** Include physical per-page markdown alongside document markdown. */
- pageMarkdown: z.boolean().optional(),
- /** Include per-page typed layout blocks (bounding boxes, block types,
- * reading order) alongside document markdown. */
- blocks: z.boolean().optional(),
- // Experimental: route this request through the fire-pdf async pipeline
- // (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back
- // to sync on any async-path failure, so user-visible behavior is unchanged
- // beyond latency variance. Underscored to mark as internal/experimental.
- __firePdfAsync: z.boolean().optional(),
-});
+const pdfParserWithOptions = z
+ .strictObject({
+ type: z.literal("pdf"),
+ mode: pdfModeSchema.optional(),
+ maxPages: z.int().positive().finite().max(10000).optional(),
+ /** Include physical per-page markdown alongside document markdown β€”
+ * populates `document.pages`. */
+ pages: z.boolean().optional(),
+ /**
+ * @deprecated Renamed to `pages` (2026-08). Accepted as a silent alias
+ * for callers that adopted the option pre-rename; never documented.
+ * Normalized into `pages` below β€” internal code never sees this field.
+ */
+ pageMarkdown: z.boolean().optional(),
+ /** Include per-page typed layout blocks (bounding boxes, block types,
+ * reading order) alongside document markdown β€” populates
+ * `document.blocks`. */
+ blocks: z.boolean().optional(),
+ // Experimental: route this request through the fire-pdf async pipeline
+ // (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back
+ // to sync on any async-path failure, so user-visible behavior is unchanged
+ // beyond latency variance. Underscored to mark as internal/experimental.
+ __firePdfAsync: z.boolean().optional(),
+ })
+ .transform(({ pageMarkdown, pages, ...parser }) => {
+ // Fold the deprecated alias into the canonical name at the schema
+ // boundary; `pages` wins when both are set. Keep the key optional so
+ // plain `{ type: "pdf" }` parser literals stay assignable.
+ const normalized: typeof parser & { pages?: boolean } = { ...parser };
+ const effective = pages ?? pageMarkdown;
+ if (effective !== undefined) normalized.pages = effective;
+ return normalized;
+ });
const parsersSchema = z
.array(z.union([z.literal("pdf"), pdfParserWithOptions]))
@@ -541,7 +559,15 @@ export function getPDFPageMarkdown(parsers?: Parsers): boolean {
if (!parsers) return false;
for (const parser of parsers) {
if (typeof parser === "object" && parser.type === "pdf") {
- return parser.pageMarkdown === true;
+ // The deprecated `pageMarkdown` alias is folded into `pages` at the
+ // schema boundary, so freshly parsed parsers only carry the canonical
+ // name. Serialized options bypass re-parsing (queued jobs and crawl
+ // option snapshots from before the rename), so read the legacy key
+ // defensively too; `pages` wins when both are present.
+ return (
+ (parser.pages ??
+ (parser as { pageMarkdown?: boolean }).pageMarkdown) === true
+ );
}
}
return false;
@@ -1254,7 +1280,7 @@ export type Document = {
description?: string;
url?: string;
markdown?: string;
- /** Physical PDF pages, present only for `parsers[].pageMarkdown`. */
+ /** Physical PDF pages, present only for the `parsers[].pages` option. */
pages?: Array<{ pageNumber: number; markdown: string }>;
/** Typed PDF layout blocks with bounding boxes, present only for
* `parsers[].blocks`. */
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts
index 69b8bd569..5a12a4204 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts
@@ -24,6 +24,16 @@ describe("PDF page-markdown URL index policy", () => {
try {
expect(shouldUseIndex(baseMeta)).toBe(true);
+ expect(
+ shouldUseIndex({
+ ...baseMeta,
+ options: {
+ ...baseMeta.options,
+ parsers: [{ type: "pdf", pages: true }],
+ },
+ }),
+ ).toBe(false);
+ // Deprecated pre-rename alias must keep working.
expect(
shouldUseIndex({
...baseMeta,