Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): rename PDF parser option pageMarkdown -> pages (#4352)
Abimael Martell2014d38d9c18
diff --git a/apps/api/src/__tests__/pdf-parser-options.test.ts b/apps/api/src/__tests__/pdf-parser-options.test.tsnew file mode 100644index 000000000..f891efe6e--- /dev/null+++ b/apps/api/src/__tests__/pdf-parser-options.test.ts@@ -0,0 +1,43 @@+import {+ getPDFBlocks,+ getPDFPageMarkdown,+ scrapeOptions,+} from "../controllers/v2/types";++function parsePdfParser(parser: Record<string, unknown>) {+ const parsed = scrapeOptions.parse({ parsers: [parser] });+ return parsed.parsers![0] as Record<string, unknown>;+}++describe("PDF parser option getters", () => {+ it("reads the public `pages` option", () => {+ expect(getPDFPageMarkdown([{ type: "pdf", pages: true }])).toBe(true);+ expect(getPDFPageMarkdown([{ type: "pdf", pages: false }])).toBe(false);+ expect(getPDFPageMarkdown([{ type: "pdf" }])).toBe(false);+ expect(getPDFPageMarkdown(["pdf"])).toBe(false);+ expect(getPDFPageMarkdown(undefined)).toBe(false);+ });++ it("reads the `blocks` option", () => {+ expect(getPDFBlocks([{ type: "pdf", blocks: true }])).toBe(true);+ expect(getPDFBlocks([{ type: "pdf" }])).toBe(false);+ expect(getPDFBlocks(undefined)).toBe(false);+ });+});++describe("deprecated pageMarkdown alias", () => {+ it("is folded into `pages` at the schema boundary", () => {+ const parser = parsePdfParser({ type: "pdf", pageMarkdown: true });+ expect(parser.pages).toBe(true);+ expect("pageMarkdown" in parser).toBe(false);+ });++ it("loses to an explicit `pages` when both are set", () => {+ expect(+ parsePdfParser({ type: "pdf", pages: false, pageMarkdown: true }).pages,+ ).toBe(false);+ expect(+ parsePdfParser({ type: "pdf", pages: true, pageMarkdown: false }).pages,+ ).toBe(true);+ });+});diff --git a/apps/api/src/__tests__/snips/v2/parse.test.ts b/apps/api/src/__tests__/snips/v2/parse.test.tsindex ee8a3f07d..b6426b275 100644--- a/apps/api/src/__tests__/snips/v2/parse.test.ts+++ b/apps/api/src/__tests__/snips/v2/parse.test.ts@@ -618,7 +618,7 @@ describe("/v2/parse", () => {{options: {formats: ["markdown"],- parsers: [{ type: "pdf", mode: "auto", pageMarkdown: true }],+ parsers: [{ type: "pdf", mode: "auto", pages: true }],},file: {content: pdfFixture!,diff --git a/apps/api/src/__tests__/snips/v2/parsers-pdf-page-markdown.test.ts b/apps/api/src/__tests__/snips/v2/parsers-pdf-page-markdown.test.tsindex 57b243457..3d816f6b6 100644--- a/apps/api/src/__tests__/snips/v2/parsers-pdf-page-markdown.test.ts+++ b/apps/api/src/__tests__/snips/v2/parsers-pdf-page-markdown.test.ts@@ -26,7 +26,7 @@ describeIf(SHOULD_RUN)("PDF physical page markdown", () => {{url: `${TEST_SUITE_WEBSITE}/example.pdf`,formats: ["markdown"],- parsers: [{ type: "pdf", mode: "auto", pageMarkdown: true }],+ parsers: [{ type: "pdf", mode: "auto", pages: true }],},identity,);diff --git a/apps/api/src/__tests__/snips/v2/parsers.test.ts b/apps/api/src/__tests__/snips/v2/parsers.test.tsindex dc8392353..b265ce77f 100644--- a/apps/api/src/__tests__/snips/v2/parsers.test.ts+++ b/apps/api/src/__tests__/snips/v2/parsers.test.ts@@ -388,12 +388,12 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("Parsers parameter tests", () => {);it.concurrent(- "rejects non-boolean pageMarkdown",+ "rejects non-boolean pages",async () => {const raw = await scrapeRaw({url: pdfUrl,- parsers: [{ type: "pdf", pageMarkdown: "yes" } as any],+ parsers: [{ type: "pdf", pages: "yes" } as any],},identity,);diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex 1f849e983..dbc267172 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -436,15 +436,31 @@ describe("V2 Types Validation", () => {});it("should accept physical page markdown for PDF parsers", () => {+ const result = scrapeRequestSchema.parse({+ url: "https://example.com/file.pdf",+ parsers: [{ type: "pdf", mode: "auto", pages: true }],+ });++ expect((result.parsers as any)[0].pages).toBe(true);+ });++ it("should fold the deprecated pageMarkdown alias into pages", () => {const result = scrapeRequestSchema.parse({url: "https://example.com/file.pdf",parsers: [{ type: "pdf", mode: "auto", pageMarkdown: true }],});- expect((result.parsers as any)[0].pageMarkdown).toBe(true);+ expect((result.parsers as any)[0].pages).toBe(true);+ expect("pageMarkdown" in (result.parsers as any)[0]).toBe(false);});it("should reject non-boolean physical page markdown", () => {+ expect(() =>+ scrapeRequestSchema.parse({+ url: "https://example.com/file.pdf",+ parsers: [{ type: "pdf", pages: "yes" }],+ }),+ ).toThrow();expect(() =>scrapeRequestSchema.parse({url: "https://example.com/file.pdf",diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex b8844ece7..64f8a8e3d 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1006,7 +1006,7 @@ export type Document = {description?: string;url?: string;markdown?: string;- /** Physical PDF pages, populated by the v2 pageMarkdown parser option. */+ /** Physical PDF pages, populated by the v2 `pages` parser option. */pages?: Array<{ pageNumber: number; markdown: string }>;/** Typed PDF layout blocks with bounding boxes, populated by the v2* `blocks` parser option. */diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 7eba70eb3..2dd98a199 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -479,21 +479,39 @@ const pdfModeSchema = z.enum(["fast", "auto", "ocr"]);export type PDFMode = z.infer<typeof pdfModeSchema>;-const pdfParserWithOptions = z.strictObject({- type: z.literal("pdf"),- mode: pdfModeSchema.optional(),- maxPages: z.int().positive().finite().max(10000).optional(),- /** Include physical per-page markdown alongside document markdown. */- pageMarkdown: z.boolean().optional(),- /** Include per-page typed layout blocks (bounding boxes, block types,- * reading order) alongside document markdown. */- blocks: z.boolean().optional(),- // Experimental: route this request through the fire-pdf async pipeline- // (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back- // to sync on any async-path failure, so user-visible behavior is unchanged- // beyond latency variance. Underscored to mark as internal/experimental.- __firePdfAsync: z.boolean().optional(),-});+const pdfParserWithOptions = z+ .strictObject({+ type: z.literal("pdf"),+ mode: pdfModeSchema.optional(),+ maxPages: z.int().positive().finite().max(10000).optional(),+ /** Include physical per-page markdown alongside document markdown β+ * populates `document.pages`. */+ pages: z.boolean().optional(),+ /**+ * @deprecated Renamed to `pages` (2026-08). Accepted as a silent alias+ * for callers that adopted the option pre-rename; never documented.+ * Normalized into `pages` below β internal code never sees this field.+ */+ pageMarkdown: z.boolean().optional(),+ /** Include per-page typed layout blocks (bounding boxes, block types,+ * reading order) alongside document markdown β populates+ * `document.blocks`. */+ blocks: z.boolean().optional(),+ // Experimental: route this request through the fire-pdf async pipeline+ // (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back+ // to sync on any async-path failure, so user-visible behavior is unchanged+ // beyond latency variance. Underscored to mark as internal/experimental.+ __firePdfAsync: z.boolean().optional(),+ })+ .transform(({ pageMarkdown, pages, ...parser }) => {+ // Fold the deprecated alias into the canonical name at the schema+ // boundary; `pages` wins when both are set. Keep the key optional so+ // plain `{ type: "pdf" }` parser literals stay assignable.+ const normalized: typeof parser & { pages?: boolean } = { ...parser };+ const effective = pages ?? pageMarkdown;+ if (effective !== undefined) normalized.pages = effective;+ return normalized;+ });const parsersSchema = z.array(z.union([z.literal("pdf"), pdfParserWithOptions]))@@ -541,7 +559,15 @@ export function getPDFPageMarkdown(parsers?: Parsers): boolean {if (!parsers) return false;for (const parser of parsers) {if (typeof parser === "object" && parser.type === "pdf") {- return parser.pageMarkdown === true;+ // The deprecated `pageMarkdown` alias is folded into `pages` at the+ // schema boundary, so freshly parsed parsers only carry the canonical+ // name. Serialized options bypass re-parsing (queued jobs and crawl+ // option snapshots from before the rename), so read the legacy key+ // defensively too; `pages` wins when both are present.+ return (+ (parser.pages ??+ (parser as { pageMarkdown?: boolean }).pageMarkdown) === true+ );}}return false;@@ -1254,7 +1280,7 @@ export type Document = {description?: string;url?: string;markdown?: string;- /** Physical PDF pages, present only for `parsers[].pageMarkdown`. */+ /** Physical PDF pages, present only for the `parsers[].pages` option. */pages?: Array<{ pageNumber: number; markdown: string }>;/** Typed PDF layout blocks with bounding boxes, present only for* `parsers[].blocks`. */diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.tsindex 69b8bd569..5a12a4204 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts@@ -24,6 +24,16 @@ describe("PDF page-markdown URL index policy", () => {try {expect(shouldUseIndex(baseMeta)).toBe(true);+ expect(+ shouldUseIndex({+ ...baseMeta,+ options: {+ ...baseMeta.options,+ parsers: [{ type: "pdf", pages: true }],+ },+ }),+ ).toBe(false);+ // Deprecated pre-rename alias must keep working.expect(shouldUseIndex({...baseMeta,