Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
refactor(core): consolidate frontmatter record parsing on core codec (#2734)
Andrew Mikofalvyc2dbf3b7ba64
diff --git a/packages/cli/src/utils/frontmatter.test.ts b/packages/cli/src/utils/frontmatter.test.tsdeleted file mode 100644index cb53005ea..000000000--- a/packages/cli/src/utils/frontmatter.test.ts+++ /dev/null@@ -1,119 +0,0 @@-import { describe, expect, test } from 'bun:test';-import { z } from 'zod';-import { parseFrontmatter, serializeFrontmatter } from './frontmatter.ts';--describe('parseFrontmatter', () => {- test('parses valid YAML frontmatter', () => {- const content = '---\ntitle: Hello\ndescription: World\n---\n\nBody text.';- const result = parseFrontmatter(content);- expect(result).toEqual({ title: 'Hello', description: 'World' });- });-- test('returns null for content without frontmatter', () => {- expect(parseFrontmatter('# Just a heading\n\nSome text.')).toBeNull();- });-- test('returns null for empty string', () => {- expect(parseFrontmatter('')).toBeNull();- });-- test('returns null for malformed YAML', () => {- const content = '---\n[invalid: yaml: : :\n---\n\nBody.';- expect(parseFrontmatter(content)).toBeNull();- });-- test('parses tags array', () => {- const content = '---\ntitle: Test\ntags:\n - auth\n - sso\n---\n\nBody.';- const result = parseFrontmatter(content);- expect(result?.tags).toEqual(['auth', 'sso']);- });-- test('handles frontmatter with no trailing newline after closing ---', () => {- const content = '---\ntitle: Test\n---\nBody.';- const result = parseFrontmatter(content);- expect(result).toEqual({ title: 'Test' });- });-- test('returns null when frontmatter YAML parses to a scalar', () => {- const content = '---\njust a string\n---\n\nBody.';- expect(parseFrontmatter(content)).toBeNull();- });-- test('handles Windows line endings (\\r\\n)', () => {- const content = '---\r\ntitle: Windows\r\n---\r\n\r\nBody.';- const result = parseFrontmatter(content);- expect(result).toEqual({ title: 'Windows' });- });-- test('handles frontmatter at end of file (no trailing content)', () => {- const content = '---\ntitle: EOF\n---';- const result = parseFrontmatter(content);- expect(result).toEqual({ title: 'EOF' });- });-- test('validates and types with Zod schema', () => {- const ArticleSchema = z.object({- title: z.string(),- description: z.string(),- tags: z.array(z.string()).default([]),- });- const content = '---\ntitle: Auth\ndescription: How auth works\ntags:\n - auth\n---\n\nBody.';- const result = parseFrontmatter(content, ArticleSchema);- expect(result).toEqual({ title: 'Auth', description: 'How auth works', tags: ['auth'] });- });-- test('returns null when Zod schema validation fails', () => {- const StrictSchema = z.object({- title: z.string(),- count: z.number(),- });- const content = '---\ntitle: Test\ncount: not-a-number\n---\n\nBody.';- expect(parseFrontmatter(content, StrictSchema)).toBeNull();- });-- test('parses frontmatter whose opening fence carries a trailing space', () => {- // CommonMark / micromark-extension-frontmatter tolerate spaces/tabs- // after the fence sequence; the CLI parser must agree, or a stray fence- // keystroke makes the whole block invisible.- const content = '--- \ntitle: Hello\ndescription: World\n---\n\nBody text.';- expect(parseFrontmatter(content)).toEqual({ title: 'Hello', description: 'World' });- });-- test('parses frontmatter whose closing fence carries a trailing tab', () => {- const content = '---\ntitle: Hello\n---\t\n\nBody text.';- expect(parseFrontmatter(content)).toEqual({ title: 'Hello' });- });-- test('rejects leading whitespace before the opening fence', () => {- expect(parseFrontmatter(' ---\ntitle: Not FM\n---\n\nBody.')).toBeNull();- });-- test('Zod schema applies defaults for missing fields', () => {- const WithDefaults = z.object({- title: z.string(),- status: z.string().default('draft'),- });- const content = '---\ntitle: Test\n---\n\nBody.';- const result = parseFrontmatter(content, WithDefaults);- expect(result).toEqual({ title: 'Test', status: 'draft' });- });-});--describe('serializeFrontmatter', () => {- test('produces valid frontmatter block with --- delimiters', () => {- const result = serializeFrontmatter({ title: 'Hello', description: 'World' });- expect(result).toMatch(/^---\n/);- expect(result).toMatch(/\n---$/);- expect(result).toContain('title: Hello');- expect(result).toContain('description: World');- });-- test('round-trips through parse', () => {- const data = { title: 'Test', generated: true, schema_version: 1 };- const serialized = `${serializeFrontmatter(data)}\n\nBody.`;- const parsed = parseFrontmatter(serialized);- expect(parsed?.title).toBe('Test');- expect(parsed?.generated).toBe(true);- expect(parsed?.schema_version).toBe(1);- });-});diff --git a/packages/cli/src/utils/frontmatter.ts b/packages/cli/src/utils/frontmatter.tsdeleted file mode 100644index 904bf9ec1..000000000--- a/packages/cli/src/utils/frontmatter.ts+++ /dev/null@@ -1,70 +0,0 @@-/**- * YAML frontmatter parsing and serialization for markdown files.- *- * Follows the Jekyll frontmatter convention (2008) — the de facto standard- * used by Hugo, gray-matter, Astro, Next.js, and most static site generators.- * There is no formal spec (not part of CommonMark or GFM).- *- * Convention:- * - Opening `---` must be the first line of the file (byte position 0)- * - Closing `---` on its own line terminates the block- * - Content between delimiters is parsed as YAML 1.2 (via the `yaml` package)- * - Empty frontmatter (`---\n---`) is valid (parses to null)- *- * All frontmatter parsing should go through these functions — no raw regex- * matching elsewhere.- */--import { stripFrontmatter, unwrapFrontmatterFences } from '@inkeep/open-knowledge-core';-import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';-import type { output, ZodType } from 'zod';-import { isObject } from './is-object.ts';--/** Force TS to fully resolve a type in tooltips instead of showing aliases. */-type Resolve<T> = { [K in keyof T]: T[K] } & {};--/**- * Parse YAML frontmatter from a markdown string into a key-value record.- * Returns null if no frontmatter block is found or YAML is invalid.- *- * Optionally accepts a Zod schema for validation and strong typing:- *- * ```ts- * const ArticleFm = z.object({ title: z.string(), tags: z.array(z.string()) });- * const fm = parseFrontmatter(content, ArticleFm);- * // fm is { title: string; tags: string[] } | null- * ```- *- * Without a schema, returns `Record<string, unknown> | null`.- */-export function parseFrontmatter<S extends ZodType = ZodType<Record<string, unknown>>>(- content: string,- schema?: S,-): Resolve<output<S>> | null {- // Fence recognition is core's contract (`FRONTMATTER_RE` via- // stripFrontmatter/unwrapFrontmatterFences): opening `---` at byte 0,- // trailing spaces/tabs on fence lines tolerated, `\n` and `\r\n` both- // accepted, empty frontmatter (`---\n---`) valid.- const { frontmatter } = stripFrontmatter(content);- if (frontmatter === '') return null;- try {- const parsed = parseYaml(unwrapFrontmatterFences(frontmatter));- if (isObject(parsed)) {- if (schema) {- const result = schema.safeParse(parsed);- return result.success ? result.data : null;- }- return parsed as Resolve<output<S>>;- }- } catch {- // gracefully handle invalid YAML- }- return null;-}--/**- * Serialize a record into a YAML frontmatter block (with --- delimiters).- */-export function serializeFrontmatter(data: Record<string, unknown>): string {- return `---\n${stringifyYaml(data).trim()}\n---`;-}diff --git a/packages/core/src/frontmatter/record.test.ts b/packages/core/src/frontmatter/record.test.tsnew file mode 100644index 000000000..ca49127a2--- /dev/null+++ b/packages/core/src/frontmatter/record.test.ts@@ -0,0 +1,103 @@+import { describe, expect, test } from 'vitest';+import { parseFrontmatterRecord } from './record.ts';++describe('parseFrontmatterRecord', () => {+ test('parses valid YAML frontmatter into a record', () => {+ const content = '---\ntitle: Hello\ndescription: World\n---\n\nBody text.';+ expect(parseFrontmatterRecord(content)).toEqual({ title: 'Hello', description: 'World' });+ });++ test('returns null for content without a frontmatter block', () => {+ expect(parseFrontmatterRecord('# Just a heading\n\nSome text.')).toBeNull();+ });++ test('returns null for the empty string', () => {+ expect(parseFrontmatterRecord('')).toBeNull();+ });++ test('returns null for an empty frontmatter block', () => {+ expect(parseFrontmatterRecord('---\n---\n\nBody.')).toBeNull();+ expect(parseFrontmatterRecord('---\n---')).toBeNull();+ });++ test('returns null for malformed YAML', () => {+ const content = '---\n[invalid: yaml: : :\n---\n\nBody.';+ expect(parseFrontmatterRecord(content)).toBeNull();+ });++ test('returns null for duplicate keys (a YAML error)', () => {+ const content = '---\ntitle: One\ntitle: Two\n---\n\nBody.';+ expect(parseFrontmatterRecord(content)).toBeNull();+ });++ test('returns null when the YAML parses to a scalar', () => {+ expect(parseFrontmatterRecord('---\njust a string\n---\n\nBody.')).toBeNull();+ });++ test('returns null when the YAML parses to a sequence', () => {+ expect(parseFrontmatterRecord('---\n- a\n- b\n---\n\nBody.')).toBeNull();+ });++ test('parses block-style tag arrays', () => {+ const content = '---\ntitle: Test\ntags:\n - auth\n - sso\n---\n\nBody.';+ expect(parseFrontmatterRecord(content)?.tags).toEqual(['auth', 'sso']);+ });++ test('handles no trailing newline after the closing fence', () => {+ expect(parseFrontmatterRecord('---\ntitle: Test\n---\nBody.')).toEqual({ title: 'Test' });+ });++ test('handles frontmatter at end of file (no trailing content)', () => {+ expect(parseFrontmatterRecord('---\ntitle: EOF\n---')).toEqual({ title: 'EOF' });+ });++ test('handles Windows line endings (\\r\\n)', () => {+ const content = '---\r\ntitle: Windows\r\n---\r\n\r\nBody.';+ expect(parseFrontmatterRecord(content)).toEqual({ title: 'Windows' });+ });++ test('tolerates a trailing space on the opening fence', () => {+ // micromark-extension-frontmatter tolerates spaces/tabs after the fence+ // sequence; recognition here must agree with `FRONTMATTER_RE`, or a+ // stray fence keystroke makes the whole block invisible.+ const content = '--- \ntitle: Hello\ndescription: World\n---\n\nBody text.';+ expect(parseFrontmatterRecord(content)).toEqual({ title: 'Hello', description: 'World' });+ });++ test('tolerates a trailing tab on the closing fence', () => {+ const content = '---\ntitle: Hello\n---\t\n\nBody text.';+ expect(parseFrontmatterRecord(content)).toEqual({ title: 'Hello' });+ });++ test('rejects leading whitespace before the opening fence', () => {+ expect(parseFrontmatterRecord(' ---\ntitle: Not FM\n---\n\nBody.')).toBeNull();+ });++ test('keeps values verbatim: a bare key stays null', () => {+ // Contrast with `parseFrontmatterYaml`, whose map schema coerces a+ // bare-key null to '' — open-shape readers must see what the file said.+ expect(parseFrontmatterRecord('---\nbare:\n---\n\nBody.')).toEqual({ bare: null });+ });++ test('keeps values verbatim: mixed scalar arrays keep their types', () => {+ // Contrast with `parseFrontmatterYaml`, which stringifies scalar array+ // elements (['travel', '2026']).+ const content = '---\ntags: [travel, 2026]\n---\n\nBody.';+ expect(parseFrontmatterRecord(content)).toEqual({ tags: ['travel', 2026] });+ });++ test('returns null when toJS throws despite an error-free parse (unresolved alias)', () => {+ // Pins the catch guard: `parseDocument('foo: *nope')` reports zero+ // `doc.errors`, but `toJS()` throws "Unresolved alias". Circular anchors+ // do NOT throw (they yield a circular object), so this is the minimal+ // real input that reaches the catch.+ expect(parseFrontmatterRecord('---\nfoo: *nope\n---\n\nBody.')).toBeNull();+ });++ test('keeps nested mappings verbatim', () => {+ const content = '---\nicon:\n url: https://example.com/img.png\n alt: example\n---\n';+ expect(parseFrontmatterRecord(content)).toEqual({+ icon: { url: 'https://example.com/img.png', alt: 'example' },+ });+ });+});diff --git a/packages/core/src/frontmatter/record.ts b/packages/core/src/frontmatter/record.tsnew file mode 100644index 000000000..9f8ea0ddf--- /dev/null+++ b/packages/core/src/frontmatter/record.ts@@ -0,0 +1,42 @@+/**+ * Whole-document → raw frontmatter record parse — the read-side complement+ * to `parseFrontmatterYaml`.+ *+ * Open-shape readers (MCP enrichment metadata, managed `.md` files such as+ * SKILL.md, content tooling) need every key/value the document declared,+ * verbatim. `parseFrontmatterYaml` is deliberately NOT that: it validates+ * against `FrontmatterMapSchema` (the constrained editor-write value shapes)+ * and coerces on the way through (bare-key `null` → `''`, scalar array+ * elements → string), and a single out-of-shape value rejects the whole map.+ * This parse shares the canonical fence contract (`FRONTMATTER_RE` via+ * `stripFrontmatter` / `unwrapFrontmatterFences`) and the codec's+ * total-function discipline, but returns the parsed YAML mapping untouched.+ */+import { parseDocument } from 'yaml';+import { stripFrontmatter, unwrapFrontmatterFences } from '../extensions/frontmatter.ts';++/**+ * Parse a markdown document's YAML frontmatter into a raw key-value record.+ *+ * Returns `null` when the document has no frontmatter block, the block is+ * empty (`---\n---`), the YAML is malformed, or its top-level value is not a+ * mapping (scalar / sequence). Values are returned verbatim — no schema+ * validation or coercion.+ */+export function parseFrontmatterRecord(markdown: string): Record<string, unknown> | null {+ const { frontmatter } = stripFrontmatter(markdown);+ if (frontmatter === '') return null;+ let parsed: unknown;+ try {+ const doc = parseDocument(unwrapFrontmatterFences(frontmatter));+ if (doc.errors.length > 0) return null;+ // yaml@2's `toJS()` can throw even when `doc.errors` is empty (unresolved+ // aliases, the maxAliasCount resource-exhaustion guard) — same+ // total-function guard as `parseFrontmatterYaml`.+ parsed = doc.toJS();+ } catch {+ return null;+ }+ if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) return null;+ return parsed as Record<string, unknown>;+}diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex c1f76c352..27901cf42 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -1189,6 +1189,7 @@ export {fieldErrorsFromError,toFrontmatterIssue,} from './frontmatter/errors.ts';+export { parseFrontmatterRecord } from './frontmatter/record.ts';export {FRONTMATTER_TYPES,type FrontmatterMap,diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.tsindex 6a70a0226..add978b97 100644--- a/packages/server/src/api-extension.ts+++ b/packages/server/src/api-extension.ts@@ -143,6 +143,7 @@ import {type Principal,PrincipalSuccessSchema,type ProblemType,+ parseFrontmatterRecord,parseTemplateFile,prependFrontmatter,projectSkillContentDocName,@@ -13572,19 +13573,10 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const parseFrontmatterDoc = (raw: string,): { frontmatter: Record<string, unknown>; body: string } => {- const { frontmatter: fenced, body } = stripFrontmatter(raw);- let frontmatter: Record<string, unknown> = {};- if (fenced !== '') {- try {- const parsed = parseYaml(unwrapFrontmatterFences(fenced));- if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {- frontmatter = parsed as Record<string, unknown>;- }- } catch {- // Malformed YAML — return the FM-stripped body, frontmatter empty.- }- }- return { frontmatter, body };+ const { body } = stripFrontmatter(raw);+ // Malformed / missing / non-mapping frontmatter degrades to an empty+ // record — callers still get the FM-stripped body.+ return { frontmatter: parseFrontmatterRecord(raw) ?? {}, body };};const handleTemplateGet = withValidation(diff --git a/packages/server/src/content/enrichment.ts b/packages/server/src/content/enrichment.tsindex f17ada5b6..1e433bc79 100644--- a/packages/server/src/content/enrichment.ts+++ b/packages/server/src/content/enrichment.ts@@ -15,8 +15,7 @@import type { Dirent } from 'node:fs';import { readdir, readFile, stat } from 'node:fs/promises';import { basename, relative, resolve } from 'node:path';-import { OK_DIR, stripFrontmatter, unwrapFrontmatterFences } from '@inkeep/open-knowledge-core';-import { parse as parseYaml } from 'yaml';+import { OK_DIR, parseFrontmatterRecord } from '@inkeep/open-knowledge-core';import { resolveWithinRoot } from '../mcp/tools/path-safety.ts';import { httpGet } from '../mcp/tools/shared.ts';import { readFolderFrontmatter } from './nested-folder-rules.ts';@@ -24,25 +23,6 @@ import { type GitCommit, type ProjectHistorySource, readProjectGitLog } from './import { type HistorySource, readShadowLog, type ShadowCommit } from './shadow-log.ts';import { resolveTemplatesAvailable, type TemplateEntry } from './templates-resolver.ts';-// Inline frontmatter parser — keeps server-side parsing self-contained-// (no cli-package import). Returns the raw YAML object verbatim so the-// open-shape merge sees every key the file declared. Type narrowing-// for well-known scalars happens downstream. Fence recognition is core's-// contract via stripFrontmatter/unwrapFrontmatterFences.-function parseFrontmatterRaw(content: string): Record<string, unknown> | null {- const { frontmatter } = stripFrontmatter(content);- if (frontmatter === '') return null;- try {- const parsed = parseYaml(unwrapFrontmatterFences(frontmatter));- if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) {- return parsed as Record<string, unknown>;- }- } catch {- // Invalid YAML — degrade gracefully to "no frontmatter".- }- return null;-}-/** Bound on recursive directory scan when computing `DirectoryMeta`. */const DIRECTORY_SCAN_CAP = 1000;@@ -293,7 +273,9 @@ const fmReadWarnedPaths = new Set<string>();async function readFrontmatter(absPath: string): Promise<Record<string, unknown> | null> {try {const content = await readFile(absPath, 'utf-8');- const fm = parseFrontmatterRaw(content);+ // Raw record on purpose (not `parseFrontmatterYaml`): the open-shape+ // merge must see every key/value the file declared, verbatim.+ const fm = parseFrontmatterRecord(content);return fm ?? {};} catch (err) {// ENOENT is expected (caller is enriching paths from a stale listing or a