Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): stop the frontmatter-collision from eating body content (#3070)
Nick Gomezabde72c96cc6
diff --git a/.changeset/fm-collision-guard.md b/.changeset/fm-collision-guard.mdnew file mode 100644index 000000000..ae2b4340b--- /dev/null+++ b/.changeset/fm-collision-guard.md@@ -0,0 +1,9 @@+---+"@inkeep/open-knowledge": patch+---++Fix a family of frontmatter-boundary bugs that could silently remove body content from a document's view while leaving the bytes on disk intact.++A document whose body opens with a `---` rule pair and carries no frontmatter was re-read as if the span between the rules were YAML, so that content stopped reaching the editor. Compositions that derive one of their two sides now keep the boundary unambiguous — a freshly serialized body re-spells its leading rule, and the property panel keeps an explicit empty block when removing the last property would hand the body's own bytes to the frontmatter parser. Documents that carry real frontmatter are unaffected, and content the agent writes still lands byte-for-byte.++Also fixed at the same seam: a frontmatter block with no trailing newline was destroyed when a body was appended to it (reachable from ordinary typing and from an agent append), and agent `append` / `prepend` silently dropped a leading fenced span of the payload — eating body content when that span was not valid frontmatter. Such a payload is now refused with the same 400 the identical payload already returns for `replace`; payloads carrying well-formed frontmatter are unchanged.diff --git a/packages/app/src/editor/provider-pool.ts b/packages/app/src/editor/provider-pool.tsindex 28e527a9d..9b496148c 100644--- a/packages/app/src/editor/provider-pool.ts+++ b/packages/app/src/editor/provider-pool.ts@@ -1,10 +1,10 @@import { HocuspocusProvider } from '@hocuspocus/provider';import {addsBlankLines,+ composeWithDerivedBody,LINEAGE_EPOCH_KEY,MarkdownManager,normalizeBridge,- prependFrontmatter,stripFrontmatter,} from '@inkeep/open-knowledge-core';import type { HocuspocusAuthRejectionReason } from '@inkeep/open-knowledge-server';@@ -2834,7 +2834,11 @@ export class ProviderPool {if (ytextClean) {// Un-drained WYSIWYG edit: the fragment moved while Y.Text stayed// at the acked base the server rebuilt from disk.- ours = prependFrontmatter(oursFm, oursFragBody);+ // The only serialize-composed writer outside the server. Without the+ // guard, an un-drained doc-start rule pair replayed through the recycle+ // re-mints the collision server-side after every server writer is+ // fixed.+ ours = composeWithDerivedBody(oursFm, oursFragBody).md;} else if (fragClean) {// Unacked source-mode edit: Y.Text moved, fragment still at base.ours = oursYtext;diff --git a/packages/app/tests/integration/fm-collision-guard.test.ts b/packages/app/tests/integration/fm-collision-guard.test.tsnew file mode 100644index 000000000..2185d782d--- /dev/null+++ b/packages/app/tests/integration/fm-collision-guard.test.ts@@ -0,0 +1,467 @@+/**+ * Frontmatter-collision guard — end-to-end pins on the real server rigs.+ *+ * A document whose body OPENS with a `---` fence pair and carries no+ * frontmatter satisfies `FRONTMATTER_RE`, so the span between the rules is+ * read as YAML and never reaches the fragment: the bytes on disk stay perfect+ * while the content disappears from the editor. Observer A composes those+ * bytes, so it is the mint site and the fix site.+ *+ * The composition contract itself is unit-pinned alongside `core`'s bridge+ * compose module; this file pins the behaviours only the running server can+ * show — the drain,+ * the derive-timing defer bookkeeping, remote-peer convergence, the property+ * panel's HTTP surface, and the byte-sacred agent paths that must stay+ * untouched.+ */++import { setTimeout as wait } from 'node:timers/promises';+import { normalizeBridge, stripFrontmatter } from '@inkeep/open-knowledge-core';+import { afterAll, beforeAll, describe, expect, test } from 'vitest';+import * as Y from 'yjs';+import { HARNESS_BOOT_TIMEOUT_MS } from './harness-boot-timeout';+import {+ agentWriteMd,+ assertAllConverged,+ attachBridgeInvariantWatcher,+ awaitDocQuiescence,+ createTestClient,+ createTestClients,+ createTestServer,+ getServerState,+ pollUntil,+ readTestDoc,+ type TestClient,+ type TestServer,+} from './test-harness';++let server: TestServer;++beforeAll(async () => {+ server = await createTestServer();+}, HARNESS_BOOT_TIMEOUT_MS);++afterAll(async () => {+ await server.cleanup();+});++/** Append a `thematicBreak` node — the WYSIWYG mint that opens the collision. */+function appendRule(client: TestClient): void {+ client.fragment.push([new Y.XmlElement('thematicBreak')]);+}++/** Append a paragraph carrying `text`. */+function appendParagraph(client: TestClient, text: string): void {+ const paragraph = new Y.XmlElement('paragraph');+ const inner = new Y.XmlText();+ inner.applyDelta([{ insert: text }]);+ paragraph.insert(0, [inner]);+ client.fragment.push([paragraph]);+}++/** Build the family's canonical shape in the fragment: rule, x, rule, y. */+function mintRulePair(client: TestClient): void {+ client.doc.transact(() => {+ appendRule(client);+ appendParagraph(client, 'x');+ appendRule(client);+ appendParagraph(client, 'y');+ });+}++/**+ * Wait for Observer A to drain, then read the SERVER's Y.Text — the bytes+ * persistence writes and every witness compares against.+ */+async function settled(client: TestClient): Promise<string> {+ await awaitDocQuiescence(client.doc);+ await pollUntil(() => getServerState(server, client.docName) !== null, 5_000);+ await awaitDocQuiescence(client.doc);+ return getServerState(server, client.docName)?.ytext.toString() ?? '';+}++describe('Observer A drain', () => {+ test('a minted rule pair survives its own re-derivation', async () => {+ const client = await createTestClient(server.port);+ try {+ mintRulePair(client);+ const ytext = await settled(client);++ // The guard re-spelled the leading rule, so the partition finds no+ // frontmatter and the whole body reaches Observer B.+ expect(stripFrontmatter(ytext).frontmatter).toBe('');+ expect(ytext.startsWith('***')).toBe(true);+ expect(ytext).toContain('x');+ expect(ytext).toContain('y');++ // The view survives: all four nodes are still there after the round trip.+ const state = getServerState(server, client.docName);+ expect(state?.md).toContain('x');+ expect(state?.md).toContain('y');+ expect(state?.fragment.length).toBe(4);+ } finally {+ await client.cleanup();+ }+ });++ /**+ * The cell the withdrawn attempt failed. With real frontmatter the+ * composition is already unambiguous, so a re-spell here would be a+ * gratuitous rewrite of an authored rule AND a false positive in the+ * bridge-invariant channel — `normalizeBridge`'s doc-start step is anchored+ * at offset 0 and reaches only the frontmatter's own fence.+ *+ */+ test('does not fire on a frontmatter-bearing document', async () => {+ const client = await createTestClient(server.port);+ const violations: unknown[] = [];+ const detach = attachBridgeInvariantWatcher(client.doc, {+ onViolation: (info) => violations.push(info),+ });+ try {+ client.doc.transact(() => {+ client.ytext.insert(0, '---\ntitle: t\n---\nseed\n');+ });+ await settled(client);+ mintRulePair(client);+ const ytext = await settled(client);++ expect(stripFrontmatter(ytext).frontmatter).toBe('---\ntitle: t\n---\n');+ // The authored `---` spelling is untouched — no `***` anywhere.+ expect(ytext).not.toContain('***');+ expect(violations).toEqual([]);+ } finally {+ detach();+ await client.cleanup();+ }+ });++ /**+ * Adding frontmatter makes the composition unambiguous again, so the guard+ * stops firing — but the re-spelled rule does NOT churn back. The serializer+ * preserves the current spelling through `sourceRaw`, so the guarded rule is+ * simply what the document now says, and a frontmatter edit costs zero body+ * bytes. The transition is where a naive guard would oscillate, so the pin+ * is that the bytes settle and stay settled.+ *+ */+ test('adding frontmatter churns no body bytes', async () => {+ const client = await createTestClient(server.port);+ try {+ mintRulePair(client);+ const guarded = await settled(client);+ expect(guarded).toContain('***');++ client.doc.transact(() => {+ client.ytext.insert(0, '---\ntitle: t\n---\n');+ });+ const withFm = await settled(client);+ expect(withFm).toBe(`---\ntitle: t\n---\n${guarded}`);+ // Settled: a second quiescence pass moves nothing — no oscillation.+ expect(await settled(client)).toBe(withFm);+ // …and the content is all still reachable through the partition.+ expect(stripFrontmatter(withFm).body).toContain('x');+ } finally {+ await client.cleanup();+ }+ });+});++describe('guard consistency', () => {+ /**+ * The derive-timing defer predicate is a raw line-count three-way with no+ * `normalizeBridge` pass. If the drain composes the re-spelled rule while a+ * witness or the fresh-fragment comparand composes the original, the surplus+ * line reads as pending content: every drain defers, and at the bound the+ * document is force-resolved with a checkpoint and a ring event it never+ * earned. Sustained editing on a guarded document must produce neither.+ *+ */+ test('sustained editing on a guarded document mints no defer-exhaustion checkpoint', async () => {+ const client = await createTestClient(server.port);+ try {+ mintRulePair(client);+ await settled(client);++ for (let i = 0; i < 12; i++) {+ appendParagraph(client, `edit ${i}`);+ await wait(15);+ }+ const ytext = await settled(client);++ expect(ytext.startsWith('***')).toBe(true);+ for (let i = 0; i < 12; i++) expect(ytext).toContain(`edit ${i}`);++ // Y.Text carries the guarded spelling while the fragment still holds the+ // rule the user minted — the exact difference `doc-start-thematic`+ // exists to absorb, and the bridge invariant is stated modulo the+ // tolerance set, not on raw bytes.+ const state = getServerState(server, client.docName);+ expect(normalizeBridge(stripFrontmatter(ytext).body)).toBe(normalizeBridge(state?.md ?? ''));++ const events = await fetch(+ `http://127.0.0.1:${server.port}/api/timeline?docName=${encodeURIComponent(client.docName)}`,+ )+ .then((r) => (r.ok ? r.json() : { entries: [] }))+ .catch(() => ({ entries: [] }));+ const entries = (events as { entries?: Array<{ kind?: string }> }).entries ?? [];+ expect(entries.filter((e) => e.kind === 'derive-timing-defer-exhausted')).toEqual([]);+ } finally {+ await client.cleanup();+ }+ });++ /**+ * Layer-A unit tests run with `transaction.local = true`, which is not the+ * production path — a remote peer receives the guarded write over the wire+ * with `local = false`. Both surfaces must land the same bytes.+ *+ */+ test('a remote peer converges raw-byte-equal on a guard-active edit', async () => {+ const [author, peer] = await createTestClients(server.port, { count: 2 });+ try {+ mintRulePair(author);+ await settled(author);+ await assertAllConverged([author, peer]);++ const authorBytes = author.ytext.toString();+ expect(authorBytes.startsWith('***')).toBe(true);+ // Raw bytes, not normalized — the peer must not hold a different+ // spelling that only agrees within tolerance.+ expect(peer.ytext.toString()).toBe(authorBytes);+ expect(peer.ytext.toString()).toContain('x');+ } finally {+ await Promise.all([author.cleanup(), peer.cleanup()]);+ }+ });+});++describe('byte-sacred paths stay untouched', () => {+ /**+ * The agent write path is byte-faithful by contract (precedent #57) and+ * `evaluateContentDivergence` re-reads Y.Text after every write. A guard+ * leaking into it would trip that tripwire on the first `---`-leading+ * payload, so its silence is the regression signal.+ *+ */+ test('an agent write of rule-leading content lands verbatim', async () => {+ const client = await createTestClient(server.port);+ const warnings: string[] = [];+ const originalWarn = console.warn;+ console.warn = (...args: unknown[]) => {+ warnings.push(args.map(String).join(' '));+ originalWarn(...(args as []));+ };+ try {+ const payload = '---\n\nx\n\n---\n\ny\n';+ await agentWriteMd(server.port, payload, {+ docName: client.docName,+ position: 'replace',+ }).catch((err: Error & { status?: number }) => {+ // The composed FM span is a non-mapping, so replace already refuses+ // it — that refusal IS the byte-sacred contract holding.+ expect(err.status).toBe(400);+ });++ // A payload that carries no ambiguity lands byte-for-byte.+ await agentWriteMd(server.port, '***\n\nx\n\n---\n\ny\n', {+ docName: client.docName,+ position: 'replace',+ });+ await settled(client);+ expect(getServerState(server, client.docName)?.ytext.toString()).toBe(+ '***\n\nx\n\n---\n\ny\n',+ );+ expect(warnings.filter((w) => w.includes('content-divergence'))).toEqual([]);+ } finally {+ console.warn = originalWarn;+ await client.cleanup();+ }+ });+});++describe('A1 — fence termination at the compose seam', () => {+ /**+ * A frontmatter region captured at end-of-string has no trailing newline, so+ * the drain's composition used to glue the closing fence onto the body's+ * first line and destroy the block — corrupt bytes reaching disk.+ *+ */+ test('typing on a frontmatter-only document does not destroy the block', async () => {+ const client = await createTestClient(server.port);+ try {+ client.doc.transact(() => {+ client.ytext.insert(0, '---\ntitle: noeol\n---');+ });+ await settled(client);++ appendParagraph(client, 'typed after fm');+ const ytext = await settled(client);++ expect(stripFrontmatter(ytext).frontmatter).toBe('---\ntitle: noeol\n---\n');+ expect(stripFrontmatter(ytext).body).toContain('typed after fm');+ expect(ytext).not.toContain('---typed');+ } finally {+ await client.cleanup();+ }+ });++ test('an agent append to a frontmatter-only document does not destroy the block', async () => {+ const client = await createTestClient(server.port);+ try {+ client.doc.transact(() => {+ client.ytext.insert(0, '---\ntitle: noeol\n---');+ });+ await settled(client);++ await agentWriteMd(server.port, 'appended body\n', {+ docName: client.docName,+ position: 'append',+ });+ const ytext = await settled(client);++ expect(stripFrontmatter(ytext).frontmatter).toBe('---\ntitle: noeol\n---\n');+ expect(ytext).not.toContain('---appended');+ } finally {+ await client.cleanup();+ }+ });+});++describe('A2 — emptying the frontmatter region', () => {+ async function patchFrontmatter(+ docName: string,+ patch: Record<string, unknown>,+ ): Promise<number> {+ const res = await fetch(`http://127.0.0.1:${server.port}/api/frontmatter-patch`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ docName, patch }),+ });+ return res.status;+ }++ /**+ * Deleting the last property removes the whole block. Over a body that opens+ * with a rule pair that hands the body's own bytes to the next partition —+ * the family-3 loss, minted from the frontmatter side where a serialize-side+ * guard can never see it, and reaching disk through HTTP.+ *+ */+ test('keeps an explicit block when removing it would re-partition the body', async () => {+ const client = await createTestClient(server.port);+ try {+ client.doc.transact(() => {+ client.ytext.insert(0, '---\ntitle: t\n---\n---\n\nx\n\n---\n\ny\n');+ });+ await settled(client);++ expect(await patchFrontmatter(client.docName, { title: null })).toBe(200);+ const ytext = await settled(client);++ // The body is byte-identical and still partitions off cleanly.+ const partition = stripFrontmatter(ytext);+ expect(partition.frontmatter).toBe('---\n\n---\n');+ expect(partition.body).toBe('---\n\nx\n\n---\n\ny\n');+ // The content the old behaviour ate is still in the view.+ expect(getServerState(server, client.docName)?.md).toContain('x');++ // The panel path reaches disk — B13's loss was observed there, so the+ // fix has to be observed there too.+ await pollUntil(() => readTestDoc(server.contentDir, client.docName).includes('x'), 5_000);+ } finally {+ await client.cleanup();+ }+ });++ /**+ * The retention is narrow — an ordinary document must still lose the block+ * completely, or every property deletion would leave an artifact.+ *+ */+ test('an ordinary document still loses the block completely', async () => {+ const client = await createTestClient(server.port);+ try {+ client.doc.transact(() => {+ client.ytext.insert(0, '---\ntitle: t\n---\nhello\n');+ });+ await settled(client);++ expect(await patchFrontmatter(client.docName, { title: null })).toBe(200);+ const ytext = await settled(client);++ expect(stripFrontmatter(ytext).frontmatter).toBe('');+ expect(ytext).toContain('hello');+ } finally {+ await client.cleanup();+ }+ });+});++describe('A3 — append/prepend payload partition', () => {+ async function writeRaw(+ docName: string,+ markdown: string,+ position: 'append' | 'prepend' | 'replace',+ ): Promise<number> {+ const res = await fetch(`http://127.0.0.1:${server.port}/api/agent-write-md`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ docName, markdown, position }),+ });+ return res.status;+ }++ /**+ * Append and prepend partition the payload with the same regex and discard+ * the claimed span. When that span is not a YAML mapping the payload never+ * had frontmatter and the partition just ate the agent's BODY — the same+ * payload sent as `replace` is already refused, so the asymmetry was the bug.+ *+ */+ test('an ambiguous payload is refused, matching replace', async () => {+ const client = await createTestClient(server.port);+ try {+ await agentWriteMd(server.port, 'seed\n', {+ docName: client.docName,+ position: 'replace',+ });+ await settled(client);++ const ambiguous = '---\n\nx\n\n---\n\ny\n';+ expect(await writeRaw(client.docName, ambiguous, 'replace')).toBe(400);+ expect(await writeRaw(client.docName, ambiguous, 'append')).toBe(400);+ expect(await writeRaw(client.docName, ambiguous, 'prepend')).toBe(400);++ // Refused writes leave the document untouched.+ expect(getServerState(server, client.docName)?.ytext.toString()).toBe('seed\n');+ } finally {+ await client.cleanup();+ }+ });++ /**+ * A well-formed frontmatter payload keeps its documented silent drop: the+ * span parses as a mapping, the body still lands, and the bound is specified.+ * Only the unparseable case — body content being destroyed — became a+ * refusal.+ *+ */+ test('a well-formed frontmatter payload keeps its documented drop', async () => {+ const client = await createTestClient(server.port);+ try {+ await agentWriteMd(server.port, 'seed\n', {+ docName: client.docName,+ position: 'replace',+ });+ await settled(client);++ expect(await writeRaw(client.docName, '---\nsecond: fm\n---\ntail\n', 'append')).toBe(200);+ const ytext = await settled(client);+ expect(ytext).toContain('tail');+ expect(ytext).not.toContain('second: fm');+ } finally {+ await client.cleanup();+ }+ });+});diff --git a/packages/core/src/bridge/bind-frontmatter-doc.ts b/packages/core/src/bridge/bind-frontmatter-doc.tsindex c014b1413..d7a845280 100644--- a/packages/core/src/bridge/bind-frontmatter-doc.ts+++ b/packages/core/src/bridge/bind-frontmatter-doc.ts@@ -9,6 +9,7 @@ import {frontmatterValuesEqual,RESERVED_FRONTMATTER_KEY,} from '../frontmatter/schema.ts';+import { composeWithDerivedFrontmatter } from './compose.ts';import {applyPatchToFm,applyPathDeleteToFm,@@ -294,9 +295,13 @@ export function bindFrontmatterDoc(provider: FrontmatterDocProvider): Frontmatteoutcome = op(currentFenced);if (!outcome.ok) return;if (outcome.nextFenced === currentFenced) return;+ const nextFenced = composeWithDerivedFrontmatter(+ outcome.nextFenced,+ currentFull.slice(currentFenced.length),+ ).frontmatter;ytext.delete(0, currentFenced.length);- if (outcome.nextFenced !== '') {- ytext.insert(0, outcome.nextFenced);+ if (nextFenced !== '') {+ ytext.insert(0, nextFenced);}}, FORM_WRITE_ORIGIN);return (diff --git a/packages/core/src/bridge/compose.ts b/packages/core/src/bridge/compose.tsnew file mode 100644index 000000000..f4cc8a286--- /dev/null+++ b/packages/core/src/bridge/compose.ts@@ -0,0 +1,77 @@+import { prependFrontmatter, stripFrontmatter } from '../extensions/frontmatter.ts';++export type ComposeAdjustment = 'none' | 'body' | 'frontmatter' | 'unresolved';++export interface BridgeComposition {+ readonly md: string;+ readonly frontmatter: string;+ readonly body: string;+ readonly adjusted: ComposeAdjustment;+}++const EMPTY_FM_BLOCK_RE = /^---[ \t]*\r?\n---[ \t]*(\r?\n|$)/;++const LEADING_FENCE_RE = /^---([ \t]*)(?=\r?\n|$)/;++const eolOf = (s: string): string => (s.includes('\r\n') ? '\r\n' : '\n');++const respellLeadingRule = (body: string): string => body.replace(LEADING_FENCE_RE, '***$1');++const stableEmptyBlock = (eol: string): string => `---${eol}${eol}---${eol}`;++function composed(frontmatter: string, body: string): { md: string; frontmatter: string } {+ const md = prependFrontmatter(frontmatter, body);+ return { md, frontmatter: md.slice(0, md.length - body.length) };+}++function isUnambiguous(frontmatter: string, body: string): boolean {+ const { md, frontmatter: effective } = composed(frontmatter, body);+ const round = stripFrontmatter(md);+ return round.frontmatter === effective && round.body === body;+}++function settle(+ requested: string,+ frontmatter: string,+ body: string,+ adjusted: ComposeAdjustment,+): BridgeComposition {+ const { md, frontmatter: effective } = composed(frontmatter, body);+ const resolved: ComposeAdjustment =+ adjusted === 'none' && effective !== requested ? 'frontmatter' : adjusted;+ return { md, frontmatter: effective, body, adjusted: resolved };+}++export function composeWithDerivedBody(frontmatter: string, body: string): BridgeComposition {+ if (isUnambiguous(frontmatter, body)) return settle(frontmatter, frontmatter, body, 'none');++ const respelled = respellLeadingRule(body);+ if (respelled !== body && isUnambiguous(frontmatter, respelled)) {+ return settle(frontmatter, frontmatter, respelled, 'body');+ }++ if (EMPTY_FM_BLOCK_RE.test(frontmatter)) {+ const stable = stableEmptyBlock(eolOf(frontmatter));+ if (isUnambiguous(stable, body)) return settle(frontmatter, stable, body, 'frontmatter');+ }++ return settle(frontmatter, frontmatter, body, 'unresolved');+}++export function composeWithDerivedFrontmatter(+ frontmatter: string,+ body: string,+): BridgeComposition {+ if (isUnambiguous(frontmatter, body)) return settle(frontmatter, frontmatter, body, 'none');++ const stable = stableEmptyBlock(eolOf(frontmatter || body));+ if (frontmatter === '' || EMPTY_FM_BLOCK_RE.test(frontmatter)) {+ if (isUnambiguous(stable, body)) return settle(frontmatter, stable, body, 'frontmatter');+ }++ return settle(frontmatter, frontmatter, body, 'unresolved');+}++export function composesAmbiguously(frontmatter: string, body: string): boolean {+ return !isUnambiguous(frontmatter, body);+}diff --git a/packages/core/src/bridge/index.ts b/packages/core/src/bridge/index.tsindex 287afac69..fe65c0547 100644--- a/packages/core/src/bridge/index.ts+++ b/packages/core/src/bridge/index.ts@@ -23,6 +23,12 @@ export {type InvariantViolation,toBridgeInvariantLog,} from './bridge-invariant.ts';+export {+ type BridgeComposition,+ type ComposeAdjustment,+ composeWithDerivedBody,+ composeWithDerivedFrontmatter,+} from './compose.ts';export { type DiffChange, diffLinesFast } from './diff-lines.ts';export {type DocBoundarySplit,diff --git a/packages/core/src/extensions/frontmatter.test.ts b/packages/core/src/extensions/frontmatter.test.tsindex d49cbe4e2..71e3936c2 100644--- a/packages/core/src/extensions/frontmatter.test.ts+++ b/packages/core/src/extensions/frontmatter.test.ts@@ -212,3 +212,67 @@ describe('round-trip', () => {expect(reassembled).toBe(original);});});++describe('fence termination at the compose seam', () => {+ // `FRONTMATTER_RE`'s closing `(\r?\n|$)` alternative captures a region that+ // ends at end-of-string with no trailing newline. Concatenating a body onto+ // it glues the closing fence to the body's first line and destroys the+ // block — reachable in production from Observer A's drain after typing on a+ // frontmatter-only document, and from an agent append.+ test('terminates a newline-less region instead of gluing', () => {+ const fm = '---\ntitle: x\n---';+ const composed = prependFrontmatter(fm, 'hello\n');+ expect(composed).toBe('---\ntitle: x\n---\nhello\n');+ expect(stripFrontmatter(composed)).toEqual({+ frontmatter: '---\ntitle: x\n---\n',+ body: 'hello\n',+ });+ });++ test('follows the region’s own EOL convention', () => {+ const composed = prependFrontmatter('---\r\ntitle: x\r\n---', 'hello\r\n');+ expect(composed).toBe('---\r\ntitle: x\r\n---\r\nhello\r\n');+ expect(stripFrontmatter(composed).frontmatter).toBe('---\r\ntitle: x\r\n---\r\n');+ });++ test('leaves an already-terminated region byte-identical', () => {+ expect(prependFrontmatter('---\ntitle: x\n---\n', 'hello\n')).toBe(+ '---\ntitle: x\n---\nhello\n',+ );+ });++ test('does not fabricate a newline for an empty body', () => {+ expect(prependFrontmatter('---\ntitle: x\n---', '')).toBe('---\ntitle: x\n---');+ });++ // The identity every recompose site depends on. Without the `m` flag JS `$`+ // matches only at end of input, so a region captured without a trailing+ // newline consumed the whole string and left an empty body — which is why+ // terminating the fence cannot change any strip-derived recomposition.+ test('strip ∘ prepend stays byte-identity across the edge corpus', () => {+ const corpus: string[] = ['', '---', '---\n', 'plain', '---\nfoo\n', ' ---\na: 1\n---\n'];+ for (const fence of ['---', '--- ', '---\t']) {+ for (const eol of ['\n', '\r\n']) {+ for (const yaml of ['', 'title: x', 'a: 1\nb: 2', 'not: [valid', '\nx\n']) {+ for (const tail of ['', 'body', 'body\n', '---\nmore\n', '\n\n---\n\ny\n']) {+ const region = `${fence}${eol}${yaml ? yaml + eol : ''}${fence}`;+ corpus.push(`${region}${eol}${tail}`, `${region}${tail}`);+ }+ }+ }+ }+ let newlineLessRegions = 0;+ for (const input of corpus) {+ const { frontmatter, body } = stripFrontmatter(input);+ if (frontmatter !== '' && !/\r?\n$/.test(frontmatter)) {+ newlineLessRegions++;+ expect(body, `newline-less region with a non-empty body: ${JSON.stringify(input)}`).toBe(+ '',+ );+ }+ expect(prependFrontmatter(frontmatter, body), JSON.stringify(input)).toBe(input);+ }+ // Non-vacuity: the corpus must actually contain the edge the guard covers.+ expect(newlineLessRegions).toBeGreaterThan(0);+ });+});diff --git a/packages/core/src/extensions/frontmatter.ts b/packages/core/src/extensions/frontmatter.tsindex 20b59935a..8f609e235 100644--- a/packages/core/src/extensions/frontmatter.ts+++ b/packages/core/src/extensions/frontmatter.ts@@ -35,9 +35,31 @@ export function stripFrontmatter(markdown: string): { frontmatter: string; body:return { frontmatter: '', body: markdown };}+/**+ * Compose a frontmatter region with a body.+ *+ * `FRONTMATTER_RE`'s closing `(\r?\n|$)` alternative captures a region that+ * ends at end-of-string with no trailing newline, so a naive concatenation+ * GLUES the closing fence onto the body's first line and destroys the block:+ * `---\ntitle: x\n---` + `hello\n` would yield `---\ntitle: x\n---hello\n`,+ * whose re-partition finds no frontmatter at all. Reachable in production from+ * Observer A's drain after WYSIWYG typing on a frontmatter-only document and+ * from an agent append, both landing corrupt bytes on disk. Terminate the+ * fence instead, following the region's own EOL convention so a CRLF document+ * doesn't gain a mixed line ending.+ *+ * This does NOT weaken the `stripFrontmatter` ∘ `prependFrontmatter` identity+ * that every recompose site depends on: without the `m` flag JS `$` matches+ * only at end of input, so a match ending there consumed the whole string and+ * `stripFrontmatter` returned an empty body — the branch below short-circuits+ * on exactly that case. Every strip-derived pair therefore recomposes+ * byte-for-byte; this module's tests pin that as a property over the edge+ * corpus, so a future relaxation of the short-circuit fails there.+ */export function prependFrontmatter(frontmatter: string, body: string): string {if (!frontmatter) return body;- return frontmatter + body;+ if (body === '' || /\r?\n$/.test(frontmatter)) return frontmatter + body;+ return frontmatter + (frontmatter.includes('\r\n') ? '\r\n' : '\n') + body;}/**diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 730830605..139382d12 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -1282,6 +1282,7 @@ export {applyReorderToFm,assertContentPreservation,BRIDGE_TOLERANCE_CLASSES,+ type BridgeComposition,type BridgeInvariantLogPayload,type BridgeInvariantSite,type BridgeInvariantViolation,@@ -1295,9 +1296,12 @@ export {type BridgeToleranceSignal,bindFrontmatterDoc,type ComparePmStructuralOptions,+ type ComposeAdjustment,classifySeverity,comparePmStructural,compareRoundTripStructural,+ composeWithDerivedBody,+ composeWithDerivedFrontmatter,createStructuralFreshnessChecker,type DiffChange,type DocBoundarySplit,diff --git a/packages/server/src/agent-sessions.ts b/packages/server/src/agent-sessions.tsindex 54c45b5c1..30df2e39b 100644--- a/packages/server/src/agent-sessions.ts+++ b/packages/server/src/agent-sessions.ts@@ -372,6 +372,14 @@ export function snapshotBlocks(document: Document): string[] {interface ComposedAgentWrite {existingFm: string;finalFm: string;+ /**+ * The span append/prepend partitioned off the payload and discarded. Empty+ * on replace/patch, which supersede the FM region instead of dropping it.+ * Dropping a well-formed block is the documented bound; dropping a span that+ * is NOT a YAML mapping means the payload never had frontmatter and the+ * partition just ate the agent's BODY, which the gate refuses.+ */+ droppedPayloadFm: string;/** Full document bytes (FM + body) the primitives apply verbatim. */newContent: string;}@@ -428,7 +436,12 @@ function composeAgentWrite(break;}- return { existingFm, finalFm, newContent: prependFrontmatter(finalFm, newBody) };+ return {+ existingFm,+ finalFm,+ droppedPayloadFm: position === 'append' || position === 'prepend' ? payloadFm : '',+ newContent: prependFrontmatter(finalFm, newBody),+ };}function applyAgentMarkdownWriteInner(@@ -449,7 +462,7 @@ function applyAgentMarkdownWriteInner(if (composed === undefined) {return;}- const { existingFm, finalFm, newContent } = composed;+ const { existingFm, finalFm, droppedPayloadFm, newContent } = composed;// A `replace`/`patch` that rebuilds the fragment over un-propagated WYSIWYG// content silently discards it — wire the paired-intake derive-loss detector@@ -480,11 +493,11 @@ function applyAgentMarkdownWriteInner(// banner, and the file's keys are unrecoverable without a hand-edit.//// Gate is targeted: only fires when the agent CHANGES the FM- // (`finalFm !== existingFm`). Append/prepend never touch FM (payload- // FM is dropped earlier in this function), so they skip the gate.- // Existing docs that already carry malformed FM keep accepting- // body-only writes — the rejection follows the introducer, not the- // inheritor.+ // (`finalFm !== existingFm`). Append/prepend can never satisfy that —+ // they inherit `existingFm` by construction — so they carry their own+ // arm below, keyed on the payload span they discard. Existing docs that+ // already carry malformed FM keep accepting body-only writes — the+ // rejection follows the introducer, not the inheritor.//// No byte mutation: we parse for validation only. The agent's bytes// are preserved verbatim once they pass (Y.Text-is-truth, precedent@@ -517,6 +530,21 @@ function applyAgentMarkdownWriteInner(// `frontmatter-malformed-write-refused` structured log already// carries the refusal signal.recordFrontmatterEditSurface('mcp-write');+ } else if (droppedPayloadFm !== '') {+ // Append/prepend partition the payload with the same `FRONTMATTER_RE` and+ // discard the claimed span. Dropping a well-formed block is the+ // documented bound and stays silent — the payload's BODY still lands+ // intact. But a span that isn't a YAML mapping was never frontmatter:+ // the partition just ate the agent's content, and the same payload sent+ // as `replace` is already refused with this exact error and class. Same+ // predicate, same envelope, same event — the asymmetry was the bug.+ const droppedParsed = parseFrontmatterYaml(unwrapFrontmatterFences(droppedPayloadFm));+ if (droppedParsed.map === null) {+ throw new FrontmatterMalformedError({+ file: docNameToFile(document.name),+ parseError: droppedParsed.parseError ?? 'unknown YAML parse error',+ });+ }}// Hand the composed full bytes (FM + body) to the shared primitive.diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.tsindex 9cb90bcb9..b0251807b 100644--- a/packages/server/src/api-extension.ts+++ b/packages/server/src/api-extension.ts@@ -66,6 +66,7 @@ import {CreatePageSuccessSchema,changedBlockRange,colorFromSeed,+ composeWithDerivedFrontmatter,createCodeFenceTracker,createWorkspaceSearchCorpus,createWorkspaceSearchDocument,@@ -6544,8 +6545,15 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// empty (FM-only doc) or already starts with a newline.const needsFenceSeparator =currentFenced === '' && currentBody !== '' && !currentBody.startsWith('\n');- const newFull =- result.nextFenced + (needsFenceSeparator ? '\n' : '') + currentBody;+ // Same boundary as the panel binding's commit funnel, same+ // primitive: emptying the region on a document whose body+ // opens with a rule pair would hand those bytes to the next+ // partition. Verified end-to-end at `/api/frontmatter-patch`+ // because this path reaches disk.+ const newFull = composeWithDerivedFrontmatter(+ result.nextFenced,+ (needsFenceSeparator ? '\n' : '') + currentBody,+ ).md;composeAndWriteRawBody(session.dc.document,newFull,diff --git a/packages/server/src/bridge-intake.ts b/packages/server/src/bridge-intake.tsindex bd2b0fffd..766f3fd76 100644--- a/packages/server/src/bridge-intake.ts+++ b/packages/server/src/bridge-intake.ts@@ -44,7 +44,11 @@* next non-paired ytext mutation would re-derive fragment from the STALE* ytext bytes, silently reverting the write.*/-import { applyFastDiff, prependFrontmatter, stripFrontmatter } from '@inkeep/open-knowledge-core';+import {+ applyFastDiff,+ composeWithDerivedBody,+ stripFrontmatter,+} from '@inkeep/open-knowledge-core';import type { JSONContent } from '@tiptap/core';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import type * as Y from 'yjs';@@ -174,7 +178,12 @@ function reportPairedDeriveLoss(baselineBody,ytextDerivedBody,rebuiltBody,- restorePayload: prependFrontmatter(restoreFrontmatter, pendingBody),+ // The one checkpoint kind whose contents are compose-derived rather than+ // Y.Text's own bytes. Un-guarded, a payload whose body opens with a rule+ // pair cannot restore what it captured — the restore re-partitions and+ // loses the span again, so the recovery channel inherits the bug it exists+ // to recover from.+ restorePayload: composeWithDerivedBody(restoreFrontmatter, pendingBody).md,});}@@ -439,7 +448,7 @@ export function deriveFragmentFromYtext(baselineBody,ytextDerivedBody,rebuiltBody,- restorePayload: prependFrontmatter(frontmatter, pendingBody),+ restorePayload: composeWithDerivedBody(frontmatter, pendingBody).md,});}}diff --git a/packages/server/src/derive-pre-drain.test.ts b/packages/server/src/derive-pre-drain.test.tsindex 468fbdc3f..9a451b21e 100644--- a/packages/server/src/derive-pre-drain.test.ts+++ b/packages/server/src/derive-pre-drain.test.ts@@ -212,3 +212,48 @@ describe('pre-drain paired-vector arms (H15)', () => {}});});++describe('pre-drain frontmatter-ambiguity decline', () => {+ beforeEach(() => {+ vi.useFakeTimers({ toFake: ['Date'] });+ vi.setSystemTime(1_000_000);+ });+ afterEach(() => {+ vi.useRealTimers();+ });++ test('a pending doc-start rule pair declines the flush instead of writing un-adjusted bytes', async () => {+ const rig = await createWiredPreDrainRig({ docName: 'fm-ambiguous-predrain.md' });+ try {+ // Y.Text holds an ordinary body; the fragment then advances to one whose+ // serialization OPENS with a fence pair, un-propagated (freshness+ // suppressed by the recent external write).+ rig.rig.seedSource('seed body\n');+ rig.rig.externalYtextEdit('poke', (yt) => {+ yt.insert(yt.length, 'trailing\n');+ });+ rig.rig.editFragment('---\n\nx\n\n---\n\nseed body\n', { advanceFreshness: false });++ const pending = rig.serializeFragment();+ expect(pending.startsWith('---')).toBe(true);+ expect(rig.ytextString().startsWith('---')).toBe(false);++ // The flush splice is modelled in un-adjusted body space, so writing it+ // would seed bytes the settlement witness (composed through the guard)+ // disagrees with — an incoherent raw witness permanently disables+ // freshness re-derives for the doc. Decline to the checkpoint floor.+ const verdict = getPreDrainController(rig.doc)?.preDrain({+ kind: 'agent-write',+ composedBody: 'anything',+ writeKind: 'append',+ });+ expect(verdict?.preDrain).toBe(false);+ expect(verdict?.reason).toBe('checkpoint-fm-ambiguous');++ // Nothing was written by the declined flush.+ expect(rig.ytextString().startsWith('---')).toBe(false);+ } finally {+ await rig.cleanup();+ }+ });+});diff --git a/packages/server/src/pre-drain-discriminator.ts b/packages/server/src/pre-drain-discriminator.tsindex 335ad0659..d8cee4f8c 100644--- a/packages/server/src/pre-drain-discriminator.ts+++ b/packages/server/src/pre-drain-discriminator.ts@@ -73,6 +73,11 @@ type PreDrainReason =| 'checkpoint-witness-mismatch'| 'checkpoint-null-splice'| 'checkpoint-no-target'+ // The flush would compose frontmatter-ambiguous bytes: the splice model runs+ // in un-adjusted body space, so writing it would disagree with the guarded+ // settlement witness. Declined by the controller, not by `planPreDrain` —+ // the plan stays a pure function of the two spans and the body bytes.+ | 'checkpoint-fm-ambiguous'| 'checkpoint-substantive-overlap'| 'pre-drain-disjoint'| 'pre-drain-whitespace-graze';diff --git a/packages/server/src/server-observers.ts b/packages/server/src/server-observers.tsindex 4dc02579f..bad55753c 100644--- a/packages/server/src/server-observers.ts+++ b/packages/server/src/server-observers.ts@@ -35,6 +35,7 @@ import {BridgeInvariantViolationError,BridgeMergeContentLossError,comparePmStructural,+ composeWithDerivedBody,DUPLICATION_GATE_MIN_LINE_LENGTH,fnv1aDigest,fragmentHoldsPendingContent,@@ -1517,7 +1518,7 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => voidconst initialJson = yXmlFragmentToProseMirrorRootNode(xmlFragment, schema).toJSON();const initialBody = mdManager.serialize(initialJson);const initialFrontmatter = readCurrentFm();- const canonicalInit = prependFrontmatter(initialFrontmatter, initialBody);+ const canonicalInit = composeWithDerivedBody(initialFrontmatter, initialBody).md;// Observers normally attach after the persistence paired-write seed, so// fragment = parse(ytext) and attach is a true settlement point — the raw// witness then captures the seed bytes so the first fragment change on a@@ -1743,7 +1744,14 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void// guard against the re-derived bytes.if (freshnessSafe) runProducerGuard(json as PmStructuralNode, body);const frontmatter = readCurrentFm();- const md = prependFrontmatter(frontmatter, body);+ // The family-3 mint site. A fragment whose serialization opens with a+ // rule pair composes bytes the next partition reads as frontmatter, so+ // the span never reaches Observer B's re-derive; the guard re-spells the+ // leading rule when — and only when — the composition is genuinely+ // ambiguous. Every witness and predicate below composes the same way, or+ // the surplus line reads as pending content and defer-loops the doc.+ const composition = composeWithDerivedBody(frontmatter, body);+ const md = composition.md;const currentText = ytext.toString();// Duplication gate (precedent #38, Y.Text-is-truth). A CRDT@@ -1957,7 +1965,14 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void// (canonical-base fragment-delta splice into raw bytes) — the splice// owns Path A's domain only, replacing the wholesale canonical// rewrite, never the residual-preservation path.- ytextInSync && residualMergeEligible+ // The splice is a pure body-space rewrite computed straight off the+ // fragment, so it would write the un-adjusted rule while `md` (and+ // every witness below) carries the guarded spelling. Decline on the+ // transition drain and let the canonical rewrite below apply `md`;+ // once Y.Text holds the guarded bytes the serializer preserves them+ // via sourceRaw, the composition is unambiguous, and the splice —+ // the more byte-preserving path — runs again.+ (ytextInSync && residualMergeEligible) || composition.adjusted !== 'none'? null: tryComputeMapDrivenSplice({currentText,@@ -2209,7 +2224,7 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => voidtry {const recoveryJson = yXmlFragmentToProseMirrorRootNode(xmlFragment, schema).toJSON();const recoveryBody = mdManager.serialize(recoveryJson);- const recoveryMd = prependFrontmatter(readCurrentFm(), recoveryBody);+ const recoveryMd = composeWithDerivedBody(readCurrentFm(), recoveryBody).md;if (settlesSplitBrainChecked(ytext.toString(), recoveryMd)) {recordSplitBrainRecoveryBaselines(recoveryMd);textDirty = true;@@ -2331,7 +2346,10 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => voidconst json = yXmlFragmentToProseMirrorRootNode(xmlFragment, schema).toJSON();const body = mdManager.serialize(json);const frontmatter = readCurrentFm();- const md = prependFrontmatter(frontmatter, body);+ // Same ambiguity as the drain, one seed earlier: a fragment carrying a+ // doc-start rule pair would seed bytes an empty Y.Text can never+ // re-derive from.+ const md = composeWithDerivedBody(frontmatter, body).md;doc.transact(() => {ytext.insert(0, md);}, OBSERVER_SYNC_ORIGIN);@@ -2441,7 +2459,12 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => voidconst freshFragmentBody = mdManager.serialize(yXmlFragmentToProseMirrorRootNode(xmlFragment, schema).toJSON(),);- const freshFragmentMd = prependFrontmatter(frontmatter, freshFragmentBody);+ // Guard-consistent with the drain's `md` and the converged witness:+ // the predicate is a raw line-count three-way, so an un-guarded+ // comparand here reads the re-spelled rule as a surplus line, defers+ // every drain, and force-resolves an innocent doc at the exhaustion+ // bound.+ const freshFragmentMd = composeWithDerivedBody(frontmatter, freshFragmentBody).md;if (fragmentHoldsPendingContent(freshFragmentMd, md, lastConvergedFragmentMd)) {if (consecutiveDeriveTimingDefers >= MAX_DERIVE_TIMING_DEFERS) {// Exhaustion. The keystroke has stayed un-propagated across the full@@ -2534,6 +2557,16 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void// computation sees a coherent starting point.try {const canonicalBody = mdManager.serialize(parsedJson);+ // Faithful compose, deliberately. This value is both the invariant's+ // RHS (a comparator — guarding it recreates the withdrawn attempt's+ // false positive) and the settlement witness (which must agree with+ // what the writers produce). Those requirements coincide here: this+ // body derives from `stripFrontmatter(ytext).body`, so an empty+ // frontmatter means `FRONTMATTER_RE` did not match Y.Text, which means+ // the body cannot open a region — the guard's precondition is+ // unreachable on this leg and the two composes are byte-equal. The+ // bridge compose module's tests pin that no-op rather than assuming+ // it, so this stays a checked coincidence and not a lucky one.const canonicalYText = prependFrontmatter(frontmatter, canonicalBody);assertBridgeInvariant(ytext.toString(), canonicalYText, {site: 'observer-b',@@ -2594,7 +2627,7 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => voidconst postJson = yXmlFragmentToProseMirrorRootNode(xmlFragment, schema).toJSON();const postBody = mdManager.serialize(postJson);const fm = readCurrentFm();- refreshCanonicalWitnessOnly(prependFrontmatter(fm, postBody));+ refreshCanonicalWitnessOnly(composeWithDerivedBody(fm, postBody).md);} catch (innerErr) {// Mirror the two `instanceof BridgeInvariantViolationError` catches// above — preserve `BridgeInvariantViolationError` throws past this@@ -2838,6 +2871,20 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void// discriminated union makes the old `|| splice === null` arm dead.if (!preDrain) return verdict;+ // The flush splice is modelled in UN-adjusted body space, so on the one+ // transition drain where the fragment newly mints a doc-start rule pair+ // it would write bytes the settlement witness below (composed through+ // the guard) disagrees with — an incoherent raw witness disables+ // freshness re-derives for the doc permanently. Decline instead: the+ // pending content still survives via the checkpoint floor, and the next+ // Observer A drain applies the guard properly. Steady state is+ // unaffected — once Y.Text holds the re-spelled rule the serializer+ // preserves it via sourceRaw and the composition is unambiguous again.+ const flushComposition = composeWithDerivedBody(readCurrentFm(), mdManager.serialize(json));+ if (flushComposition.adjusted !== 'none') {+ return { preDrain: false, reason: 'checkpoint-fm-ambiguous' };+ }+// Flush: apply the drain's splice to Y.Text under the observer// self-origin, then record the settlement — post-flush Y.Text equals the// fragment's canonical serialization, a true fixed point, so the@@ -2851,7 +2898,7 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => voidnewSlice: splice.newSlice,});}, OBSERVER_SYNC_ORIGIN);- const canonicalMd = prependFrontmatter(readCurrentFm(), mdManager.serialize(json));+ const canonicalMd = flushComposition.md;recordSettledBaselines(canonicalMd);lastConvergedFragmentMd = canonicalMd;fragmentMutatedSinceConverge = false;