Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(open-knowledge): MDX brace-guard UTF-16/codepoint index mismatch (#1849)
Nick Gomez858d33e345be
diff --git a/.changeset/astral-brace-byte-preservation.md b/.changeset/astral-brace-byte-preservation.mdnew file mode 100644index 000000000..d1cdd3c6c--- /dev/null+++ b/.changeset/astral-brace-byte-preservation.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Markdown documents containing an astral (non-BMP) codepoint such as an emoji before a guard-unmatched `{` no longer get a stray `{` minted into otherwise-untouched text when the document is parsed and re-serialized. Previously, each astral codepoint before the brace shifted an internal protection sentinel one position too far, so on restore a real character was overwritten with `{` and the genuine `{` was left unprotected. This affected every editor render, every `position: 'replace'` agent write, and every disk save of such a document.diff --git a/packages/app/tests/fidelity/invariant-astral-brace-byte-preservation.test.ts b/packages/app/tests/fidelity/invariant-astral-brace-byte-preservation.test.tsnew file mode 100644index 000000000..6bb268684--- /dev/null+++ b/packages/app/tests/fidelity/invariant-astral-brace-byte-preservation.test.ts@@ -0,0 +1,22 @@+import { describe, expect, test } from 'bun:test';+import { mdRoundTrip } from './helpers';++describe('Astral codepoint before an unmatched brace — embed bytes survive parse→serialize', () => {+ test('4 astral codepoints before a JS embed — embed round-trips byte-identical', () => {+ const src =+ '\u{1F3A3}\u{1F9AD}\u{1F3A3}\u{1F9AD} fishing log\n\n```js\n(function () {\n\n const DATA = { track: 1 };\n})();\n```\n';+ expect(mdRoundTrip(src)).toBe(src);+ });++ test('1 astral codepoint before a JS embed — embed round-trips byte-identical', () => {+ const src =+ '\u{1F3A3} fishing log\n\n```js\n(function () {\n\n const DATA = { track: 1 };\n})();\n```\n';+ expect(mdRoundTrip(src)).toBe(src);+ });++ test('control: same embed with BMP prose round-trips byte-identical (baseline)', () => {+ const src =+ 'xxxx fishing log\n\n```js\n(function () {\n\n const DATA = { track: 1 };\n})();\n```\n';+ expect(mdRoundTrip(src)).toBe(src);+ });+});diff --git a/packages/core/src/markdown/autolink-void-html-guard.astral-brace.test.ts b/packages/core/src/markdown/autolink-void-html-guard.astral-brace.test.tsnew file mode 100644index 000000000..6d8ff64c4--- /dev/null+++ b/packages/core/src/markdown/autolink-void-html-guard.astral-brace.test.ts@@ -0,0 +1,56 @@+import { describe, expect, test } from 'bun:test';+import * as fc from 'fast-check';+import { protectFromMdx } from './autolink-void-html-guard.ts';++function restoreString(s: string): string {+ return s+ .replaceAll('\u{E000}', '<')+ .replaceAll('\u{E001}', '>')+ .replaceAll('\u{E002}', ':')+ .replaceAll('\u{E003}', '@')+ .replaceAll('\u{E004}', '{');+}++const roundTrip = (s: string) => restoreString(protectFromMdx(s));+const countBraces = (s: string) => [...s].filter((c) => c === '{').length;++describe('R23 brace guard: astral codepoint before an unmatched brace', () => {+ const buildInput = (k: number) => `${'\u{1F3A3}'.repeat(k)}a{bcdefgh`;++ for (let k = 0; k <= 4; k++) {+ test(`protect->restore is identity with ${k} astral codepoint(s) before the brace`, () => {+ const input = buildInput(k);+ expect(roundTrip(input)).toBe(input);+ });+ }++ test('restore mints no brace the input did not contain', () => {+ const input = '\u{1F3A3}\u{1F9AD}\u{1F3A3}\u{1F9AD}a{bcdefghij';+ expect(countBraces(roundTrip(input))).toBe(countBraces(input));+ });++ test('astral before a brace flushed at a blockquote boundary', () => {+ const input = '\u{1F3A3}a{\n>}rest';+ expect(roundTrip(input)).toBe(input);+ });++ test('protect->restore identity holds for astral-bearing inputs (closes the BMP-only generator gap)', () => {+ const astralOrBmp = fc+ .array(+ fc.oneof(+ fc.constantFrom('\u{1F3A3}', '\u{1F9AD}', '\u{1F600}', '\u{1F9EA}', '\u{1F30A}'),+ fc.constantFrom(...'abc {}>\n'),+ ),+ { maxLength: 60 },+ )+ .map((parts) => parts.join(''));+ for (const seed of [42, 137, 2718, 31415, 99991]) {+ fc.assert(+ fc.property(astralOrBmp, (s) => {+ expect(roundTrip(s)).toBe(s);+ }),+ { numRuns: 200, seed },+ );+ }+ });+});diff --git a/packages/core/src/markdown/autolink-void-html-guard.ts b/packages/core/src/markdown/autolink-void-html-guard.tsindex 101069790..3dd04c233 100644--- a/packages/core/src/markdown/autolink-void-html-guard.ts+++ b/packages/core/src/markdown/autolink-void-html-guard.ts@@ -297,7 +297,7 @@ export function protectFromMdx(source: string): string {unmatchedPositions.push(...stack);if (unmatchedPositions.length > 0) {- const chars = [...result];+ const chars = result.split('');for (const pos of unmatchedPositions) {chars[pos] = GUARD_OPEN_BRACE;}