Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(open-knowledge): MDX brace-guard UTF-16/codepoint index mismatch (#1849)

Nick Gomez858d33e345be
diff --git a/.changeset/astral-brace-byte-preservation.md b/.changeset/astral-brace-byte-preservation.md
new file mode 100644
index 000000000..d1cdd3c6c
--- /dev/null
+++ b/.changeset/astral-brace-byte-preservation.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Markdown documents containing an astral (non-BMP) codepoint such as an emoji before a guard-unmatched `{` no longer get a stray `{` minted into otherwise-untouched text when the document is parsed and re-serialized. Previously, each astral codepoint before the brace shifted an internal protection sentinel one position too far, so on restore a real character was overwritten with `{` and the genuine `{` was left unprotected. This affected every editor render, every `position: 'replace'` agent write, and every disk save of such a document.
diff --git a/packages/app/tests/fidelity/invariant-astral-brace-byte-preservation.test.ts b/packages/app/tests/fidelity/invariant-astral-brace-byte-preservation.test.ts
new file mode 100644
index 000000000..6bb268684
--- /dev/null
+++ b/packages/app/tests/fidelity/invariant-astral-brace-byte-preservation.test.ts
@@ -0,0 +1,22 @@
+import { describe, expect, test } from 'bun:test';
+import { mdRoundTrip } from './helpers';
+
+describe('Astral codepoint before an unmatched brace — embed bytes survive parse→serialize', () => {
+ test('4 astral codepoints before a JS embed — embed round-trips byte-identical', () => {
+ const src =
+ '\u{1F3A3}\u{1F9AD}\u{1F3A3}\u{1F9AD} fishing log\n\n```js\n(function () {\n\n const DATA = { track: 1 };\n})();\n```\n';
+ expect(mdRoundTrip(src)).toBe(src);
+ });
+
+ test('1 astral codepoint before a JS embed — embed round-trips byte-identical', () => {
+ const src =
+ '\u{1F3A3} fishing log\n\n```js\n(function () {\n\n const DATA = { track: 1 };\n})();\n```\n';
+ expect(mdRoundTrip(src)).toBe(src);
+ });
+
+ test('control: same embed with BMP prose round-trips byte-identical (baseline)', () => {
+ const src =
+ 'xxxx fishing log\n\n```js\n(function () {\n\n const DATA = { track: 1 };\n})();\n```\n';
+ expect(mdRoundTrip(src)).toBe(src);
+ });
+});
diff --git a/packages/core/src/markdown/autolink-void-html-guard.astral-brace.test.ts b/packages/core/src/markdown/autolink-void-html-guard.astral-brace.test.ts
new file mode 100644
index 000000000..6d8ff64c4
--- /dev/null
+++ b/packages/core/src/markdown/autolink-void-html-guard.astral-brace.test.ts
@@ -0,0 +1,56 @@
+import { describe, expect, test } from 'bun:test';
+import * as fc from 'fast-check';
+import { protectFromMdx } from './autolink-void-html-guard.ts';
+
+function restoreString(s: string): string {
+ return s
+ .replaceAll('\u{E000}', '<')
+ .replaceAll('\u{E001}', '>')
+ .replaceAll('\u{E002}', ':')
+ .replaceAll('\u{E003}', '@')
+ .replaceAll('\u{E004}', '{');
+}
+
+const roundTrip = (s: string) => restoreString(protectFromMdx(s));
+const countBraces = (s: string) => [...s].filter((c) => c === '{').length;
+
+describe('R23 brace guard: astral codepoint before an unmatched brace', () => {
+ const buildInput = (k: number) => `${'\u{1F3A3}'.repeat(k)}a{bcdefgh`;
+
+ for (let k = 0; k <= 4; k++) {
+ test(`protect->restore is identity with ${k} astral codepoint(s) before the brace`, () => {
+ const input = buildInput(k);
+ expect(roundTrip(input)).toBe(input);
+ });
+ }
+
+ test('restore mints no brace the input did not contain', () => {
+ const input = '\u{1F3A3}\u{1F9AD}\u{1F3A3}\u{1F9AD}a{bcdefghij';
+ expect(countBraces(roundTrip(input))).toBe(countBraces(input));
+ });
+
+ test('astral before a brace flushed at a blockquote boundary', () => {
+ const input = '\u{1F3A3}a{\n>}rest';
+ expect(roundTrip(input)).toBe(input);
+ });
+
+ test('protect->restore identity holds for astral-bearing inputs (closes the BMP-only generator gap)', () => {
+ const astralOrBmp = fc
+ .array(
+ fc.oneof(
+ fc.constantFrom('\u{1F3A3}', '\u{1F9AD}', '\u{1F600}', '\u{1F9EA}', '\u{1F30A}'),
+ fc.constantFrom(...'abc {}>\n'),
+ ),
+ { maxLength: 60 },
+ )
+ .map((parts) => parts.join(''));
+ for (const seed of [42, 137, 2718, 31415, 99991]) {
+ fc.assert(
+ fc.property(astralOrBmp, (s) => {
+ expect(roundTrip(s)).toBe(s);
+ }),
+ { numRuns: 200, seed },
+ );
+ }
+ });
+});
diff --git a/packages/core/src/markdown/autolink-void-html-guard.ts b/packages/core/src/markdown/autolink-void-html-guard.ts
index 101069790..3dd04c233 100644
--- a/packages/core/src/markdown/autolink-void-html-guard.ts
+++ b/packages/core/src/markdown/autolink-void-html-guard.ts
@@ -297,7 +297,7 @@ export function protectFromMdx(source: string): string {
unmatchedPositions.push(...stack);
if (unmatchedPositions.length > 0) {
- const chars = [...result];
+ const chars = result.split('');
for (const pos of unmatchedPositions) {
chars[pos] = GUARD_OPEN_BRACE;
}