Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): stop unbounded escape amplification inside MDX expression spans (#3131)
Nick Gomez29187efc07b6
diff --git a/.changeset/mdx-expression-span-escape-amplification.md b/.changeset/mdx-expression-span-escape-amplification.mdnew file mode 100644index 000000000..5db153173--- /dev/null+++ b/.changeset/mdx-expression-span-escape-amplification.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Prose containing a brace expression — an MDX comment like `before {/* note */} after`, or any `{…}` span with a `*`, `_`, `~` or an HTML entity inside it — no longer accumulates backslashes. The serializer was escaping the inside of those spans as if they were ordinary prose, but the parser reads a matched brace pair as an opaque expression and never removes the escape, so each backslash was escaped again on the next pass: one delimiter went from `\*` to three backslashes, then seven, then fifteen, doubling forever. Because every mode toggle, reload, remote refresh and agent write re-derives the document, a paragraph could grow without anyone editing it. Those spans are now written out exactly as typed and are byte-stable from the first save. Documents that already picked up extra backslashes stop growing; their existing bytes are left alone rather than rewritten.diff --git a/packages/core/src/markdown/autolink-void-html-guard.ts b/packages/core/src/markdown/autolink-void-html-guard.tsindex 03e182587..6c1fe5fce 100644--- a/packages/core/src/markdown/autolink-void-html-guard.ts+++ b/packages/core/src/markdown/autolink-void-html-guard.ts@@ -276,6 +276,80 @@ function isUppercaseJsxSelfClosingAt(return false;}+export interface BraceSpan {+ readonly start: number;+ readonly end: number;+}++export function scanBraceSpans(+ source: string,+ options: { readonly escapeAware: boolean },+): { readonly matched: readonly BraceSpan[]; readonly unmatched: readonly number[] } {+ const unmatched = unmatchedBraceOpeners(source, options.escapeAware);+ const skip = new Set<number>(unmatched);+ const matched: BraceSpan[] = [];+ const stack: number[] = [];+ forEachBrace(source, options.escapeAware, {+ onFlush: () => {+ stack.length = 0;+ },+ onBrace: (i, char) => {+ if (skip.has(i)) return;+ if (char === '{') {+ stack.push(i);+ } else if (stack.length > 0) {+ const open = stack.pop() as number;+ if (stack.length === 0) matched.push({ start: open, end: i + 1 });+ }+ },+ });+ return { matched, unmatched };+}++function unmatchedBraceOpeners(source: string, escapeAware: boolean): number[] {+ const unmatched: number[] = [];+ const stack: number[] = [];+ forEachBrace(source, escapeAware, {+ onFlush: () => {+ unmatched.push(...stack);+ stack.length = 0;+ },+ onBrace: (i, char) => {+ if (char === '{') stack.push(i);+ else if (stack.length > 0) stack.pop();+ },+ });+ unmatched.push(...stack);+ return unmatched;+}++function forEachBrace(+ source: string,+ escapeAware: boolean,+ visitor: { onFlush: () => void; onBrace: (index: number, char: '{' | '}') => void },+): void {+ for (let i = 0; i < source.length; i++) {+ if (source[i] === '\n') {+ const next = source[i + 1];+ if (next === '\n' || next === '>') {+ visitor.onFlush();+ if (next === '\n') {+ while (source[i + 1] === '\n') i++;+ }+ continue;+ }+ }+ const char = source[i];+ if (char !== '{' && char !== '}') continue;+ if (escapeAware) {+ let bs = 0;+ for (let j = i - 1; j >= 0 && source[j] === '\\'; j--) bs++;+ if (bs % 2 === 1) continue; // escaped — not a brace remark-mdx acts on+ }+ visitor.onBrace(i, char);+ }+}+export function protectFromMdx(source: string): string {let result = source;@@ -387,33 +461,11 @@ export function protectFromMdx(source: string): string {});{- const unmatchedPositions: number[] = [];- const stack: number[] = [];- for (let i = 0; i < result.length; i++) {- if (result[i] === '\n') {- const next = result[i + 1];- if (next === '\n' || next === '>') {- unmatchedPositions.push(...stack);- stack.length = 0;- if (next === '\n') {- while (result[i + 1] === '\n') i++;- }- continue;- }- }- if (result[i] === '{' || result[i] === '}') {- let bs = 0;- for (let j = i - 1; j >= 0 && result[j] === '\\'; j--) bs++;- if (bs % 2 === 1) continue; // escaped — skip stack operations- if (result[i] === '{') stack.push(i);- else if (stack.length > 0) stack.pop(); // matched pair within same block- }- }- unmatchedPositions.push(...stack);+ const { unmatched } = scanBraceSpans(result, { escapeAware: true });- if (unmatchedPositions.length > 0) {+ if (unmatched.length > 0) {const chars = result.split('');- for (const pos of unmatchedPositions) {+ for (const pos of unmatched) {chars[pos] = GUARD_OPEN_BRACE;}result = chars.join('');diff --git a/packages/core/src/markdown/to-markdown-handlers.ts b/packages/core/src/markdown/to-markdown-handlers.tsindex 6000891e3..a41534404 100644--- a/packages/core/src/markdown/to-markdown-handlers.ts+++ b/packages/core/src/markdown/to-markdown-handlers.ts@@ -3,6 +3,7 @@ import type { MdxJsxAttribute, MdxJsxExpressionAttribute, MdxJsxFlowElement } frimport type { Handle, Info, State } from 'mdast-util-to-markdown';import { classifyCharacter } from 'micromark-util-classify-character';import { isValidSourceLiteralRaw } from '../extensions/source-literal-mark.ts';+import { scanBraceSpans } from './autolink-void-html-guard.ts';import { widenFenceLength } from './code-fence.ts';import { TO_MARKDOWN_EXT } from './remark-mdx-agnostic.ts';import { isInlineWhitespaceNumericCharRef } from './whitespace-char-ref.ts';@@ -860,7 +861,39 @@ function markTypedInlineWhitespaceRefs(value: string): string {);}+function emitsAtLineStart(emittedPrefix: string): boolean {+ return /[\r\n][\t ]*$/.test(emittedPrefix);+}+function safeText(state: State, value: string, info: Info): string {+ const { matched } = scanBraceSpans(value, { escapeAware: false });+ if (matched.length === 0) return safeTextSegment(state, value, info);++ let result = '';+ let pending = 0;++ const flushNormal = (end: number): void => {+ if (end > pending) {+ result += safeTextSegment(state, value.slice(pending, end), {+ ...info,+ before: info.before + result,+ after: value.slice(end, end + 1) || info.after,+ });+ }+ pending = end;+ };++ for (const span of matched) {+ flushNormal(span.start);+ if (emitsAtLineStart(info.before + result)) continue;+ result += value.slice(span.start, span.end);+ pending = span.end;+ }+ flushNormal(value.length);+ return result;+}++function safeTextSegment(state: State, value: string, info: Info): string {const originalUnsafe = state.unsafe;state.unsafe = originalUnsafe.filter((u) => {if (u.character === '&' && u.after === '[#A-Za-z]') return false;