Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): keep raw JSX fallback bytes verbatim (PRD-8272) (#3841)
Andrew Mikofalvyda3395833c5f
diff --git a/.changeset/jsx-fallback-child-respell.md b/.changeset/jsx-fallback-child-respell.mdnew file mode 100644index 000000000..821f40357--- /dev/null+++ b/.changeset/jsx-fallback-child-respell.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Typing in source mode no longer re-indents, duplicates, or truncates a `<Steps>` block you never touched. When one nested block inside a JSX container stopped parsing, the editor re-emitted that block's raw bytes indented two spaces per level while its siblings stayed flush-left, and the server then wrote that mixed spelling over your authored source. Landing while a keystroke was still in flight, the two spellings of the same block merged into each other: a `<Step>` block appeared twice, a line lost its tail, and the document stopped parsing. The raw bytes of a block the parser could not read are now emitted exactly as authored, so there is no second spelling to merge against.diff --git a/packages/core/src/markdown/to-markdown-handlers.ts b/packages/core/src/markdown/to-markdown-handlers.tsindex 3e79f8cb6..b50bea5ec 100644--- a/packages/core/src/markdown/to-markdown-handlers.ts+++ b/packages/core/src/markdown/to-markdown-handlers.ts@@ -5,6 +5,7 @@ import { classifyCharacter } from 'micromark-util-classify-character';import { isValidSourceLiteralRaw } from '../extensions/source-literal-mark.ts';import { scanBraceSpans } from './autolink-void-html-guard.ts';import { widenFenceLength } from './code-fence.ts';+import type { RawMdxFallbackMdast } from './mdast-augmentation.ts';import { TO_MARKDOWN_EXT } from './remark-mdx-agnostic.ts';import { isInlineWhitespaceNumericCharRef } from './whitespace-char-ref.ts';@@ -52,6 +53,20 @@ const upstreamMdxJsxFlowHandler: Handle = (() => {const LAZY_UNSAFE_LINE_RE =/^[ \t]*(?:$|#{1,6}(?:[ \t]|[ \t]*$)|>|(?:[-+*]|\d{1,9}[.)])[ \t]|`{3,}|~{3,}|=+[ \t]*$|-+[ \t]*$|(?:\*[ \t]*){3,}$|(?:_[ \t]*){3,}$)/;+type JsxFlowChild = MdxJsxFlowElement['children'][number];++function keepVerbatimChildUnindented(child: JsxFlowChild): JsxFlowChild {+ const fallback = child as unknown as RawMdxFallbackMdast;+ if (fallback.type !== 'rawMdxFallback') return child;+ return {+ type: 'mdxJsxFlowElement',+ name: null,+ attributes: [],+ children: [],+ data: { sourceRaw: fallback.value ?? '' },+ } as unknown as JsxFlowChild;+}+export const toMarkdownHandlers = {text(node, _parent, state, info) {if (typeof node.data?.sourceRaw === 'string') {@@ -616,6 +631,7 @@ export const toMarkdownHandlers = {const delegated: MdxJsxFlowElement = {...mdxNode,attributes: (mdxNode.attributes ?? []).map(normalizeAttrForUpstream),+ children: (mdxNode.children ?? []).map(keepVerbatimChildUnindented),};return upstreamMdxJsxFlowHandler(delegated, parent, state, info);},diff --git a/packages/server/src/observer-a-verbatim-fallback-respell.test.ts b/packages/server/src/observer-a-verbatim-fallback-respell.test.tsnew file mode 100644index 000000000..c7d9ab41b--- /dev/null+++ b/packages/server/src/observer-a-verbatim-fallback-respell.test.ts@@ -0,0 +1,150 @@+/**+ * Observer A must not rewrite authoritative `Y.Text` with a RE-SPELLED copy of+ * a block the user never edited.+ *+ * When a nested block inside a JSX container degrades to a verbatim-bytes+ * `rawMdxFallback` — the shape the editor holds whenever a `<Step>` inside+ * `<Steps>` stops parsing under it — the container goes onto the reconstruct+ * path. `mdast-util-mdx-jsx`'s `containerFlow` exempts only `mdxJsxFlowElement`+ * children from its depth indentation, so the fallback's raw bytes came back+ * indented two spaces per JSX level while its pristine siblings stayed+ * flush-left. Observer A treats that mixed spelling as the fragment's canonical+ * bytes and writes it over the authored source.+ *+ * The drain shape is what makes it reachable. On a quiet drain the respell is+ * swallowed: `normalizeBridge` tolerates both the container-boundary blanks and+ * the child indentation, so the in-sync gate certifies and nothing is written.+ * It escapes on a DUAL drain — a fragment change landing in the same+ * transaction as a source keystroke — where the real delta denies that gate and+ * the router rewrites the whole block. That is source-mode typing while the+ * hidden-but-mounted WYSIWYG mutates the fragment, so it is the everyday shape,+ * not a corner. The write is a whole-block multi-line delta, which is how a+ * burst still in flight ends up merged against a second spelling of the same+ * span.+ *+ * The assertion is on the settled `Y.Text` because that is the authoritative+ * source persisted to disk and converged to every peer (precedent #38).+ *+ * Deterministic by construction: no wall-clock race is staged, and `NODE_ENV`+ * is the packaged posture because the producer guard throws under a test+ * runtime and would abort the very write this test exists to observe.+ */++import { sharedExtensions } from '@inkeep/open-knowledge-core';+import { getSchema, type JSONContent } from '@tiptap/core';+import { updateYFragment } from '@tiptap/y-tiptap';+import { afterEach, beforeEach, expect, test, vi } from 'vitest';+import { type BridgeRaceRig, createBridgeRaceRig } from './bridge-race-rig.test-helper.ts';+import { mdManager } from './md-manager.ts';++const schema = getSchema(sharedExtensions);++/** A JSX container with flush-left nested blocks, as a user authors it. */+const STEPS = [+ '<Steps>',+ '',+ '<Step>',+ '',+ 'Content one.',+ '',+ '</Step>',+ '',+ '<Step>',+ '',+ 'Content two.',+ '',+ '</Step>',+ '',+ '</Steps>',+ '',+].join('\n');++/** A leading-whitespace `<Step>` / `</Step>` tag — the respell's signature. */+const INDENTED_STEP = /\n[ \t]+<\/?Step\b/;++const CLIENT_ORIGIN = 'observer-a-verbatim-fallback-respell/client';++let rig: BridgeRaceRig;++beforeEach(() => {+ vi.stubEnv('NODE_ENV', 'production');+ vi.useFakeTimers({ toFake: ['Date'] });+ rig = createBridgeRaceRig({ docName: 'verbatim-fallback-respell' });+});++afterEach(() => {+ rig?.cleanup();+ vi.useRealTimers();+ vi.unstubAllEnvs();+});++/** Parse `STEPS`, then degrade the nested block at `index` into the+ * verbatim-bytes fallback the editor holds for a block it cannot read. */+function fragmentWithDegradedStep(index: number, raw: string): JSONContent {+ const json = mdManager.parse(STEPS) as JSONContent;+ const container = json.content?.[0];+ const children = container?.content;+ if (!container || !children) throw new Error('fixture parse did not yield a JSX container');+ children[index] = {+ type: 'rawMdxFallback',+ attrs: { reason: 'Unregistered component: Step', originalSpan: null },+ content: [{ type: 'text', text: raw }],+ };+ container.attrs = { ...container.attrs, sourceDirty: true };+ return json;+}++/** One dual drain: the degraded fragment and a source keystroke land in the+ * same transaction, exactly as they do when a user types in source mode while+ * the hidden WYSIWYG rewrites the fragment underneath. */+function degradeWhileTyping(json: JSONContent, typeAfter: string, char: string): void {+ const at = rig.ytext.toString().indexOf(typeAfter) + typeAfter.length;+ rig.stimulus('degrade-while-typing', () => {+ rig.doc.transact(() => {+ updateYFragment(rig.doc, rig.xmlFragment, schema.nodeFromJSON(json), {+ mapping: new Map(),+ isOMark: new Map(),+ });+ rig.ytext.insert(at, char);+ }, CLIENT_ORIGIN);+ });+}++test('a degraded nested block does not re-indent the authored source in Y.Text', () => {+ rig.seedSource(STEPS);+ expect(rig.ytext.toString()).toBe(STEPS);++ degradeWhileTyping(+ fragmentWithDegradedStep(1, '<Step>\n\nContent two.\n\n</Step>'),+ 'Content one.',+ 'Z',+ );+ rig.settle(3);++ const settled = rig.ytext.toString();+ expect(settled).not.toMatch(INDENTED_STEP);+ // The block the user never touched keeps its authored bytes, character for+ // character — a fallback holds source OK could not read, so a respell of it+ // is not re-derivable.+ expect(settled).toContain('<Step>\n\nContent two.\n\n</Step>');+ // The keystroke still lands: declining to respell must not decline the edit.+ expect(settled).toContain('Content one.Z');+ expect((settled.match(/<Step>/g) ?? []).length).toBe(2);+ expect((settled.match(/<\/Step>/g) ?? []).length).toBe(2);+});++test('the authored bytes survive a degraded FIRST nested block', () => {+ rig.seedSource(STEPS);++ degradeWhileTyping(+ fragmentWithDegradedStep(0, '<Step>\n\nContent one.\n\n</Step>'),+ 'Content two.',+ 'Z',+ );+ rig.settle(3);++ const settled = rig.ytext.toString();+ expect(settled).not.toMatch(INDENTED_STEP);+ expect(settled).toContain('Content one.');+ expect(settled).toContain('Content two.Z');+});