Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): keep two typed serializer states alive across re-derivation (#3032)
Nick Gomez6a9a6215980a
diff --git a/.changeset/serializer-seam-typed-view.md b/.changeset/serializer-seam-typed-view.mdnew file mode 100644index 000000000..c31537390--- /dev/null+++ b/.changeset/serializer-seam-typed-view.md@@ -0,0 +1,9 @@+---+'@inkeep/open-knowledge': patch+---++Stop two typed editor states from losing content on the next re-derivation.++A hard break typed at the end of a block (Shift+Enter) was dropped at serialize time, so the break vanished with no trace in the bytes and none in the view. It is now re-spelled to `<br />`, the same lossless spelling the serializer already uses inside table cells and headings.++An empty task item lost its checkbox and came back as a plain list item whose text was the literal `[ ]`, at every position in the list. GFM cannot spell a task item with no content, so an empty one now serializes with a ` ` the parse side takes back out, and the checkbox round-trips.diff --git a/packages/app/tests/dom/serializer-seam-rederive.dom.test.tsx b/packages/app/tests/dom/serializer-seam-rederive.dom.test.tsxnew file mode 100644index 000000000..d8a5eefa3--- /dev/null+++ b/packages/app/tests/dom/serializer-seam-rederive.dom.test.tsx@@ -0,0 +1,171 @@+/**+ * Two typed states that used to lose content on the next re-derivation.+ *+ * The bytes on disk were always perfect; it was the VIEW that did not survive+ * the round trip. Each family here is driven through a real mounted editor+ * with the real keymap and command chain, then put through the production+ * re-derivation — `parseWithFallback(stripFrontmatter(serialize(D)).body)`,+ * which is Observer A's serialize composed with Observer B's parse — and the+ * result is compared against what was typed.+ *+ * The `stripFrontmatter` leg is load-bearing, not decoration: Observer B parses+ * the body the intake keeps, never the raw bytes.+ *+ * The unit-level contracts live next to their producers+ * (`strip-trailing-edge` and `empty-task-item` in core). What this file adds is+ * the reachability half — that a user typing ordinary keystrokes actually+ * reaches each state.+ *+ */++import { MarkdownManager, stripFrontmatter } from '@inkeep/open-knowledge-core';+import { cleanup } from '@testing-library/react';+import { Editor, type JSONContent } from '@tiptap/core';+import { afterEach, describe, expect, test } from 'vitest';+import { sharedExtensions } from '../../src/editor/extensions/shared';++/** Configured as the server's shared manager is. */+const mdManager = new MarkdownManager({+ extensions: sharedExtensions,+ deriveStructuralFreshness: true,+});++const mounted: Editor[] = [];++afterEach(() => {+ for (const editor of mounted.splice(0)) editor.destroy();+ cleanup();+});++function mountEditor(): Editor {+ const container = document.createElement('div');+ document.body.appendChild(container);+ const editor = new Editor({ element: container, extensions: sharedExtensions, editable: true });+ mounted.push(editor);+ return editor;+}++/** Press a key through the keymap; report whether the document actually moved. */+function pressKey(editor: Editor, shortcut: string): boolean {+ const before = editor.state.doc;+ editor.commands.keyboardShortcut(shortcut);+ return editor.state.doc !== before;+}++/** Type text the way the view feeds it, so input rules get their chance. */+function type(editor: Editor, characters: string): void {+ for (const character of characters) {+ const { from, to } = editor.state.selection;+ const handled =+ editor.view.someProp('handleTextInput', (handler) =>+ (handler as (v: typeof editor.view, f: number, t: number, s: string) => boolean)(+ editor.view,+ from,+ to,+ character,+ ),+ ) ?? false;+ if (!handled) editor.view.dispatch(editor.state.tr.insertText(character));+ }+}++/** Observer A's serialize composed with Observer B's parse. */+function rederive(typed: JSONContent): JSONContent {+ return mdManager.parseWithFallback(stripFrontmatter(mdManager.serialize(typed)).body);+}++function bytes(typed: JSONContent): string {+ return mdManager.serialize(typed);+}++function countNodes(node: JSONContent, type: string): number {+ const self = node.type === type ? 1 : 0;+ return (node.content ?? []).reduce((sum, child) => sum + countNodes(child, type), self);+}++/** The `checked` attr of every item in the document's first list. */+function checkedAttrs(d: JSONContent): Array<boolean | null> {+ const items = (d.content ?? []).find((n) => n.type === 'list')?.content ?? [];+ return items.map((i) => (i.attrs?.checked ?? null) as boolean | null);+}++describe('Shift+Enter at the end of a block', () => {+ test('the break the user typed is still there after re-derivation', () => {+ const editor = mountEditor();+ type(editor, 'foo');+ expect(pressKey(editor, 'Shift-Enter')).toBe(true);++ const typed = editor.getJSON();+ expect(countNodes(typed, 'hardBreak')).toBe(1);+ expect(bytes(typed)).toBe('foo<br />\n');+ expect(countNodes(rederive(typed), 'hardBreak')).toBe(1);+ });++ test('a break typed before an existing paragraph survives too', () => {+ const editor = mountEditor();+ type(editor, 'foo');+ expect(pressKey(editor, 'Enter')).toBe(true);+ type(editor, 'bar');+ editor.commands.setTextSelection(4);+ expect(pressKey(editor, 'Shift-Enter')).toBe(true);++ const typed = editor.getJSON();+ expect(countNodes(rederive(typed), 'hardBreak')).toBe(countNodes(typed, 'hardBreak'));+ });++ test('CONTROL: a mid-paragraph break keeps the backslash spelling', () => {+ const editor = mountEditor();+ type(editor, 'foo');+ expect(pressKey(editor, 'Shift-Enter')).toBe(true);+ type(editor, 'bar');++ const typed = editor.getJSON();+ expect(bytes(typed)).toBe('foo\\\nbar\n');+ expect(countNodes(rederive(typed), 'hardBreak')).toBe(1);+ });+});++describe('an empty task item', () => {+ test('keeps its checkbox after re-derivation, at the end of a list', () => {+ const editor = mountEditor();+ expect(editor.commands.toggleTaskList()).toBe(true);++ const typed = editor.getJSON();+ expect(bytes(typed)).toContain('- [ ]  ');+ const back = rederive(typed);+ const items = back.content?.[0]?.content ?? [];+ expect(items.map((i) => i.attrs?.checked)).toEqual([false]);+ });++ test('keeps its checkbox in the MIDDLE of a list', () => {+ // Enter on an already-empty item lifts out of the list, so the empty item+ // is reached the way a user actually reaches it: build the filled items+ // first, then press Enter from the end of the first one.+ const editor = mountEditor();+ expect(editor.commands.toggleTaskList()).toBe(true);+ type(editor, 'a');+ expect(pressKey(editor, 'Enter')).toBe(true);+ type(editor, 'c');+ editor.commands.setTextSelection(3);+ expect(pressKey(editor, 'Enter')).toBe(true);++ const typed = editor.getJSON();+ const typedChecked = checkedAttrs(typed);+ expect(typedChecked).toEqual([false, false, false]);+ expect(bytes(typed)).toContain('- [ ]  ');+ expect(checkedAttrs(rederive(typed))).toEqual(typedChecked);+ });++ test('CONTROL: a task item with content is unchanged', () => {+ const editor = mountEditor();+ expect(editor.commands.toggleTaskList()).toBe(true);+ type(editor, 'buy milk');++ const typed = editor.getJSON();+ // The mounted editor appends a trailing paragraph, so this is a `contain`+ // rather than a byte equality; the unit tier pins the exact emission.+ expect(bytes(typed)).toContain('- [ ] buy milk\n');+ expect(bytes(typed)).not.toContain(' ');+ expect(checkedAttrs(rederive(typed))).toEqual([false]);+ });+});diff --git a/packages/app/tests/integration/trailing-hard-break-bridge.test.ts b/packages/app/tests/integration/trailing-hard-break-bridge.test.tsindex 88cb78adb..c6193e656 100644--- a/packages/app/tests/integration/trailing-hard-break-bridge.test.ts+++ b/packages/app/tests/integration/trailing-hard-break-bridge.test.ts@@ -7,6 +7,11 @@* failure was a split-brain where deleting the `\` from source left the* hardBreak node in the fragment and the next Observer A drain re-emitted it,* masked from the watchdog by parse-equivalence tolerance.+ *+ * The break must also SURVIVE: it reaches Y.Text as `<br />`, the one spelling+ * that survives its own re-parse. Dropping it instead would leave the+ * keystroke with no trace in the bytes and none in the view after the next+ * re-derivation. See `strip-trailing-edge.ts`.*/import { updateYFragment } from '@tiptap/y-tiptap';@@ -28,7 +33,7 @@ describe('trailing hardBreak bridge convergence', () => {server = undefined;});- test('WYSIWYG trailing hardBreak: no stray backslash, fragment and Y.Text converge', async () => {+ test('WYSIWYG trailing hardBreak: no stray backslash, survives, fragment and Y.Text converge', async () => {server = await createTestServer();const docName = `trailing-hardbreak-${crypto.randomUUID()}`;const client = await createTestClient(server.port, docName);@@ -58,7 +63,10 @@ describe('trailing hardBreak bridge convergence', () => {const fragMd = serializeFragment(client.fragment);// The stray-character symptom: no backslash may reach source bytes.- expect(ytext).toBe('hello\n');+ expect(ytext).not.toContain('\\');+ // ...and the break is not silently discarded either: it lands in the one+ // spelling that survives its own re-parse.+ expect(ytext).toBe('hello<br />\n');// The split-brain: fragment serialization must equal Y.Text, so a// subsequent Observer A drain cannot re-emit a deleted character.expect(fragMd).toBe(ytext);@@ -82,7 +90,8 @@ describe('trailing hardBreak bridge convergence', () => {await awaitDocQuiescence(client.doc);await wait(500);const ytext2 = client.ytext.toString();- expect(ytext2).toBe('hello there\n');+ expect(ytext2).not.toContain('\\');+ expect(ytext2).toBe('hello there<br />\n');expect(serializeFragment(client.fragment)).toBe(ytext2);} finally {await client.cleanup();diff --git a/packages/core/schema-snapshot.json b/packages/core/schema-snapshot.jsonindex 4d2c978da..c15b9c629 100644--- a/packages/core/schema-snapshot.json+++ b/packages/core/schema-snapshot.json@@ -944,6 +944,11 @@"version": null,"hasOptions": false},+ {+ "name": "empty-task-item-unmint",+ "version": null,+ "hasOptions": false+ },{"name": "ensure-non-empty-doc","version": null,diff --git a/packages/core/src/markdown/empty-task-item.ts b/packages/core/src/markdown/empty-task-item.tsnew file mode 100644index 000000000..70eecbfa2--- /dev/null+++ b/packages/core/src/markdown/empty-task-item.ts@@ -0,0 +1,50 @@+import type { ListItem, Nodes, Paragraph, Root, Text } from 'mdast';+import { visit } from 'unist-util-visit';++const SPACE_CHAR_REF = ' ';++function isTaskItem(node: Nodes): node is ListItem {+ return node.type === 'listItem' && typeof (node as ListItem).checked === 'boolean';+}++function isEmptyTaskItem(item: ListItem): boolean {+ const children = item.children ?? [];+ if (children.length !== 1) return false;+ const only = children[0];+ return only?.type === 'paragraph' && (only.children?.length ?? 0) === 0;+}++function isMintedEmptyTaskItem(item: ListItem): boolean {+ const children = item.children ?? [];+ if (children.length !== 1) return false;+ const only = children[0];+ if (only?.type !== 'paragraph' || (only.children?.length ?? 0) !== 1) return false;+ const text = only.children[0];+ if (text?.type !== 'text' || text.value !== SPACE_CHAR_REF) return false;+ const spans = text.data?.entityRefSpans;+ if (!Array.isArray(spans) || spans.length !== 1) return false;+ const span = spans[0];+ return span?.offset === 0 && span.length === SPACE_CHAR_REF.length && span.raw === SPACE_CHAR_REF;+}++export function mintEmptyTaskItemContent(tree: Nodes): void {+ visit(tree, 'listItem', (node: ListItem) => {+ if (!isTaskItem(node) || !isEmptyTaskItem(node)) return;+ const paragraph = node.children[0] as Paragraph;+ const minted: Text = { type: 'text', value: ' ', data: { sourceRaw: SPACE_CHAR_REF } };+ paragraph.children = [minted];+ });+}++function unmintEmptyTaskItemContent(tree: Nodes): void {+ visit(tree, 'listItem', (node: ListItem) => {+ if (!isTaskItem(node) || !isMintedEmptyTaskItem(node)) return;+ (node.children[0] as Paragraph).children = [];+ });+}++export function emptyTaskItemUnmintPlugin() {+ return (tree: Root) => {+ unmintEmptyTaskItemContent(tree);+ };+}diff --git a/packages/core/src/markdown/pipeline.ts b/packages/core/src/markdown/pipeline.tsindex 8bcbe2692..a0cfe2886 100644--- a/packages/core/src/markdown/pipeline.ts+++ b/packages/core/src/markdown/pipeline.ts@@ -23,6 +23,7 @@ import { commentPromoterPlugin } from './comment-promoter.ts';import { dedentBlockJsxClose } from './dedent-block-jsx-close.ts';import { detailsAccordionPromoterPlugin } from './details-accordion-promoter.ts';import { divAlignPromoterPlugin } from './div-align-promoter.ts';+import { emptyTaskItemUnmintPlugin, mintEmptyTaskItemContent } from './empty-task-item.ts';import { encodeEntityRefs, restoreEntityRefsPlugin } from './entity-ref-guard.ts';import { highlightPromoterPlugin } from './highlight-promoter.ts';import { imagePromoterPlugin } from './image-promoter.ts';@@ -96,6 +97,7 @@ export const ACTIVE_MDAST_PLUGINS = [{ name: 'mermaid-promoter', plugin: mermaidPromoterPlugin },{ name: 'comment-promoter', plugin: commentPromoterPlugin },{ name: 'merged-post-parse-walker', plugin: mergedPostParseWalkerPlugin },+ { name: 'empty-task-item-unmint', plugin: emptyTaskItemUnmintPlugin },{ name: 'ensure-non-empty-doc', plugin: () => ensureNonEmptyDoc },] as const;@@ -252,6 +254,8 @@ export function serializeMd(doc: PmNode, processor: Processor, opts: SerializeMdmarkHandlers: opts.pmMarkHandlers,});+ mintEmptyTaskItemContent(mdast);+stripTrailingEdge(mdast);pruneZeroEmissionBlocks(mdast);diff --git a/packages/core/src/markdown/strip-trailing-edge.ts b/packages/core/src/markdown/strip-trailing-edge.tsindex de594c528..2cd5a08ab 100644--- a/packages/core/src/markdown/strip-trailing-edge.ts+++ b/packages/core/src/markdown/strip-trailing-edge.ts@@ -1,13 +1,21 @@-import type { Nodes, Parent, RootContent, Text } from 'mdast';+import type { Break, Nodes, Parent, RootContent, Text } from 'mdast';const MARK_WRAPPERS = new Set(['strong', 'emphasis', 'delete', 'link', 'mark', 'comment']);-function isBareTrailingBreak(node: RootContent | undefined): boolean {+const HTML_BREAK_SPELLING = '<br />';++function isBareTrailingBreak(node: RootContent | undefined): node is Break {if (!node || node.type !== 'break') return false;- const sourceRaw = (node.data as { sourceRaw?: unknown } | undefined)?.sourceRaw;+ const sourceRaw = node.data?.sourceRaw;return typeof sourceRaw !== 'string' || sourceRaw.length === 0;}+function respellAsHtmlBreak(node: Break): void {+ node.data ??= {};+ node.data.sourceRaw = HTML_BREAK_SPELLING;+ node.data.sourceStyle = 'html';+}+/** True for a plain text node whose bytes are its visible value — no stored* sourceRaw provenance that re-emits different bytes. */function isPlainText(node: RootContent | undefined): node is Text {@@ -29,8 +37,8 @@ function stripTrailing(children: RootContent[], isWrapperContent = false): voidwhile (children.length > 0) {const last = children[children.length - 1];if (isBareTrailingBreak(last)) {- children.pop();- continue;+ respellAsHtmlBreak(last);+ break;}if (last && MARK_WRAPPERS.has(last.type) && 'children' in last) {const inner = (last as Parent).children as RootContent[];@@ -63,7 +71,9 @@ function stripTrailing(children: RootContent[], isWrapperContent = false): voidconst BLOCK_CONTAINERS = new Set(['paragraph', 'heading', 'tableCell']);/** In-place: canonicalize the trailing edge of every block whose phrasing- * content can end in a meaningless-at-block-edge construct. */+ * content can end in a construct whose markdown spelling is meaningless at a+ * block edge — re-spelling the ones that have a surviving form, dropping the+ * ones that do not. */export function stripTrailingEdge(tree: Nodes): void {const visit = (node: Nodes): void => {if (BLOCK_CONTAINERS.has(node.type) && 'children' in node) {