Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): keep two typed serializer states alive across re-derivation (#3032)

Nick Gomez6a9a6215980a
diff --git a/.changeset/serializer-seam-typed-view.md b/.changeset/serializer-seam-typed-view.md
new file mode 100644
index 000000000..c31537390
--- /dev/null
+++ b/.changeset/serializer-seam-typed-view.md
@@ -0,0 +1,9 @@
+---
+'@inkeep/open-knowledge': patch
+---
+
+Stop two typed editor states from losing content on the next re-derivation.
+
+A hard break typed at the end of a block (Shift+Enter) was dropped at serialize time, so the break vanished with no trace in the bytes and none in the view. It is now re-spelled to `<br />`, the same lossless spelling the serializer already uses inside table cells and headings.
+
+An empty task item lost its checkbox and came back as a plain list item whose text was the literal `[ ]`, at every position in the list. GFM cannot spell a task item with no content, so an empty one now serializes with a `&#x20;` the parse side takes back out, and the checkbox round-trips.
diff --git a/packages/app/tests/dom/serializer-seam-rederive.dom.test.tsx b/packages/app/tests/dom/serializer-seam-rederive.dom.test.tsx
new file mode 100644
index 000000000..d8a5eefa3
--- /dev/null
+++ b/packages/app/tests/dom/serializer-seam-rederive.dom.test.tsx
@@ -0,0 +1,171 @@
+/**
+ * Two typed states that used to lose content on the next re-derivation.
+ *
+ * The bytes on disk were always perfect; it was the VIEW that did not survive
+ * the round trip. Each family here is driven through a real mounted editor
+ * with the real keymap and command chain, then put through the production
+ * re-derivation — `parseWithFallback(stripFrontmatter(serialize(D)).body)`,
+ * which is Observer A's serialize composed with Observer B's parse — and the
+ * result is compared against what was typed.
+ *
+ * The `stripFrontmatter` leg is load-bearing, not decoration: Observer B parses
+ * the body the intake keeps, never the raw bytes.
+ *
+ * The unit-level contracts live next to their producers
+ * (`strip-trailing-edge` and `empty-task-item` in core). What this file adds is
+ * the reachability half — that a user typing ordinary keystrokes actually
+ * reaches each state.
+ *
+ */
+
+import { MarkdownManager, stripFrontmatter } from '@inkeep/open-knowledge-core';
+import { cleanup } from '@testing-library/react';
+import { Editor, type JSONContent } from '@tiptap/core';
+import { afterEach, describe, expect, test } from 'vitest';
+import { sharedExtensions } from '../../src/editor/extensions/shared';
+
+/** Configured as the server's shared manager is. */
+const mdManager = new MarkdownManager({
+ extensions: sharedExtensions,
+ deriveStructuralFreshness: true,
+});
+
+const mounted: Editor[] = [];
+
+afterEach(() => {
+ for (const editor of mounted.splice(0)) editor.destroy();
+ cleanup();
+});
+
+function mountEditor(): Editor {
+ const container = document.createElement('div');
+ document.body.appendChild(container);
+ const editor = new Editor({ element: container, extensions: sharedExtensions, editable: true });
+ mounted.push(editor);
+ return editor;
+}
+
+/** Press a key through the keymap; report whether the document actually moved. */
+function pressKey(editor: Editor, shortcut: string): boolean {
+ const before = editor.state.doc;
+ editor.commands.keyboardShortcut(shortcut);
+ return editor.state.doc !== before;
+}
+
+/** Type text the way the view feeds it, so input rules get their chance. */
+function type(editor: Editor, characters: string): void {
+ for (const character of characters) {
+ const { from, to } = editor.state.selection;
+ const handled =
+ editor.view.someProp('handleTextInput', (handler) =>
+ (handler as (v: typeof editor.view, f: number, t: number, s: string) => boolean)(
+ editor.view,
+ from,
+ to,
+ character,
+ ),
+ ) ?? false;
+ if (!handled) editor.view.dispatch(editor.state.tr.insertText(character));
+ }
+}
+
+/** Observer A's serialize composed with Observer B's parse. */
+function rederive(typed: JSONContent): JSONContent {
+ return mdManager.parseWithFallback(stripFrontmatter(mdManager.serialize(typed)).body);
+}
+
+function bytes(typed: JSONContent): string {
+ return mdManager.serialize(typed);
+}
+
+function countNodes(node: JSONContent, type: string): number {
+ const self = node.type === type ? 1 : 0;
+ return (node.content ?? []).reduce((sum, child) => sum + countNodes(child, type), self);
+}
+
+/** The `checked` attr of every item in the document's first list. */
+function checkedAttrs(d: JSONContent): Array<boolean | null> {
+ const items = (d.content ?? []).find((n) => n.type === 'list')?.content ?? [];
+ return items.map((i) => (i.attrs?.checked ?? null) as boolean | null);
+}
+
+describe('Shift+Enter at the end of a block', () => {
+ test('the break the user typed is still there after re-derivation', () => {
+ const editor = mountEditor();
+ type(editor, 'foo');
+ expect(pressKey(editor, 'Shift-Enter')).toBe(true);
+
+ const typed = editor.getJSON();
+ expect(countNodes(typed, 'hardBreak')).toBe(1);
+ expect(bytes(typed)).toBe('foo<br />\n');
+ expect(countNodes(rederive(typed), 'hardBreak')).toBe(1);
+ });
+
+ test('a break typed before an existing paragraph survives too', () => {
+ const editor = mountEditor();
+ type(editor, 'foo');
+ expect(pressKey(editor, 'Enter')).toBe(true);
+ type(editor, 'bar');
+ editor.commands.setTextSelection(4);
+ expect(pressKey(editor, 'Shift-Enter')).toBe(true);
+
+ const typed = editor.getJSON();
+ expect(countNodes(rederive(typed), 'hardBreak')).toBe(countNodes(typed, 'hardBreak'));
+ });
+
+ test('CONTROL: a mid-paragraph break keeps the backslash spelling', () => {
+ const editor = mountEditor();
+ type(editor, 'foo');
+ expect(pressKey(editor, 'Shift-Enter')).toBe(true);
+ type(editor, 'bar');
+
+ const typed = editor.getJSON();
+ expect(bytes(typed)).toBe('foo\\\nbar\n');
+ expect(countNodes(rederive(typed), 'hardBreak')).toBe(1);
+ });
+});
+
+describe('an empty task item', () => {
+ test('keeps its checkbox after re-derivation, at the end of a list', () => {
+ const editor = mountEditor();
+ expect(editor.commands.toggleTaskList()).toBe(true);
+
+ const typed = editor.getJSON();
+ expect(bytes(typed)).toContain('- [ ] &#x20;');
+ const back = rederive(typed);
+ const items = back.content?.[0]?.content ?? [];
+ expect(items.map((i) => i.attrs?.checked)).toEqual([false]);
+ });
+
+ test('keeps its checkbox in the MIDDLE of a list', () => {
+ // Enter on an already-empty item lifts out of the list, so the empty item
+ // is reached the way a user actually reaches it: build the filled items
+ // first, then press Enter from the end of the first one.
+ const editor = mountEditor();
+ expect(editor.commands.toggleTaskList()).toBe(true);
+ type(editor, 'a');
+ expect(pressKey(editor, 'Enter')).toBe(true);
+ type(editor, 'c');
+ editor.commands.setTextSelection(3);
+ expect(pressKey(editor, 'Enter')).toBe(true);
+
+ const typed = editor.getJSON();
+ const typedChecked = checkedAttrs(typed);
+ expect(typedChecked).toEqual([false, false, false]);
+ expect(bytes(typed)).toContain('- [ ] &#x20;');
+ expect(checkedAttrs(rederive(typed))).toEqual(typedChecked);
+ });
+
+ test('CONTROL: a task item with content is unchanged', () => {
+ const editor = mountEditor();
+ expect(editor.commands.toggleTaskList()).toBe(true);
+ type(editor, 'buy milk');
+
+ const typed = editor.getJSON();
+ // The mounted editor appends a trailing paragraph, so this is a `contain`
+ // rather than a byte equality; the unit tier pins the exact emission.
+ expect(bytes(typed)).toContain('- [ ] buy milk\n');
+ expect(bytes(typed)).not.toContain('&#x20;');
+ expect(checkedAttrs(rederive(typed))).toEqual([false]);
+ });
+});
diff --git a/packages/app/tests/integration/trailing-hard-break-bridge.test.ts b/packages/app/tests/integration/trailing-hard-break-bridge.test.ts
index 88cb78adb..c6193e656 100644
--- a/packages/app/tests/integration/trailing-hard-break-bridge.test.ts
+++ b/packages/app/tests/integration/trailing-hard-break-bridge.test.ts
@@ -7,6 +7,11 @@
* failure was a split-brain where deleting the `\` from source left the
* hardBreak node in the fragment and the next Observer A drain re-emitted it,
* masked from the watchdog by parse-equivalence tolerance.
+ *
+ * The break must also SURVIVE: it reaches Y.Text as `<br />`, the one spelling
+ * that survives its own re-parse. Dropping it instead would leave the
+ * keystroke with no trace in the bytes and none in the view after the next
+ * re-derivation. See `strip-trailing-edge.ts`.
*/
import { updateYFragment } from '@tiptap/y-tiptap';
@@ -28,7 +33,7 @@ describe('trailing hardBreak bridge convergence', () => {
server = undefined;
});
- test('WYSIWYG trailing hardBreak: no stray backslash, fragment and Y.Text converge', async () => {
+ test('WYSIWYG trailing hardBreak: no stray backslash, survives, fragment and Y.Text converge', async () => {
server = await createTestServer();
const docName = `trailing-hardbreak-${crypto.randomUUID()}`;
const client = await createTestClient(server.port, docName);
@@ -58,7 +63,10 @@ describe('trailing hardBreak bridge convergence', () => {
const fragMd = serializeFragment(client.fragment);
// The stray-character symptom: no backslash may reach source bytes.
- expect(ytext).toBe('hello\n');
+ expect(ytext).not.toContain('\\');
+ // ...and the break is not silently discarded either: it lands in the one
+ // spelling that survives its own re-parse.
+ expect(ytext).toBe('hello<br />\n');
// The split-brain: fragment serialization must equal Y.Text, so a
// subsequent Observer A drain cannot re-emit a deleted character.
expect(fragMd).toBe(ytext);
@@ -82,7 +90,8 @@ describe('trailing hardBreak bridge convergence', () => {
await awaitDocQuiescence(client.doc);
await wait(500);
const ytext2 = client.ytext.toString();
- expect(ytext2).toBe('hello there\n');
+ expect(ytext2).not.toContain('\\');
+ expect(ytext2).toBe('hello there<br />\n');
expect(serializeFragment(client.fragment)).toBe(ytext2);
} finally {
await client.cleanup();
diff --git a/packages/core/schema-snapshot.json b/packages/core/schema-snapshot.json
index 4d2c978da..c15b9c629 100644
--- a/packages/core/schema-snapshot.json
+++ b/packages/core/schema-snapshot.json
@@ -944,6 +944,11 @@
"version": null,
"hasOptions": false
},
+ {
+ "name": "empty-task-item-unmint",
+ "version": null,
+ "hasOptions": false
+ },
{
"name": "ensure-non-empty-doc",
"version": null,
diff --git a/packages/core/src/markdown/empty-task-item.ts b/packages/core/src/markdown/empty-task-item.ts
new file mode 100644
index 000000000..70eecbfa2
--- /dev/null
+++ b/packages/core/src/markdown/empty-task-item.ts
@@ -0,0 +1,50 @@
+import type { ListItem, Nodes, Paragraph, Root, Text } from 'mdast';
+import { visit } from 'unist-util-visit';
+
+const SPACE_CHAR_REF = '&#x20;';
+
+function isTaskItem(node: Nodes): node is ListItem {
+ return node.type === 'listItem' && typeof (node as ListItem).checked === 'boolean';
+}
+
+function isEmptyTaskItem(item: ListItem): boolean {
+ const children = item.children ?? [];
+ if (children.length !== 1) return false;
+ const only = children[0];
+ return only?.type === 'paragraph' && (only.children?.length ?? 0) === 0;
+}
+
+function isMintedEmptyTaskItem(item: ListItem): boolean {
+ const children = item.children ?? [];
+ if (children.length !== 1) return false;
+ const only = children[0];
+ if (only?.type !== 'paragraph' || (only.children?.length ?? 0) !== 1) return false;
+ const text = only.children[0];
+ if (text?.type !== 'text' || text.value !== SPACE_CHAR_REF) return false;
+ const spans = text.data?.entityRefSpans;
+ if (!Array.isArray(spans) || spans.length !== 1) return false;
+ const span = spans[0];
+ return span?.offset === 0 && span.length === SPACE_CHAR_REF.length && span.raw === SPACE_CHAR_REF;
+}
+
+export function mintEmptyTaskItemContent(tree: Nodes): void {
+ visit(tree, 'listItem', (node: ListItem) => {
+ if (!isTaskItem(node) || !isEmptyTaskItem(node)) return;
+ const paragraph = node.children[0] as Paragraph;
+ const minted: Text = { type: 'text', value: ' ', data: { sourceRaw: SPACE_CHAR_REF } };
+ paragraph.children = [minted];
+ });
+}
+
+function unmintEmptyTaskItemContent(tree: Nodes): void {
+ visit(tree, 'listItem', (node: ListItem) => {
+ if (!isTaskItem(node) || !isMintedEmptyTaskItem(node)) return;
+ (node.children[0] as Paragraph).children = [];
+ });
+}
+
+export function emptyTaskItemUnmintPlugin() {
+ return (tree: Root) => {
+ unmintEmptyTaskItemContent(tree);
+ };
+}
diff --git a/packages/core/src/markdown/pipeline.ts b/packages/core/src/markdown/pipeline.ts
index 8bcbe2692..a0cfe2886 100644
--- a/packages/core/src/markdown/pipeline.ts
+++ b/packages/core/src/markdown/pipeline.ts
@@ -23,6 +23,7 @@ import { commentPromoterPlugin } from './comment-promoter.ts';
import { dedentBlockJsxClose } from './dedent-block-jsx-close.ts';
import { detailsAccordionPromoterPlugin } from './details-accordion-promoter.ts';
import { divAlignPromoterPlugin } from './div-align-promoter.ts';
+import { emptyTaskItemUnmintPlugin, mintEmptyTaskItemContent } from './empty-task-item.ts';
import { encodeEntityRefs, restoreEntityRefsPlugin } from './entity-ref-guard.ts';
import { highlightPromoterPlugin } from './highlight-promoter.ts';
import { imagePromoterPlugin } from './image-promoter.ts';
@@ -96,6 +97,7 @@ export const ACTIVE_MDAST_PLUGINS = [
{ name: 'mermaid-promoter', plugin: mermaidPromoterPlugin },
{ name: 'comment-promoter', plugin: commentPromoterPlugin },
{ name: 'merged-post-parse-walker', plugin: mergedPostParseWalkerPlugin },
+ { name: 'empty-task-item-unmint', plugin: emptyTaskItemUnmintPlugin },
{ name: 'ensure-non-empty-doc', plugin: () => ensureNonEmptyDoc },
] as const;
@@ -252,6 +254,8 @@ export function serializeMd(doc: PmNode, processor: Processor, opts: SerializeMd
markHandlers: opts.pmMarkHandlers,
});
+ mintEmptyTaskItemContent(mdast);
+
stripTrailingEdge(mdast);
pruneZeroEmissionBlocks(mdast);
diff --git a/packages/core/src/markdown/strip-trailing-edge.ts b/packages/core/src/markdown/strip-trailing-edge.ts
index de594c528..2cd5a08ab 100644
--- a/packages/core/src/markdown/strip-trailing-edge.ts
+++ b/packages/core/src/markdown/strip-trailing-edge.ts
@@ -1,13 +1,21 @@
-import type { Nodes, Parent, RootContent, Text } from 'mdast';
+import type { Break, Nodes, Parent, RootContent, Text } from 'mdast';
const MARK_WRAPPERS = new Set(['strong', 'emphasis', 'delete', 'link', 'mark', 'comment']);
-function isBareTrailingBreak(node: RootContent | undefined): boolean {
+const HTML_BREAK_SPELLING = '<br />';
+
+function isBareTrailingBreak(node: RootContent | undefined): node is Break {
if (!node || node.type !== 'break') return false;
- const sourceRaw = (node.data as { sourceRaw?: unknown } | undefined)?.sourceRaw;
+ const sourceRaw = node.data?.sourceRaw;
return typeof sourceRaw !== 'string' || sourceRaw.length === 0;
}
+function respellAsHtmlBreak(node: Break): void {
+ node.data ??= {};
+ node.data.sourceRaw = HTML_BREAK_SPELLING;
+ node.data.sourceStyle = 'html';
+}
+
/** True for a plain text node whose bytes are its visible value — no stored
* sourceRaw provenance that re-emits different bytes. */
function isPlainText(node: RootContent | undefined): node is Text {
@@ -29,8 +37,8 @@ function stripTrailing(children: RootContent[], isWrapperContent = false): void
while (children.length > 0) {
const last = children[children.length - 1];
if (isBareTrailingBreak(last)) {
- children.pop();
- continue;
+ respellAsHtmlBreak(last);
+ break;
}
if (last && MARK_WRAPPERS.has(last.type) && 'children' in last) {
const inner = (last as Parent).children as RootContent[];
@@ -63,7 +71,9 @@ function stripTrailing(children: RootContent[], isWrapperContent = false): void
const BLOCK_CONTAINERS = new Set(['paragraph', 'heading', 'tableCell']);
/** In-place: canonicalize the trailing edge of every block whose phrasing
- * content can end in a meaningless-at-block-edge construct. */
+ * content can end in a construct whose markdown spelling is meaningless at a
+ * block edge — re-spelling the ones that have a surviving form, dropping the
+ * ones that do not. */
export function stripTrailingEdge(tree: Nodes): void {
const visit = (node: Nodes): void => {
if (BLOCK_CONTAINERS.has(node.type) && 'children' in node) {