Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
test(open-knowledge): editor-construct fixed-point sweep gate (#2746)
Nick Gomezfe1dafc4db33
diff --git a/.changeset/trailing-whitespace-fixed-point.md b/.changeset/trailing-whitespace-fixed-point.mdnew file mode 100644index 000000000..ed88b66a5--- /dev/null+++ b/.changeset/trailing-whitespace-fixed-point.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Trailing spaces or tabs at the end of a paragraph, heading, or table cell no longer round-trip through an unstable intermediate form. Typing a trailing space in the editor used to serialize the space byte, which the next parse silently discarded — an avoidable divergence between the WYSIWYG view and the stored source that cost an extra canonicalization cycle on every save/load. The serializer now strips insignificant line-final whitespace up front (the exact canonicalization the parser already applies to files on disk), so the first emission is a stable fixed point. Significant whitespace is untouched: boundary spaces inside bold/italic/strike/highlight still mint their ` ` character references, mid-paragraph hard breaks of both markdown styles stay byte-exact, non-breaking spaces survive, and whitespace-only paragraphs keep their minted character reference. A new editor-construct fixed-point sweep now guards this whole class: every editor-constructible document shape must serialize to bytes that are their own re-parse fixed point, with an explicit fail-closed allowlist for documented benign canonicalizations.diff --git a/packages/core/src/markdown/pipeline.ts b/packages/core/src/markdown/pipeline.tsindex 452c91a34..7f2d5adca 100644--- a/packages/core/src/markdown/pipeline.ts+++ b/packages/core/src/markdown/pipeline.ts@@ -33,7 +33,7 @@ import { mermaidPromoterPlugin } from './mermaid-promoter.ts';import { positionAwareBlankLineJoin } from './position-aware-join.ts';import { remarkMdxAgnostic } from './remark-mdx-agnostic.ts';import { singleDollarMathPromoterPlugin } from './single-dollar-math-promoter.ts';-import { stripTrailingHardBreaks } from './strip-trailing-hard-break.ts';+import { stripTrailingEdge } from './strip-trailing-edge.ts';import { remarkTags } from './tag-to-markdown.ts';import { voidBrPromoterPlugin } from './void-br-promoter.ts';import { remarkWikiLink } from './wiki-link-micromark.ts';@@ -269,7 +269,7 @@ export function serializeMd(doc: PmNode, processor: Processor, opts: SerializeMdmarkHandlers: opts.pmMarkHandlers,});- stripTrailingHardBreaks(mdast);+ stripTrailingEdge(mdast);const boundary = readDocBoundary(doc.attrs?.sourceDocBoundary);const gaps = boundary?.gapBlankLines;diff --git a/packages/core/src/markdown/strip-trailing-edge.ts b/packages/core/src/markdown/strip-trailing-edge.tsnew file mode 100644index 000000000..de594c528--- /dev/null+++ b/packages/core/src/markdown/strip-trailing-edge.ts@@ -0,0 +1,77 @@+import type { Nodes, Parent, RootContent, Text } from 'mdast';++const MARK_WRAPPERS = new Set(['strong', 'emphasis', 'delete', 'link', 'mark', 'comment']);++function isBareTrailingBreak(node: RootContent | undefined): boolean {+ if (!node || node.type !== 'break') return false;+ const sourceRaw = (node.data as { sourceRaw?: unknown } | undefined)?.sourceRaw;+ return typeof sourceRaw !== 'string' || sourceRaw.length === 0;+}++/** True for a plain text node whose bytes are its visible value — no stored+ * sourceRaw provenance that re-emits different bytes. */+function isPlainText(node: RootContent | undefined): node is Text {+ if (!node || node.type !== 'text') return false;+ const sourceRaw = (node.data as { sourceRaw?: unknown } | undefined)?.sourceRaw;+ return typeof sourceRaw !== 'string' || sourceRaw.length === 0;+}++const TRAILING_INSIGNIFICANT_WS = /[ \t]+$/;+const WS_ONLY = /^[ \t]*$/;++/** True when the block still holds content other than insignificant-whitespace+ * text — the anchor that licenses dropping a whitespace-only tail node. */+function hasNonWhitespaceContent(children: RootContent[]): boolean {+ return children.some((child) => !(isPlainText(child) && WS_ONLY.test(child.value)));+}++function stripTrailing(children: RootContent[], isWrapperContent = false): void {+ while (children.length > 0) {+ const last = children[children.length - 1];+ if (isBareTrailingBreak(last)) {+ children.pop();+ continue;+ }+ if (last && MARK_WRAPPERS.has(last.type) && 'children' in last) {+ const inner = (last as Parent).children as RootContent[];+ stripTrailing(inner, true);+ if (inner.length === 0) {+ children.pop();+ continue;+ }+ }+ if (!isWrapperContent && isPlainText(last) && TRAILING_INSIGNIFICANT_WS.test(last.value)) {+ const trimmed = last.value.replace(TRAILING_INSIGNIFICANT_WS, '');+ if (trimmed.length > 0) {+ last.value = trimmed;+ continue;+ }+ if (hasNonWhitespaceContent(children.slice(0, -1))) {+ children.pop();+ continue;+ }+ if (children.every((child) => isPlainText(child) && WS_ONLY.test(child.value))) {+ const first = children[0] as Text;+ first.value = first.value.charAt(0);+ children.splice(1);+ }+ }+ break;+ }+}++const BLOCK_CONTAINERS = new Set(['paragraph', 'heading', 'tableCell']);++/** In-place: canonicalize the trailing edge of every block whose phrasing+ * content can end in a meaningless-at-block-edge construct. */+export function stripTrailingEdge(tree: Nodes): void {+ const visit = (node: Nodes): void => {+ if (BLOCK_CONTAINERS.has(node.type) && 'children' in node) {+ stripTrailing((node as Parent).children as RootContent[]);+ }+ if ('children' in node) {+ for (const child of (node as Parent).children as Nodes[]) visit(child);+ }+ };+ visit(tree);+}diff --git a/packages/core/src/markdown/strip-trailing-hard-break.ts b/packages/core/src/markdown/strip-trailing-hard-break.tsdeleted file mode 100644index 54507b8d6..000000000--- a/packages/core/src/markdown/strip-trailing-hard-break.ts+++ /dev/null@@ -1,46 +0,0 @@-import type { Nodes, Parent, RootContent } from 'mdast';--const MARK_WRAPPERS = new Set(['strong', 'emphasis', 'delete', 'link', 'mark', 'comment']);--function isBareTrailingBreak(node: RootContent | undefined): boolean {- if (!node || node.type !== 'break') return false;- const sourceRaw = (node.data as { sourceRaw?: unknown } | undefined)?.sourceRaw;- return typeof sourceRaw !== 'string' || sourceRaw.length === 0;-}--/** Remove trailing bare breaks from a phrasing-children array, recursing into- * trailing mark wrappers and pruning any wrapper the removal empties. */-function stripTrailing(children: RootContent[]): void {- while (children.length > 0) {- const last = children[children.length - 1];- if (isBareTrailingBreak(last)) {- children.pop();- continue;- }- if (last && MARK_WRAPPERS.has(last.type) && 'children' in last) {- const inner = (last as Parent).children as RootContent[];- stripTrailing(inner);- if (inner.length === 0) {- children.pop();- continue;- }- }- break;- }-}--const BLOCK_CONTAINERS = new Set(['paragraph', 'heading', 'tableCell']);--/** In-place: strip trailing editor-created hard breaks from every block whose- * phrasing content can end in one. */-export function stripTrailingHardBreaks(tree: Nodes): void {- const visit = (node: Nodes): void => {- if (BLOCK_CONTAINERS.has(node.type) && 'children' in node) {- stripTrailing((node as Parent).children as RootContent[]);- }- if ('children' in node) {- for (const child of (node as Parent).children as Nodes[]) visit(child);- }- };- visit(tree);-}