Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
feat(ok): make underline a savable format via inline HTML (#3043)
Nick Gomez2168ffc4eaa2
diff --git a/.changeset/underline-mark.md b/.changeset/underline-mark.mdnew file mode 100644index 000000000..f5b31153e--- /dev/null+++ b/.changeset/underline-mark.md@@ -0,0 +1,17 @@+---+'@inkeep/open-knowledge': minor+---++Underline is now a real, savable format. The bubble-menu Underline button and Cmd+U have always been there, but nothing wrote the result to disk: the underline vanished the moment the document was saved, reloaded, or opened by a teammate. Underlined text pasted in from Word arrived as italic instead.++| Before | After |+| --- | --- |+| Cmd+U, reload the page — underline gone | Cmd+U, reload the page — still underlined |+| Paste underlined text from Word — comes in italic | Comes in underlined |+| Paste underlined text from Joplin or Logseq — formatting dropped | Comes in underlined |++Underline is written as `<u>text</u>`, the same thing Typora, MarkText and Obsidian write, so files stay readable and portable. Documents that already spell it `<ins>text</ins>` — what Joplin and Logseq produce — are read as underline too and keep that spelling when saved, so importing a vault does not rewrite it under you.++Worth knowing: markdown has no underline of its own, so this is HTML inside your file. It renders as underline in OpenKnowledge, in a published site, and when pasted into Word, Docs or Gmail. GitHub is the exception — it strips `<u>`, so the text shows up unformatted there. Files that use `<ins>` do keep their underline on GitHub.++Underline can be combined with bold, italic, strikethrough and inline code on the same text.diff --git a/packages/app/tests/dom/underline-round-trip.dom.test.tsx b/packages/app/tests/dom/underline-round-trip.dom.test.tsxnew file mode 100644index 000000000..471444905--- /dev/null+++ b/packages/app/tests/dom/underline-round-trip.dom.test.tsx@@ -0,0 +1,291 @@+/**+ * Underline reaches the document on two live paths, and must survive both.+ *+ * `Mod-U` toggles a real PM `underline` mark, and a Word/LibreOffice paste+ * carries `<u>` elements in its `text/html` flavor. Neither path had a+ * representation on the way back out: the PM→mdast direction had no underline+ * mark handler, so a typed underline was dropped at the first serialize with+ * no error, and the paste dispatcher's generic-HTML branch inherited+ * `hast-util-to-mdast`'s `u: em` default, so pasted underline arrived as+ * italic. The first is silent loss, the second is silent substitution; both+ * are invisible to a byte-level round-trip test because the bytes on disk are+ * self-consistent either way.+ *+ * The rig is the production re-derivation rather than a bare `parse()`:+ * Observer A serializes the fragment into Y.Text, Observer B splits the+ * frontmatter region off and re-parses the body with `parseWithFallback`.+ * That composition is what a live document actually runs on every settle,+ * document reload and agent undo.+ *+ * The paste route drives the real dispatcher (`createHandlePaste`) through+ * `view.pasteHTML`, so branch selection is the production one — a test that+ * called `insertContent` would exercise TipTap's own `parseHTML` rules and+ * never reach the generic-HTML branch that a foreign app's payload lands on.+ *+ */++import { MarkdownManager, sharedExtensions, stripFrontmatter } from '@inkeep/open-knowledge-core';+import { cleanup } from '@testing-library/react';+import { Editor, getSchema, type JSONContent } from '@tiptap/core';+import { AllSelection, EditorState } from '@tiptap/pm/state';+import { EditorView } from '@tiptap/pm/view';+import { afterEach, describe, expect, test } from 'vitest';+import { createHandlePaste } from '../../src/editor/clipboard/handle-paste';++const mdManager = new MarkdownManager({+ extensions: sharedExtensions,+ deriveStructuralFreshness: true,+});+const schema = getSchema(sharedExtensions);++const mounted: Array<{ editor: Editor; container: HTMLDivElement }> = [];+const views: EditorView[] = [];++afterEach(() => {+ for (const { editor, container } of mounted.splice(0)) {+ editor.destroy();+ container.remove();+ }+ for (const view of views.splice(0)) view.destroy();+ cleanup();+});++function mount(content?: JSONContent): { editor: Editor; container: HTMLDivElement } {+ const container = document.createElement('div');+ document.body.appendChild(container);+ const editor = new Editor({+ element: container,+ ...(content ? { content } : {}),+ extensions: sharedExtensions,+ editable: true,+ });+ const entry = { editor, container };+ mounted.push(entry);+ return entry;+}++/** Observer A serialize into Observer B's frontmatter-stripped re-parse. */+function rederive(doc: JSONContent): { bytes: string; next: JSONContent } {+ const bytes = mdManager.serialize(doc);+ return { bytes, next: mdManager.parseWithFallback(stripFrontmatter(bytes).body) };+}++/** Every mark name carried by the text node whose content is `text`. */+function marksOn(doc: JSONContent, text: string): string[] {+ const found: string[] = [];+ const walk = (node: JSONContent): void => {+ if (node.type === 'text' && node.text === text) {+ for (const mark of node.marks ?? []) found.push(mark.type);+ }+ for (const child of node.content ?? []) walk(child);+ };+ walk(doc);+ return found;+}++/** Concatenated text of a document, with a space at every block boundary. */+function allText(doc: JSONContent): string {+ const parts: string[] = [];+ const walk = (node: JSONContent): void => {+ if (node.type === 'text') parts.push(node.text ?? '');+ for (const child of node.content ?? []) walk(child);+ };+ walk(doc);+ return parts.join('');+}++/** Concatenated text of every node carrying `markName`. */+function textUnderMark(doc: JSONContent, markName: string): string {+ let out = '';+ const walk = (node: JSONContent): void => {+ if (node.type === 'text' && (node.marks ?? []).some((mark) => mark.type === markName)) {+ out += node.text ?? '';+ }+ for (const child of node.content ?? []) walk(child);+ };+ walk(doc);+ return out;+}++/**+ * Apply `Mod-U` over a typed run the way the keymap does: insert the prose,+ * select the range, dispatch the chord. `keyboardShortcut` reports that it+ * dispatched rather than that a handler consumed, so the mark itself is+ * asserted before anything downstream is measured.+ */+function typeUnderlined(editor: Editor, before: string, underlined: string): void {+ editor.commands.insertContent(before);+ const start = editor.state.selection.from;+ editor.commands.insertContent(underlined);+ const end = editor.state.selection.from;+ editor.commands.setTextSelection({ from: start, to: end });+ editor.commands.keyboardShortcut('Mod-u');+ editor.commands.setTextSelection(end);+}++/**+ * A DataTransfer/ClipboardEvent stand-in carrying both flavors. The dispatcher+ * only reads `types` / `getData(mime)` and `event.shiftKey`.+ */+function fakeClipboardEvent(plain: string, html: string): ClipboardEvent {+ const dt = {+ types: ['text/plain', 'text/html'],+ getData: (mime: string) => (mime === 'text/plain' ? plain : mime === 'text/html' ? html : ''),+ };+ return { clipboardData: dt } as unknown as ClipboardEvent;+}++/** Paste a foreign payload through the production dispatcher. */+function pasteForeign(plain: string, html: string): JSONContent {+ const mount = document.createElement('div');+ document.body.appendChild(mount);+ const view = new EditorView(mount, {+ state: EditorState.create({+ schema,+ doc: schema.node('doc', null, [schema.node('paragraph')]),+ }),+ handlePaste: createHandlePaste({ mdManager }),+ });+ views.push(view);+ view.dispatch(view.state.tr.setSelection(new AllSelection(view.state.doc)));+ view.pasteHTML(html, fakeClipboardEvent(plain, html));+ return view.state.doc.toJSON() as JSONContent;+}++describe('route 1 — Mod-U typed underline', () => {+ test('the chord applies a real underline mark', () => {+ const { editor } = mount();+ typeUnderlined(editor, 'plain ', 'under');+ expect(marksOn(editor.getJSON(), 'under')).toContain('underline');+ });++ test('the typed mark serializes to <u> bytes', () => {+ const { editor } = mount();+ typeUnderlined(editor, 'plain ', 'under');+ expect(rederive(editor.getJSON()).bytes).toContain('plain <u>under</u>');+ });++ test('the typed mark survives re-derivation', () => {+ const { editor } = mount();+ typeUnderlined(editor, 'plain ', 'under');+ const { next } = rederive(editor.getJSON());+ expect(textUnderMark(next, 'underline')).toBe('under');+ });++ test('re-derivation reaches a byte fixed point', () => {+ const { editor } = mount();+ typeUnderlined(editor, 'plain ', 'under');+ let doc: JSONContent = editor.getJSON();+ const first = rederive(doc);+ doc = first.next;+ for (let iteration = 0; iteration < 3; iteration += 1) {+ const step = rederive(doc);+ expect(step.bytes).toBe(first.bytes);+ doc = step.next;+ }+ });++ test('underline coexists with strong, emphasis and code on one run', () => {+ const { editor } = mount();+ editor.commands.insertContent('x');+ editor.commands.setTextSelection({ from: 1, to: 2 });+ for (const chord of ['Mod-u', 'Mod-b', 'Mod-i', 'Mod-e']) {+ editor.commands.keyboardShortcut(chord);+ }+ const applied = marksOn(editor.getJSON(), 'x');+ expect(applied).toEqual(expect.arrayContaining(['underline', 'strong', 'emphasis', 'code']));++ const { next } = rederive(editor.getJSON());+ expect(textUnderMark(next, 'underline')).toBe('x');+ });+});++describe('route 2 — foreign paste carrying <u>', () => {+ const WORD = {+ plain: 'plain under tail',+ html: '<p class="MsoNormal">plain <u>under</u> tail</p>',+ };++ test('a pasted <u> run lands as underline, not emphasis', () => {+ const pasted = pasteForeign(WORD.plain, WORD.html);+ expect(textUnderMark(pasted, 'underline')).toBe('under');+ expect(textUnderMark(pasted, 'emphasis')).toBe('');+ });++ test('a pasted <ins> run lands as underline too', () => {+ const pasted = pasteForeign('plain under tail', '<p>plain <ins>under</ins> tail</p>');+ expect(textUnderMark(pasted, 'underline')).toBe('under');+ });++ /**+ * The whole `htmlToMdast` → `expandUnderlineWrappers` → promoter → serialize+ * chain has to keep the authored spelling. Without the bytes assertion an+ * `ins`-to-`u` slip anywhere along it would still leave an underline mark+ * and pass every mark-level check.+ */+ test('a pasted <ins> run serializes back as <ins>, not <u>', () => {+ const pasted = pasteForeign('plain under tail', '<p>plain <ins>under</ins> tail</p>');+ const { bytes, next } = rederive(pasted);+ expect(bytes).toContain('<ins>under</ins>');+ expect(bytes).not.toContain('<u>');+ expect(textUnderMark(next, 'underline')).toBe('under');+ });++ test('a pasted <u> run serializes back as <u>, not <ins>', () => {+ const { bytes } = rederive(pasteForeign(WORD.plain, WORD.html));+ expect(bytes).toContain('<u>under</u>');+ expect(bytes).not.toContain('<ins>');+ });++ test('pasted underline survives re-derivation', () => {+ const { next } = rederive(pasteForeign(WORD.plain, WORD.html));+ expect(textUnderMark(next, 'underline')).toBe('under');+ });++ test('a pasted italic run is still italic (no over-claim)', () => {+ const pasted = pasteForeign('plain slanted tail', '<p>plain <i>slanted</i> tail</p>');+ expect(textUnderMark(pasted, 'emphasis')).toBe('slanted');+ expect(textUnderMark(pasted, 'underline')).toBe('');+ });++ /**+ * The underline run has to survive in every container the clipboard can put+ * it in, not just inside a `<p>`. mdast's `html` node is flow-ambiguous, so+ * a converter that emits the tags directly shatters the surrounding prose+ * into separate blocks wherever the run is not already inside a+ * phrasing-only container — the browser's own fragment wrapper for a partial+ * in-paragraph selection and `<li>` content both hit that. Each case asserts+ * the underline landed AND the prose around it stayed in one block.+ */+ const CONTAINERS = [+ {+ name: 'browser StartFragment wrapper',+ html: '<!--StartFragment-->a <u>q</u> b<!--EndFragment-->',+ },+ { name: 'bare root-level run', html: 'a <u>q</u> b' },+ { name: 'paragraph', html: '<p>a <u>q</u> b</p>' },+ { name: 'list item', html: '<ul><li>a <u>q</u> b</li></ul>' },+ { name: 'table cell', html: '<table><tr><td>a <u>q</u> b</td></tr></table>' },+ { name: 'heading', html: '<h2>a <u>q</u> b</h2>' },+ ] as const;++ for (const { name, html } of CONTAINERS) {+ test(`underline survives a paste inside a ${name}`, () => {+ const pasted = pasteForeign('a q b', html);+ expect(textUnderMark(pasted, 'underline')).toBe('q');+ expect(textUnderMark(pasted, 'emphasis')).toBe('');+ // The prose around the run must not have been split across blocks.+ expect(allText(pasted)).toContain('a q b');+ });+ }+});++describe('the underline mark renders visibly', () => {+ test('the mark renders as a <u> element in the editing surface', () => {+ const { editor, container } = mount();+ typeUnderlined(editor, 'plain ', 'under');+ const rendered = container.querySelector('u');+ expect(rendered).not.toBeNull();+ expect(rendered?.textContent).toBe('under');+ });+});diff --git a/packages/core/package.json b/packages/core/package.jsonindex 5dd84c490..2b9efa0d4 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -64,6 +64,7 @@"@tiptap/extension-link": "^3.22.3","@tiptap/extension-strike": "^3.22.3","@tiptap/extension-table": "^3.22.3",+ "@tiptap/extension-underline": "^3.22.3","@tiptap/pm": "^3.22.3","@tiptap/starter-kit": "^3.22.3","ajv": "^8.18.0",diff --git a/packages/core/schema-snapshot.json b/packages/core/schema-snapshot.jsonindex c15b9c629..ed9b65742 100644--- a/packages/core/schema-snapshot.json+++ b/packages/core/schema-snapshot.json@@ -831,7 +831,12 @@"spanning": true},"underline": {- "attrs": {},+ "attrs": {+ "sourceForm": {+ "hasDefault": true,+ "default": "u"+ }+ },"excludes": "underline","group": "","inclusive": true,@@ -929,6 +934,11 @@"version": null,"hasOptions": false},+ {+ "name": "underline-promoter",+ "version": null,+ "hasOptions": false+ },{"name": "mermaid-promoter","version": null,diff --git a/packages/core/src/extensions/shared.ts b/packages/core/src/extensions/shared.tsindex 9ced9f23e..86558542b 100644--- a/packages/core/src/extensions/shared.ts+++ b/packages/core/src/extensions/shared.ts@@ -33,6 +33,7 @@ import { StrikeFidelity } from './strike-fidelity.ts';import { TableCellFidelity, TableFidelity, TableHeaderFidelity } from './table-fidelity.ts';import { Tag } from './tag.ts';import { ThematicBreakFidelity } from './thematic-break-fidelity.ts';+import { UnderlineFidelity } from './underline-fidelity.ts';import { WikiLink } from './wiki-link.ts';import { WikiLinkEmbed } from './wiki-link-embed.ts';@@ -69,6 +70,10 @@ export const sharedExtensions = [// single-tilde strikethrough round-trips byte-equal; replaces the// StarterKit Strike.StrikeFidelity,+ // UnderlineFidelity carries `sourceForm` ('u' vs 'ins') so both inline-HTML+ // spellings of underline re-emit as authored; replaces the StarterKit+ // Underline, which had no serializable markdown form at all.+ UnderlineFidelity,// Override @tiptap/extension-code's `excludes: '_'` so the Code mark can// coexist with emphasis/strong on the same span. CommonMark allows it;// the upstream exclusion broke round-trip for `*a \`*\`*` and `_a \`_\`_`@@ -113,6 +118,7 @@ export const sharedExtensions = [italic: false,bold: false,strike: false,+ underline: false,code: false,codeBlock: false,heading: false,diff --git a/packages/core/src/extensions/underline-fidelity.ts b/packages/core/src/extensions/underline-fidelity.tsnew file mode 100644index 000000000..48af5a0d7--- /dev/null+++ b/packages/core/src/extensions/underline-fidelity.ts@@ -0,0 +1,41 @@+/**+ * Underline mark override for source-text fidelity.+ *+ * Extends @tiptap/extension-underline (preserving setUnderline/toggleUnderline/+ * unsetUnderline commands and the Mod-U shortcut) with a source-form attribute:+ *+ * - sourceForm: `'u'` or `'ins'` as authored. Markdown has no native+ * underline construct, so the construct-canonical emission IS raw inline+ * HTML. Two spellings are accepted on the way in — `<u>` (what Typora,+ * MarkText and Obsidian write, and what this mark renders as) and `<ins>`+ * (what Joplin and Logseq write, and the only one GitHub's sanitizer+ * keeps) — and each re-emits the spelling it arrived as, so an imported+ * file round-trips byte-identically instead of being rewritten under the+ * author.+ *+ * `Mod-U` and the bubble menu produce the default `'u'`.+ *+ * Registered in place of StarterKit's bundled Underline (disabled there), the+ * same way Bold/Italic/Strike/Code are replaced by their fidelity variants.+ * Upstream's `renderMarkdown` (`++text++`) is inert here: markdown+ * serialization runs through the unified pipeline+ * (packages/core/src/markdown/), never TipTap's own markdown layer.+ *+ * No `excludes` field: underline must share a span with strong / emphasis /+ * code, and the schema default (self-exclusion only) already allows that.+ * Narrowing it would violate precedent #9 the same way the Code widening+ * documents.+ */++import Underline from '@tiptap/extension-underline';++export const UnderlineFidelity = Underline.extend({+ priority: 60,++ addAttributes() {+ return {+ ...this.parent?.(),+ sourceForm: { default: 'u', rendered: false },+ };+ },+});diff --git a/packages/core/src/markdown/autolink-void-html-guard.ts b/packages/core/src/markdown/autolink-void-html-guard.tsindex d8e0e3ff3..03e182587 100644--- a/packages/core/src/markdown/autolink-void-html-guard.ts+++ b/packages/core/src/markdown/autolink-void-html-guard.ts@@ -1,5 +1,6 @@import type { Nodes, Root } from 'mdast';import { visit } from 'unist-util-visit';+import { findFencedRegions, findInlineCodeRegions, isInsideFence } from './fence-regions.ts';const GUARD_OPEN = '\uE000';const GUARD_CLOSE = '\uE001';@@ -38,7 +39,30 @@ const LOWERCASE_HTML_TAG_RE = /<([a-z][a-z0-9]*)(\s[^>]*)?\/?>/g;const LOWERCASE_JSX_CANONICAL_TAGS = new Set(['img', 'video', 'audio']);-const LOWERCASE_PAIRED_JSX_TAGS = new Set(['mark']);+const LOWERCASE_PAIRED_JSX_TAGS = new Set(['mark', 'u', 'ins']);++function countOpenersBefore(+ source: string,+ tag: string,+ offset: number,+ codeRegions: Array<[number, number]>,+): number {+ let count = 0;+ let from = 0;+ const needle = `<${tag}`;+ while (from < offset) {+ const at = source.indexOf(needle, from);+ if (at === -1 || at >= offset) break;+ from = at + needle.length;+ const after = source[at + needle.length];+ if (after !== undefined && after !== '>' && after !== '/' && !/\s/.test(after)) continue;+ if (isInsideFence(at, codeRegions)) continue;+ const gt = source.indexOf('>', at);+ if (gt !== -1 && source[gt - 1] === '/') continue;+ count++;+ }+ return count;+}const UPPERCASE_CLOSE_TAG_INDEX_RE = /<\/([A-Z][A-Za-z0-9.]*)>/g;@@ -270,8 +294,16 @@ export function protectFromMdx(source: string): string {return `${GUARD_OPEN}${safe}${GUARD_CLOSE}`;});- result = result.replace(HTML_CLOSE_TAG_RE, (match, tag: string) => {- if (LOWERCASE_PAIRED_JSX_TAGS.has(tag)) return match;+ const exemptedClosers = new Map<string, number>();+ const codeRegions = [...findFencedRegions(result), ...findInlineCodeRegions(result)];+ result = result.replace(HTML_CLOSE_TAG_RE, (match, tag: string, offset: number) => {+ if (LOWERCASE_PAIRED_JSX_TAGS.has(tag)) {+ const used = exemptedClosers.get(tag) ?? 0;+ if (used < countOpenersBefore(result, tag, offset, codeRegions)) {+ exemptedClosers.set(tag, used + 1);+ return match;+ }+ }return match.replace(/</g, GUARD_OPEN).replace(/>/g, GUARD_CLOSE);});diff --git a/packages/core/src/markdown/fence-regions.ts b/packages/core/src/markdown/fence-regions.tsindex c7518d614..6d3f3643e 100644--- a/packages/core/src/markdown/fence-regions.ts+++ b/packages/core/src/markdown/fence-regions.ts@@ -24,3 +24,26 @@ export function findFencedRegions(src: string): Array<[number, number]> {export function isInsideFence(offset: number, fences: Array<[number, number]>): boolean {return fences.some(([start, end]) => offset >= start && offset < end);}++const BACKTICK_RUN_RE = /`+/g;++export function findInlineCodeRegions(src: string): Array<[number, number]> {+ const runs: Array<{ offset: number; length: number }> = [];+ BACKTICK_RUN_RE.lastIndex = 0;+ for (const match of src.matchAll(BACKTICK_RUN_RE)) {+ runs.push({ offset: match.index, length: match[0].length });+ }++ const regions: Array<[number, number]> = [];+ for (let i = 0; i < runs.length; i++) {+ const open = runs[i];+ for (let j = i + 1; j < runs.length; j++) {+ const close = runs[j];+ if (close.length !== open.length) continue;+ regions.push([open.offset, close.offset + close.length]);+ i = j;+ break;+ }+ }+ return regions;+}diff --git a/packages/core/src/markdown/html-to-mdast.ts b/packages/core/src/markdown/html-to-mdast.tsindex b1f252c31..787315157 100644--- a/packages/core/src/markdown/html-to-mdast.ts+++ b/packages/core/src/markdown/html-to-mdast.ts@@ -1,7 +1,7 @@-import type { Root as HastRoot } from 'hast';-import type { List, Root as MdastRoot } from 'mdast';+import type { Element as HastElement, Root as HastRoot } from 'hast';+import type { List, Nodes as MdastNodes, Root as MdastRoot } from 'mdast';import rehypeParse from 'rehype-parse';-import rehypeRemark from 'rehype-remark';+import rehypeRemark, { type Options as RehypeRemarkOptions } from 'rehype-remark';import remarkGfm from 'remark-gfm';import remarkStringify from 'remark-stringify';import { type Plugin, unified } from 'unified';@@ -79,6 +79,49 @@ function normalizeListSpread(tree: MdastRoot): void {});}+type RehypeRemarkHandlers = NonNullable<RehypeRemarkOptions['handlers']>;+type RehypeRemarkHandler = NonNullable<RehypeRemarkHandlers[string]>;++const underlineElementHandlers: RehypeRemarkHandlers = {+ u: underlineWrapperHandler('u'),+ ins: underlineWrapperHandler('ins'),+};++const UNDERLINE_FORM_KEY = 'okUnderlineForm';++function underlineWrapperHandler(tag: 'u' | 'ins'): RehypeRemarkHandler {+ return (state, node) => {+ const result = {+ type: 'emphasis',+ children: state.all(node as HastElement),+ data: { [UNDERLINE_FORM_KEY]: tag },+ } as unknown as MdastNodes;+ state.patch(node, result);+ return result;+ };+}++function expandUnderlineWrappers(tree: MdastRoot): void {+ const walk = (node: { children?: MdastNodes[] }): void => {+ if (!Array.isArray(node.children)) return;+ for (const child of node.children) walk(child as { children?: MdastNodes[] });+ const next: MdastNodes[] = [];+ for (const child of node.children) {+ const form = (child as { data?: Record<string, unknown> }).data?.[UNDERLINE_FORM_KEY];+ const children = (child as { children?: MdastNodes[] }).children;+ if ((form === 'u' || form === 'ins') && Array.isArray(children)) {+ next.push({ type: 'html', value: `<${form}>` } as MdastNodes);+ next.push(...children);+ next.push({ type: 'html', value: `</${form}>` } as MdastNodes);+ continue;+ }+ next.push(child);+ }+ node.children = next;+ };+ walk(tree as { children?: MdastNodes[] });+}+export function htmlToMdast(html: string, options?: HtmlToMdastOptions): MdastRoot {const maxBytes = options?.maxBytes ?? HTML_MAX_BYTES;if (html.length > maxBytes) {@@ -94,11 +137,12 @@ export function htmlToMdast(html: string, options?: HtmlToMdastOptions): MdastRoprocessor.use(plugin);}- processor.use(rehypeRemark);+ processor.use(rehypeRemark, { handlers: underlineElementHandlers });const hastTree = processor.parse(html) as HastRoot;const mdast = processor.runSync(hastTree) as unknown as MdastRoot;normalizeListSpread(mdast);+ expandUnderlineWrappers(mdast);applyCanonicalSourceFormDefaults(mdast);return mdast;}diff --git a/packages/core/src/markdown/index.ts b/packages/core/src/markdown/index.tsindex a64ecf306..f49f4b64e 100644--- a/packages/core/src/markdown/index.ts+++ b/packages/core/src/markdown/index.ts@@ -60,6 +60,7 @@ import type { PropDef } from '../registry/types.ts';import type {CommentBlockMdast,CommentMdast,+ UnderlineMdast,WikiLinkEmbedMdast,WikiLinkMdast,} from './mdast-augmentation.ts';@@ -572,6 +573,12 @@ function buildMdastToPmHandlers(if (m.highlight) handlers.mark = toPmMark(m.highlight);+ if (m.underline) {+ handlers.underline = toPmMark(m.underline, (node: UnderlineMdast) => ({+ sourceForm: node.data?.sourceForm === 'ins' ? 'ins' : 'u',+ }));+ }+if (m.comment) {handlers.comment = toPmMark(m.comment, (node: CommentMdast) => ({sourceForm: node.data?.sourceForm ?? 'percent',@@ -1484,6 +1491,12 @@ function buildPmToMdastHandlers(markHandlers.highlight = fromPmMark('mark');}+ if (m.underline) {+ markHandlers.underline = fromPmMark('underline', (mark: PmMark) => ({+ data: { sourceForm: mark.attrs.sourceForm === 'ins' ? 'ins' : 'u' },+ }));+ }+if (m.comment) {markHandlers.comment = fromPmMark('comment', (mark: PmMark) => ({data: { sourceForm: mark.attrs.sourceForm === 'html' ? 'html' : 'percent' },diff --git a/packages/core/src/markdown/mdast-augmentation.ts b/packages/core/src/markdown/mdast-augmentation.tsindex b1d866d2b..087f3caf9 100644--- a/packages/core/src/markdown/mdast-augmentation.ts+++ b/packages/core/src/markdown/mdast-augmentation.ts@@ -7,6 +7,7 @@ export const PROMOTED_MDAST_TYPES = ['mdxJsxTextElement','rawMdxFallback','mark',+ 'underline','tag','comment','commentBlock',@@ -93,6 +94,18 @@ export interface MarkMdast {position?: Position;}+export interface UnderlineMdast {+ type: 'underline';+ // biome-ignore lint/suspicious/noExplicitAny: see RootContentMap note above+ children: Array<any>;+ data?: {+ sourceForm?: 'u' | 'ins';+ sourceRaw?: string;+ [key: string]: unknown;+ };+ position?: Position;+}+export interface TagMdast {type: 'tag';value: string;@@ -193,6 +206,7 @@ declare module 'mdast' {rawMdxFallback: RawMdxFallbackMdast;wikiLinkEmbed: WikiLinkEmbedMdast;mark: MarkMdast;+ underline: UnderlineMdast;tag: TagMdast;comment: CommentMdast;commentBlock: CommentBlockMdast;diff --git a/packages/core/src/markdown/mdast-to-hast-handlers.ts b/packages/core/src/markdown/mdast-to-hast-handlers.tsindex 679c72ef7..ed89371d0 100644--- a/packages/core/src/markdown/mdast-to-hast-handlers.ts+++ b/packages/core/src/markdown/mdast-to-hast-handlers.ts@@ -10,11 +10,12 @@ import type {PromotedMdastType,RawMdxFallbackMdast,TagMdast,+ UnderlineMdast,WikiLinkEmbedMdast,WikiLinkMdast,} from './mdast-augmentation.ts';-const HTML_PRIMITIVE_TAGS = new Set(['img', 'video', 'audio', 'mark']);+const HTML_PRIMITIVE_TAGS = new Set(['img', 'video', 'audio', 'mark', 'u', 'ins']);const compatPrimitiveByName: ReadonlyMap<string, CompatMeta> = new Map(builtInComponents.flatMap((meta) =>@@ -209,6 +210,18 @@ const markHandler: Handler = (state, node) => {return state.applyData(node, result);};+const underlineHandler: Handler = (state, node) => {+ const source = node as UnderlineMdast;+ const result: Element = {+ type: 'element',+ tagName: source.data?.sourceForm === 'ins' ? 'ins' : 'u',+ properties: {},+ children: state.all(source) as ElementContent[],+ };+ state.patch(node, result);+ return state.applyData(node, result);+};+function mdastTextContent(node: unknown): string {if (typeof node !== 'object' || node === null) return '';const n = node as { type?: string; value?: unknown; children?: unknown };@@ -317,6 +330,7 @@ const promotedHandlers: Record<PromotedMdastType, Handler> = {mdxJsxTextElement: mdxJsxTextHandler,rawMdxFallback: rawMdxFallbackHandler,mark: markHandler,+ underline: underlineHandler,tag: tagHandler,comment: commentHandler,commentBlock: commentBlockHandler,diff --git a/packages/core/src/markdown/pipeline.ts b/packages/core/src/markdown/pipeline.tsindex a0cfe2886..4426ab46e 100644--- a/packages/core/src/markdown/pipeline.ts+++ b/packages/core/src/markdown/pipeline.ts@@ -38,6 +38,7 @@ import { remarkMdxAgnostic } from './remark-mdx-agnostic.ts';import { singleDollarMathPromoterPlugin } from './single-dollar-math-promoter.ts';import { stripTrailingEdge } from './strip-trailing-edge.ts';import { remarkTags } from './tag-to-markdown.ts';+import { underlinePromoterPlugin } from './underline-promoter.ts';import { voidBrPromoterPlugin } from './void-br-promoter.ts';import { remarkWikiLink } from './wiki-link-micromark.ts';import { pruneZeroEmissionBlocks } from './zero-emission-blocks.ts';@@ -94,6 +95,7 @@ export const ACTIVE_MDAST_PLUGINS = [{ name: 'math-promoter', plugin: mathPromoterPlugin },{ name: 'single-dollar-math-promoter', plugin: singleDollarMathPromoterPlugin },{ name: 'highlight-promoter', plugin: highlightPromoterPlugin },+ { name: 'underline-promoter', plugin: underlinePromoterPlugin },{ name: 'mermaid-promoter', plugin: mermaidPromoterPlugin },{ name: 'comment-promoter', plugin: commentPromoterPlugin },{ name: 'merged-post-parse-walker', plugin: mergedPostParseWalkerPlugin },diff --git a/packages/core/src/markdown/to-markdown-handlers.ts b/packages/core/src/markdown/to-markdown-handlers.tsindex 7f6dfcfc3..6000891e3 100644--- a/packages/core/src/markdown/to-markdown-handlers.ts+++ b/packages/core/src/markdown/to-markdown-handlers.ts@@ -10,6 +10,7 @@ import { isInlineWhitespaceNumericCharRef } from './whitespace-char-ref.ts';declare module 'mdast-util-to-markdown' {interface ConstructNameMap {mark: 'mark';+ underline: 'underline';comment: 'comment';commentBlock: 'commentBlock';strikethrough: 'strikethrough';@@ -636,6 +637,22 @@ export const toMarkdownHandlers = {return value;},+ underline(node, _parent, state, info) {+ const open = node.data?.sourceForm === 'ins' ? '<ins>' : '<u>';+ const close = node.data?.sourceForm === 'ins' ? '</ins>' : '</u>';+ const tracker = state.createTracker(info);+ const exit = state.enter('underline');+ let value = tracker.move(open);+ value += state.containerPhrasing(node as Parents, {+ before: value,+ after: close,+ ...tracker.current(),+ });+ value += tracker.move(close);+ exit();+ return value;+ },+comment(node, _parent, state, info) {const sourceForm = node.data?.sourceForm;const open = sourceForm === 'html' ? '<!-- ' : '%%';diff --git a/packages/core/src/markdown/underline-promoter.ts b/packages/core/src/markdown/underline-promoter.tsnew file mode 100644index 000000000..0f1b0f27d--- /dev/null+++ b/packages/core/src/markdown/underline-promoter.ts@@ -0,0 +1,27 @@+import type { Nodes, Parent, Root } from 'mdast';+import type { MdxJsxTextElement } from 'mdast-util-mdx';+import { visit } from 'unist-util-visit';+import type { UnderlineMdast } from './mdast-augmentation.ts';++const UNDERLINE_TAGS = new Set<string>(['u', 'ins']);++export function underlinePromoterPlugin() {+ return (tree: Root) => {+ visit(tree, 'mdxJsxTextElement', (node: MdxJsxTextElement, index, parent) => {+ if (parent === undefined || index === undefined || index === null) return;+ if (node.name === null || !UNDERLINE_TAGS.has(node.name)) return;+ if ((node.attributes ?? []).length > 0) return;++ const underlineNode: UnderlineMdast = {+ type: 'underline',+ children: (node.children as Nodes[]) ?? [],+ data: { sourceForm: node.name === 'ins' ? 'ins' : 'u' },+ };+ if (node.position) underlineNode.position = node.position;++ const arr = (parent as Parent).children;+ arr.splice(index, 1, underlineNode as unknown as (typeof arr)[number]);+ return index + 1;+ });+ };+}diff --git a/packages/core/src/markdown/unknown-mdast-guard.ts b/packages/core/src/markdown/unknown-mdast-guard.tsindex cd245c38c..1a082bda7 100644--- a/packages/core/src/markdown/unknown-mdast-guard.ts+++ b/packages/core/src/markdown/unknown-mdast-guard.ts@@ -39,6 +39,7 @@ export const KNOWN_MDAST_TYPES: ReadonlySet<string> = new Set(['math','inlineMath','mark',+ 'underline','comment','commentBlock','rawMdxFallbackMdast',diff --git a/packages/core/src/schema-snapshot.json b/packages/core/src/schema-snapshot.jsonindex e96b2f091..f82a60d42 100644--- a/packages/core/src/schema-snapshot.json+++ b/packages/core/src/schema-snapshot.json@@ -515,13 +515,6 @@"inclusive": true,"spanning": true},- "underline": {- "attrs": {},- "excludes": "underline",- "group": "",- "inclusive": true,- "spanning": true- },"highlight": {"attrs": {},"excludes": "highlight",@@ -562,6 +555,17 @@"inclusive": true,"spanning": true},+ "underline": {+ "attrs": {+ "sourceForm": {+ "hasDefault": true+ }+ },+ "excludes": "underline",+ "group": "",+ "inclusive": true,+ "spanning": true+ },"link": {"attrs": {"href": {@@ -649,6 +653,7 @@"emphasis","strong","strike",+ "underline","code","codeBlock","blockquote",diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 2fa637534..fcaaea949 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -887,6 +887,9 @@ importers:'@tiptap/extension-table':specifier: 3.22.3+ '@tiptap/extension-underline':+ specifier: 3.22.3+ version: 3.22.3(@tiptap/[email protected](@tiptap/[email protected]))'@tiptap/pm':specifier: 3.22.4version: 3.22.4