An open source design system that's fully customizable and agent ready
TypeScript
refactor(markdown): use canonical mdast tree (#6339)
Cindy Zhangd4cc23232be0
diff --git a/packages/core/src/Markdown/Markdown.test.tsx b/packages/core/src/Markdown/Markdown.test.tsxindex 017cfc441a..98b9d2a8ff 100644--- a/packages/core/src/Markdown/Markdown.test.tsx+++ b/packages/core/src/Markdown/Markdown.test.tsx@@ -80,6 +80,17 @@ describe('Markdown', () => {}});+ it('keeps citation markers out of released heading ids', () => {+ render(+ <Markdown sources={{cite: {title: 'Citation'}}}>+ {'# Before [cite] after'}+ </Markdown>,+ );+ expect(+ screen.getByRole('heading', {name: /Before.*after/}),+ ).toHaveAttribute('id', 'before-after');+ });+it('passes the generated id to a custom heading component', () => {const received: (string | undefined)[] = [];render(diff --git a/packages/core/src/Markdown/Markdown.tsx b/packages/core/src/Markdown/Markdown.tsxindex 75e68af7c7..0d6fe93638 100644--- a/packages/core/src/Markdown/Markdown.tsx+++ b/packages/core/src/Markdown/Markdown.tsx@@ -48,22 +48,21 @@ import type {CitationSource} from '../Citation/Citation';import {useLinkComponent} from '../Link/useLinkComponent';import type {LinkComponentType} from '../Link/types';import {- parseInline,- parseMarkdown,- parseMarkdownIncremental,+ parseInlineAst,+ parseMarkdownAst,+ parseMarkdownAstIncremental,createIncrementalState,trimStreamingArtifacts,- inlineText,slugify,uniqueSlug,} from './parser';+import type {IncrementalState, MathParseOptions, ParseOptions} from './parser';+import {markdownAstText} from './ast';import type {- BlockNodeWithMath,- InlineNodeWithMath,- IncrementalState,- MathParseOptions,- ParseOptions,-} from './parser';+ MarkdownAstBlockContent,+ MarkdownAstPhrasingContent,+ MarkdownAstTable,+} from './ast';import {themeProps} from '../utils/themeProps';import {useTranslator, type TranslatorFn} from '../i18n';@@ -487,25 +486,27 @@ interface StreamingCursor {* Count the total text characters in inline nodes without rendering.* Used to advance the cursor past a block that will be faded as a whole unit.*/-function countInlineTextLength(nodes: InlineNodeWithMath[]): number {+function countInlineTextLength(+ nodes: ReadonlyArray<MarkdownAstPhrasingContent>,+): number {let len = 0;for (const node of nodes) {switch (node.type) {case 'text':- len += node.content.length;+ len += node.value.length;break;- case 'code':- len += node.content.length;+ case 'inlineCode':+ len += node.value.length;break;- case 'math':+ case 'inlineMath':len += node.value.length;break;case 'image':len += node.alt.length;break;- case 'bold':- case 'italic':- case 'strikethrough':+ case 'strong':+ case 'emphasis':+ case 'delete':case 'link':len += countInlineTextLength(node.children);break;@@ -523,7 +524,9 @@ function countInlineTextLength(nodes: InlineNodeWithMath[]): number {/*** Count total text characters in a block node tree.*/-function countBlockTextLength(nodes: BlockNodeWithMath[]): number {+function countBlockTextLength(+ nodes: ReadonlyArray<MarkdownAstBlockContent>,+): number {let len = 0;for (const node of nodes) {switch (node.type) {@@ -531,8 +534,8 @@ function countBlockTextLength(nodes: BlockNodeWithMath[]): number {case 'paragraph':len += countInlineTextLength(node.children);break;- case 'codeblock':- len += node.content.length;+ case 'code':+ len += node.value.length;break;case 'math':len += node.value.length;@@ -541,21 +544,18 @@ function countBlockTextLength(nodes: BlockNodeWithMath[]): number {len += countBlockTextLength(node.children);break;case 'list':- for (const item of node.items) {+ for (const item of node.children) {len += countBlockTextLength(item.children);}break;case 'table':- for (const h of node.headers) {- len += countInlineTextLength(h.children);- }- for (const row of node.rows) {- for (const cell of row) {+ for (const row of node.children) {+ for (const cell of row.children) {len += countInlineTextLength(cell.children);}}break;- case 'hr':+ case 'thematicBreak':break;case 'image':len += node.alt.length;@@ -768,7 +768,7 @@ function getCitationNumber(ctx: CitationContext, sourceId: string): number {}function renderInline(- node: InlineNodeWithMath,+ node: MarkdownAstPhrasingContent,index: number,onLinkClick: MarkdownProps['onLinkClick'] | undefined,cursor: StreamingCursor,@@ -780,7 +780,7 @@ function renderInline(switch (node.type) {case 'text': {if (inlinePlugins && inlinePlugins.length > 0) {- const segments = applyInlinePlugins(node.content, inlinePlugins);+ const segments = applyInlinePlugins(node.value, inlinePlugins);// If no plugin matched, fall through to the normal path// O(1) guard: applyInlinePlugins returns a single text segment when// nothing matched — skip the plugin path entirely in that case.@@ -803,9 +803,9 @@ function renderInline(return <Fragment key={index}>{result}</Fragment>;}}- return wrapTextWithFade(node.content, cursor, index);+ return wrapTextWithFade(node.value, cursor, index);}- case 'bold':+ case 'strong':return (<strong key={index} {...stylex.props(styles.bold)}>{node.children.map((c, i) =>@@ -822,7 +822,7 @@ function renderInline()}</strong>);- case 'italic':+ case 'emphasis':return (<em key={index}>{node.children.map((c, i) =>@@ -839,7 +839,7 @@ function renderInline()}</em>);- case 'strikethrough':+ case 'delete':return (<del key={index} {...stylex.props(styles.strikethrough)}>{node.children.map((c, i) =>@@ -856,18 +856,18 @@ function renderInline()}</del>);- case 'code': {+ case 'inlineCode': {// Track code content length for cursor but don't split inside code- cursor.offset += node.content.length;+ cursor.offset += node.value.length;const InlineCodeComp = components?.inlineCode;const codeEl = InlineCodeComp ? (- <InlineCodeComp key={index}>{node.content}</InlineCodeComp>+ <InlineCodeComp key={index}>{node.value}</InlineCodeComp>) : (- <Code key={index}>{node.content}</Code>+ <Code key={index}>{node.value}</Code>);return codeEl;}- case 'math': {+ case 'inlineMath': {const MathComp = components?.math;if (MathComp == null) {return wrapTextWithFade(`$${node.value}$`, cursor, index);@@ -876,7 +876,7 @@ function renderInline(return <MathComp key={index} value={node.value} display="inline" />;}case 'link': {- const safeHref = sanitizeUrl(node.href);+ const safeHref = sanitizeUrl(node.url);if (safeHref == null) {// Unsafe URL — render as plain textreturn (@@ -954,7 +954,7 @@ function renderInline();}case 'image': {- const safeSrc = sanitizeUrl(node.src);+ const safeSrc = sanitizeUrl(node.url);if (safeSrc == null) {return <span key={index}>[{node.alt}]</span>;}@@ -1012,13 +1012,13 @@ function renderInline(// ---------------------------------------------------------------------------function getElementSpacing(- node: BlockNodeWithMath,+ node: MarkdownAstBlockContent,density: 'default' | 'compact',): StyleXStyles {const compact = density === 'compact';switch (node.type) {case 'heading':- return node.level <= 3+ return node.depth <= 3? compact? styles.spacingHeadingMajorCompact: styles.spacingHeadingMajorDefault@@ -1029,7 +1029,7 @@ function getElementSpacing(return compact? styles.spacingParagraphCompact: styles.spacingParagraphDefault;- case 'codeblock':+ case 'code':case 'math':return compact? styles.spacingCodeblockCompact@@ -1042,7 +1042,7 @@ function getElementSpacing(return compact ? styles.spacingListCompact : styles.spacingListDefault;case 'table':return compact ? styles.spacingTableCompact : styles.spacingTableDefault;- case 'hr':+ case 'thematicBreak':return compact ? styles.spacingHrCompact : styles.spacingHrDefault;case 'image':return compact ? styles.spacingImageCompact : styles.spacingImageDefault;@@ -1057,15 +1057,17 @@ function getElementSpacing(* Compute per-column min-widths from table AST content.* Buckets: ≤6 chars → 60px, 7–15 → 80px, >15 → 120px.*/-function computeTableColumnMinWidths(node: {- headers: {children: InlineNodeWithMath[]}[];- rows: {children: InlineNodeWithMath[]}[][];-}): number[] {- return node.headers.map((h, colIdx) => {- let maxLen = countInlineTextLength(h.children);- for (const row of node.rows) {- if (row[colIdx]) {- const len = countInlineTextLength(row[colIdx].children);+function computeTableColumnMinWidths(node: MarkdownAstTable): number[] {+ const [header, ...rows] = node.children;+ if (header == null) {+ return [];+ }+ return header.children.map((cell, colIdx) => {+ let maxLen = countInlineTextLength(cell.children);+ for (const row of rows) {+ const rowCell = row.children[colIdx];+ if (rowCell != null) {+ const len = countInlineTextLength(rowCell.children);if (len > maxLen) {maxLen = len;}@@ -1076,7 +1078,7 @@ function computeTableColumnMinWidths(node: {}function renderBlock(- node: BlockNodeWithMath,+ node: MarkdownAstBlockContent,index: number,blockCount: number,density: 'default' | 'compact',@@ -1090,7 +1092,7 @@ function renderBlock(inlinePlugins: MarkdownInlinePlugin[] | undefined,components: Partial<MarkdownComponents> | undefined,t: TranslatorFn,- headingIdMap?: ReadonlyMap<BlockNodeWithMath, string>,+ headingIdMap?: ReadonlyMap<MarkdownAstBlockContent, string>,): SyncReactNode {const blockAlignMargin = BLOCK_ALIGN_MARGIN[contentAlign];const blockAlignStyle =@@ -1103,7 +1105,7 @@ function renderBlock(switch (node.type) {case 'heading': {- const level = Math.min(node.level + headingLevelStart - 1, 6) as+ const level = Math.min(node.depth + headingLevelStart - 1, 6) as1 | 2 | 3 | 4 | 5 | 6;const headingChildren = node.children.map((c, i) =>renderInline(@@ -1203,16 +1205,16 @@ function renderBlock(</div>);}- case 'codeblock': {+ case 'code': {// Track codeblock content in cursor for accurate character counting- cursor.offset += node.content.length;+ cursor.offset += node.value.length;const CodeBlockComp = components?.code;if (CodeBlockComp) {return (<CodeBlockCompkey={index}- code={node.content}- language={node.language}+ code={node.value}+ language={node.lang ?? 'plaintext'}/>);}@@ -1229,8 +1231,8 @@ function renderBlock(),)}><CodeBlock- code={node.content}- language={node.language}+ code={node.value}+ language={node.lang ?? 'plaintext'}isCollapsiblexstyle={[contentWidthValue != null@@ -1316,11 +1318,12 @@ function renderBlock(case 'list': {// Detect task lists: all items have a checked stateconst isTaskList =- node.items.length > 0 && node.items.every(item => item.checked != null);+ node.children.length > 0 &&+ node.children.every(item => item.checked != null);if (isTaskList) {// Extract labels from task items — render as rich inline content- const checkedValues = node.items+ const checkedValues = node.children.map((item, i) => ({item, key: `task-${i}`})).filter(({item}) => item.checked).map(({key}) => key);@@ -1343,7 +1346,7 @@ function renderBlock(xstyle={styles.blockIndent}isReadOnlydensity="compact">- {node.items.map((item, i) => {+ {node.children.map((item, i) => {const firstChild = item.children[0];const isInline =item.children.length === 1 &&@@ -1423,7 +1426,7 @@ function renderBlock(density="compact"start={node.ordered ? node.start : undefined}xstyle={styles.blockIndent}>- {node.items.map((item, i) => {+ {node.children.map((item, i) => {const firstChild = item.children[0];const isInline =item.children.length === 1 && firstChild?.type === 'paragraph';@@ -1483,6 +1486,7 @@ function renderBlock(}case 'table': {const colMinWidths = computeTableColumnMinWidths(node);+ const [header, ...rows] = node.children;return (<div@@ -1512,14 +1516,14 @@ function renderBlock(<Table dividers="rows" textOverflow="wrap"><TableHeader><TableRow>- {node.headers.map((h, i) => (+ {header?.children.map((h, i) => (<TableHeaderCell// eslint-disable-next-line @eslint-react/no-array-index-key -- markdown table columns are positional by definitionkey={i}xstyle={[dynamicStyles.cellMinWidth(`${colMinWidths[i]}px`),- node.alignments[i] === 'center' && cellAlignStyles.center,- node.alignments[i] === 'right' && cellAlignStyles.end,+ node.align[i] === 'center' && cellAlignStyles.center,+ node.align[i] === 'right' && cellAlignStyles.end,]}>{h.children.map((c, j) =>renderInline(@@ -1538,14 +1542,14 @@ function renderBlock(</TableRow></TableHeader><TableBody>- {node.rows.map((row, i) => {- const cells = row.map((cell, j) => (+ {rows.map((row, i) => {+ const cells = row.children.map((cell, j) => (<TableCell// eslint-disable-next-line @eslint-react/no-array-index-key -- markdown table cells are positional by row and columnkey={j}xstyle={[- node.alignments[j] === 'center' && cellAlignStyles.center,- node.alignments[j] === 'right' && cellAlignStyles.end,+ node.align[j] === 'center' && cellAlignStyles.center,+ node.align[j] === 'right' && cellAlignStyles.end,]}>{cell.children.map((c, k) =>renderInline(@@ -1574,7 +1578,7 @@ function renderBlock(</div>);}- case 'hr': {+ case 'thematicBreak': {const HrComp = components?.hr;if (HrComp) {return <HrComp key={index} />;@@ -1595,7 +1599,7 @@ function renderBlock();}case 'image': {- const safeSrc = sanitizeUrl(node.src);+ const safeSrc = sanitizeUrl(node.url);if (safeSrc == null) {return (<div@@ -1719,21 +1723,25 @@ export function Markdown({}const options = hasMathRenderer ? mathParseOptions : legacyParseOptions;const input = trimStreamingArtifacts(smoothedText, options);- return hasMathRenderer- ? parseMarkdownIncremental(- input,- incrementalStateRef.current as IncrementalState<true>,- mathParseOptions,- )- : parseMarkdownIncremental(- input,- incrementalStateRef.current as IncrementalState<false>,- legacyParseOptions,- );+ return (+ hasMathRenderer+ ? parseMarkdownAstIncremental(+ input,+ incrementalStateRef.current,+ mathParseOptions,+ )+ : parseMarkdownAstIncremental(+ input,+ incrementalStateRef.current,+ legacyParseOptions,+ )+ ).children;}- return hasMathRenderer- ? parseMarkdown(children, mathParseOptions)- : parseMarkdown(children, legacyParseOptions);+ return (+ hasMathRenderer+ ? parseMarkdownAst(children, mathParseOptions)+ : parseMarkdownAst(children, legacyParseOptions)+ ).children;}, [display,smoothedText,@@ -1754,11 +1762,11 @@ export function Markdown({if (display === 'inline' || blocks.length === 0) {return undefined;}- const map = new Map<BlockNodeWithMath, string>();+ const map = new Map<MarkdownAstBlockContent, string>();const counts = new Map<string, number>();for (const block of blocks) {if (block.type === 'heading') {- const label = inlineText(block.children).trim();+ const label = markdownAstText(block.children).trim();map.set(block, uniqueSlug(slugify(label), counts));}}@@ -1774,8 +1782,8 @@ export function Markdown({? trimStreamingArtifacts(smoothedText, options): children;return hasMathRenderer- ? parseInline(input, mathParseOptions)- : parseInline(input, legacyParseOptions);+ ? parseInlineAst(input, mathParseOptions)+ : parseInlineAst(input, legacyParseOptions);}, [display,smoothedText,@@ -1800,8 +1808,10 @@ export function Markdown({return Math.min(Math.ceil(duration / tickMs), 12);}, [token]);- const prevBlocksRef = useRef<BlockNodeWithMath[]>([]);- const prevInlineNodesRef = useRef<InlineNodeWithMath[]>([]);+ const prevBlocksRef = useRef<ReadonlyArray<MarkdownAstBlockContent>>([]);+ const prevInlineNodesRef = useRef<ReadonlyArray<MarkdownAstPhrasingContent>>(+ [],+ );const boundariesRef = useRef<number[]>([]);const smoothedLen = smoothedText.length;const boundaries = useMemo(() => {diff --git a/packages/core/src/Markdown/ast.test.ts b/packages/core/src/Markdown/ast.test.tsnew file mode 100644index 0000000000..206cf82b1a--- /dev/null+++ b/packages/core/src/Markdown/ast.test.ts@@ -0,0 +1,169 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {+ createIncrementalState,+ parseMarkdown,+ parseMarkdownAst,+} from './parser';++describe('canonical Markdown AST', () => {+ it('uses MDAST-aligned names and fields for every built-in structure', () => {+ const ast = parseMarkdownAst(+ '# Title\n\n**bold** and *emphasis* with `code` and [link](/docs).\n\n- item\n\n| A | B |\n| - | - |\n| 1 | 2 |\n\n```ts\nconst value = 1;\n```\n\n---',+ );++ expect(ast).toMatchObject({+ type: 'root',+ children: [+ {+ type: 'heading',+ depth: 1,+ children: [{type: 'text', value: 'Title'}],+ },+ {+ type: 'paragraph',+ children: [+ {type: 'strong'},+ {type: 'text'},+ {type: 'emphasis'},+ {type: 'text'},+ {type: 'inlineCode', value: 'code'},+ {type: 'text'},+ {type: 'link', url: '/docs'},+ {type: 'text', value: '.'},+ ],+ },+ {+ type: 'list',+ ordered: false,+ children: [{type: 'listItem'}],+ },+ {+ type: 'table',+ align: [null, null],+ children: [+ {+ type: 'tableRow',+ children: [{type: 'tableCell'}, {type: 'tableCell'}],+ },+ {+ type: 'tableRow',+ children: [{type: 'tableCell'}, {type: 'tableCell'}],+ },+ ],+ },+ {type: 'code', lang: 'ts', value: 'const value = 1;'},+ {type: 'thematicBreak'},+ ],+ });+ });++ it('represents remaining built-ins without legacy aliases', () => {+ const ast = parseMarkdownAst(+ '> quote\n\n\n\nParagraph  [src] $x$ \nnext\n\n$$\ny\n$$',+ {math: true, sourceIds: new Set(['src'])},+ );++ expect(ast.children).toMatchObject([+ {+ type: 'blockquote',+ children: [+ {type: 'paragraph', children: [{type: 'text', value: 'quote'}]},+ ],+ },+ {type: 'image', url: '/block.png', alt: 'Block'},+ {+ type: 'paragraph',+ children: [+ {type: 'text', value: 'Paragraph '},+ {type: 'image', url: '/inline.png', alt: 'Inline'},+ {type: 'text', value: ' '},+ {type: 'citation', sourceId: 'src'},+ {type: 'text', value: ' '},+ {type: 'inlineMath', value: 'x'},+ {type: 'break'},+ {type: 'text', value: 'next'},+ ],+ },+ {type: 'math', value: 'y'},+ ]);+ expect(parseMarkdownAst('```\nplain\n```').children[0]).toEqual({+ type: 'code',+ lang: null,+ value: 'plain',+ });+ expect(parseMarkdown('```\nplain\n```')[0]).toEqual({+ type: 'codeblock',+ language: 'plaintext',+ content: 'plain',+ });+ });++ it('keeps source provenance canonical while projecting the released range', () => {+ const source = '# Heading\n\nParagraph';+ const ast = parseMarkdownAst(source, {sourceRanges: true});+ const legacy = parseMarkdown(source, {sourceRanges: true});++ expect(ast.children[0].position).toEqual({+ start: {offset: 0},+ end: {offset: 9},+ });+ expect(legacy[0]).toEqual({+ type: 'heading',+ level: 1,+ children: [{type: 'text', content: 'Heading'}],+ range: {start: 0, end: 9},+ });+ expect('position' in legacy[0]).toBe(false);+ });++ it('preserves optional legacy keys and insertion order exactly', () => {+ const [image, list] = parseMarkdown('\n\n- item');++ expect(Object.keys(image)).toEqual(['type', 'alt', 'src']);+ expect(Object.keys(list)).toEqual([+ 'type',+ 'ordered',+ 'start',+ 'delimiter',+ 'loose',+ 'items',+ ]);+ if (list.type !== 'list') {+ throw new Error('Expected list');+ }+ expect(Object.keys(list.items[0])).toEqual(['checked', 'children']);+ expect(list).toStrictEqual({+ type: 'list',+ ordered: false,+ start: undefined,+ delimiter: undefined,+ loose: undefined,+ items: [+ {+ checked: undefined,+ children: [+ {+ type: 'paragraph',+ children: [{type: 'text', content: 'item'}],+ },+ ],+ },+ ],+ });+ });++ it('does not expose canonical fields through released parser results', () => {+ const state = createIncrementalState();+ const blocks = parseMarkdown('# Heading\n\nParagraph');++ expect(blocks[0]).toEqual({+ type: 'heading',+ level: 1,+ children: [{type: 'text', content: 'Heading'}],+ });+ expect('depth' in blocks[0]).toBe(false);+ expect(state.settledBlocks).toEqual([]);+ });+});diff --git a/packages/core/src/Markdown/ast.ts b/packages/core/src/Markdown/ast.tsnew file mode 100644index 0000000000..adb8f23c9c--- /dev/null+++ b/packages/core/src/Markdown/ast.ts@@ -0,0 +1,231 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file ast.ts+ * @input Parsed Markdown structure and optional Astryx extension nodes+ * @output Canonical immutable MDAST-aligned Markdown node types and text projection+ * @position Internal canonical Markdown tree shared by parsing, rendering, and Outline+ */++export interface MarkdownAstPoint {+ readonly line?: number;+ readonly column?: number;+ readonly offset?: number;+}++export interface MarkdownAstPosition {+ readonly start: MarkdownAstPoint;+ readonly end: MarkdownAstPoint;+}++export interface MarkdownAstNodeBase {+ readonly position?: MarkdownAstPosition;+}++export interface MarkdownAstExtensionNode<+ Plugin extends string = string,+ Name extends string = string,+ Data = Readonly<Record<string, unknown>>,+ Display extends 'inline' | 'block' = 'inline' | 'block',+> extends MarkdownAstNodeBase {+ readonly type: 'extension';+ readonly plugin: Plugin;+ readonly name: Name;+ readonly data: Data;+ readonly display: Display;+ readonly source?: string;+}++export interface MarkdownAstText extends MarkdownAstNodeBase {+ readonly type: 'text';+ readonly value: string;+}++export interface MarkdownAstParent<+ Type extends 'strong' | 'emphasis' | 'delete',+ Extension extends MarkdownAstExtensionNode = never,+> extends MarkdownAstNodeBase {+ readonly type: Type;+ readonly children: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>;+}++export interface MarkdownAstInlineCode extends MarkdownAstNodeBase {+ readonly type: 'inlineCode';+ readonly value: string;+}++export interface MarkdownAstInlineMath extends MarkdownAstNodeBase {+ readonly type: 'inlineMath';+ readonly value: string;+}++export interface MarkdownAstLink<+ Extension extends MarkdownAstExtensionNode = never,+> extends MarkdownAstNodeBase {+ readonly type: 'link';+ readonly url: string;+ readonly children: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>;+}++export interface MarkdownAstImage extends MarkdownAstNodeBase {+ readonly type: 'image';+ readonly url: string;+ readonly alt: string;+}++export interface MarkdownAstCitation extends MarkdownAstNodeBase {+ readonly type: 'citation';+ readonly sourceId: string;+}++export interface MarkdownAstBreak extends MarkdownAstNodeBase {+ readonly type: 'break';+}++export type MarkdownAstPhrasingContent<+ Extension extends MarkdownAstExtensionNode = never,+> =+ | MarkdownAstText+ | MarkdownAstParent<'strong', Extension>+ | MarkdownAstParent<'emphasis', Extension>+ | MarkdownAstParent<'delete', Extension>+ | MarkdownAstInlineCode+ | MarkdownAstInlineMath+ | MarkdownAstLink<Extension>+ | MarkdownAstImage+ | MarkdownAstCitation+ | MarkdownAstBreak+ | (Extension & {readonly display: 'inline'});++export interface MarkdownAstHeading<+ Extension extends MarkdownAstExtensionNode = never,+> extends MarkdownAstNodeBase {+ readonly type: 'heading';+ readonly depth: 1 | 2 | 3 | 4 | 5 | 6;+ readonly children: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>;+}++export interface MarkdownAstParagraph<+ Extension extends MarkdownAstExtensionNode = never,+> extends MarkdownAstNodeBase {+ readonly type: 'paragraph';+ readonly children: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>;+}++export interface MarkdownAstCode extends MarkdownAstNodeBase {+ readonly type: 'code';+ readonly lang: string | null;+ readonly value: string;+}++export interface MarkdownAstMath extends MarkdownAstNodeBase {+ readonly type: 'math';+ readonly value: string;+}++export interface MarkdownAstBlockquote<+ Extension extends MarkdownAstExtensionNode = never,+> extends MarkdownAstNodeBase {+ readonly type: 'blockquote';+ readonly children: ReadonlyArray<MarkdownAstBlockContent<Extension>>;+}++export interface MarkdownAstListItem<+ Extension extends MarkdownAstExtensionNode = never,+> extends MarkdownAstNodeBase {+ readonly type: 'listItem';+ readonly checked?: boolean;+ readonly children: ReadonlyArray<MarkdownAstBlockContent<Extension>>;+}++export interface MarkdownAstList<+ Extension extends MarkdownAstExtensionNode = never,+> extends MarkdownAstNodeBase {+ readonly type: 'list';+ readonly ordered: boolean;+ readonly start?: number;+ readonly spread?: boolean;+ /** Astryx-preserved ordered-list marker delimiter. */+ readonly delimiter?: '.' | ')';+ readonly children: ReadonlyArray<MarkdownAstListItem<Extension>>;+}++export type MarkdownAstTableAlignment = 'left' | 'center' | 'right' | null;++export interface MarkdownAstTableCell<+ Extension extends MarkdownAstExtensionNode = never,+> extends MarkdownAstNodeBase {+ readonly type: 'tableCell';+ readonly children: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>;+}++export interface MarkdownAstTableRow<+ Extension extends MarkdownAstExtensionNode = never,+> extends MarkdownAstNodeBase {+ readonly type: 'tableRow';+ readonly children: ReadonlyArray<MarkdownAstTableCell<Extension>>;+}++export interface MarkdownAstTable<+ Extension extends MarkdownAstExtensionNode = never,+> extends MarkdownAstNodeBase {+ readonly type: 'table';+ readonly align: ReadonlyArray<MarkdownAstTableAlignment>;+ readonly children: ReadonlyArray<MarkdownAstTableRow<Extension>>;+}++export interface MarkdownAstThematicBreak extends MarkdownAstNodeBase {+ readonly type: 'thematicBreak';+}++export type MarkdownAstBlockContent<+ Extension extends MarkdownAstExtensionNode = never,+> =+ | MarkdownAstHeading<Extension>+ | MarkdownAstParagraph<Extension>+ | MarkdownAstCode+ | MarkdownAstMath+ | MarkdownAstBlockquote<Extension>+ | MarkdownAstList<Extension>+ | MarkdownAstTable<Extension>+ | MarkdownAstThematicBreak+ | MarkdownAstImage+ | (Extension & {readonly display: 'block'});++export interface MarkdownAstRoot<+ Extension extends MarkdownAstExtensionNode = never,+> extends MarkdownAstNodeBase {+ readonly type: 'root';+ readonly children: ReadonlyArray<MarkdownAstBlockContent<Extension>>;+}++export function markdownAstText(+ nodes: ReadonlyArray<MarkdownAstPhrasingContent>,+): string {+ let text = '';+ for (const node of nodes) {+ switch (node.type) {+ case 'text':+ case 'inlineCode':+ case 'inlineMath':+ text += node.value;+ break;+ case 'strong':+ case 'emphasis':+ case 'delete':+ case 'link':+ text += markdownAstText(node.children);+ break;+ case 'image':+ text += node.alt;+ break;+ case 'citation':+ case 'break':+ break;+ default: {+ node satisfies never;+ }+ }+ }+ return text;+}diff --git a/packages/core/src/Markdown/incremental.test.ts b/packages/core/src/Markdown/incremental.test.tsindex 18b2df4578..77e2eb1bef 100644--- a/packages/core/src/Markdown/incremental.test.ts+++ b/packages/core/src/Markdown/incremental.test.ts@@ -47,6 +47,18 @@ describe('parseMarkdownIncremental', () => {}});+ it('keeps projected settled-list identity when a following block settles too', () => {+ const state = createIncrementalState();+ parseMarkdownIncremental('1. a\n\nTail', state);+ const result = parseMarkdownIncremental(+ '1. a\n\n1. b\n\nAfter\n\nTail',+ state,+ );++ expect(state.settledBlocks).toEqual(parseMarkdown(state.settledText));+ expect(result[0]).toBe(state.settledBlocks[0]);+ });+it('joins loose ) ordered list across streamed chunks', () => {const text = '1) apple\n\n1) banana\n\n1) cherry\n';const {final} = simulateStreaming(text, 5);diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.tsindex 6dee3b2eb9..af2e82e7a4 100644--- a/packages/core/src/Markdown/parser.ts+++ b/packages/core/src/Markdown/parser.ts@@ -2,14 +2,22 @@/*** @file parser.ts- * @input Markdown string- * @output Array of MarkdownNode AST nodes, including opt-in math; heading slug- * helpers- * (inlineText, slugify, uniqueSlug) shared by Markdown rendering and- * Outline's parseOutlineFromMarkdown- * @position Core parser; consumed by Markdown.tsx and Outline+ * @input Markdown string and released parse options+ * @output Canonical MDAST-aligned nodes for internal consumers plus unchanged+ * released parser-node projections; shared heading slug helpers+ * @position Core parser and compatibility boundary; consumed by Markdown and Outline*/+import type {+ MarkdownAstBlockContent,+ MarkdownAstListItem,+ MarkdownAstPhrasingContent,+ MarkdownAstPosition,+ MarkdownAstRoot,+ MarkdownAstTableCell,+ MarkdownAstTableRow,+} from './ast';+// ---------------------------------------------------------------------------// Types// ---------------------------------------------------------------------------@@ -130,6 +138,199 @@ export type TableCellNode = {children: InlineNode[]};type TableCellNodeWithMath = {children: InlineNodeWithMath[]};export type TableAlignment = 'left' | 'center' | 'right' | null;+type LegacyProjectionCache = WeakMap<object, object>;++function projectRange(+ position: MarkdownAstPosition | undefined,+): BlockMetadata {+ const start = position?.start.offset;+ const end = position?.end.offset;+ return start == null || end == null ? {} : {range: {start, end}};+}++function projectInlineNode(+ node: MarkdownAstPhrasingContent,+ cache: LegacyProjectionCache,+): InlineNodeWithMath {+ const cached = cache.get(node);+ if (cached != null) {+ return cached as InlineNodeWithMath;+ }+ let projected: InlineNodeWithMath;+ switch (node.type) {+ case 'text':+ projected = {type: 'text', content: node.value};+ break;+ case 'strong':+ projected = {+ type: 'bold',+ children: node.children.map(child => projectInlineNode(child, cache)),+ };+ break;+ case 'emphasis':+ projected = {+ type: 'italic',+ children: node.children.map(child => projectInlineNode(child, cache)),+ };+ break;+ case 'delete':+ projected = {+ type: 'strikethrough',+ children: node.children.map(child => projectInlineNode(child, cache)),+ };+ break;+ case 'inlineCode':+ projected = {type: 'code', content: node.value};+ break;+ case 'inlineMath':+ projected = {type: 'math', value: node.value};+ break;+ case 'link':+ projected = {+ type: 'link',+ href: node.url,+ children: node.children.map(child => projectInlineNode(child, cache)),+ };+ break;+ case 'image':+ projected = {type: 'image', src: node.url, alt: node.alt};+ break;+ case 'citation':+ projected = {type: 'citation', sourceId: node.sourceId};+ break;+ case 'break':+ projected = {type: 'break'};+ break;+ }+ cache.set(node, projected);+ return projected;+}++function projectTableCell(+ node: MarkdownAstTableCell,+ cache: LegacyProjectionCache,+): TableCellNodeWithMath {+ const cached = cache.get(node);+ if (cached != null) {+ return cached as TableCellNodeWithMath;+ }+ const projected = {+ children: node.children.map(child => projectInlineNode(child, cache)),+ };+ cache.set(node, projected);+ return projected;+}++function projectListItem(+ node: MarkdownAstListItem,+ cache: LegacyProjectionCache,+): ListItemNodeWithMath {+ const cached = cache.get(node);+ if (cached != null) {+ return cached as ListItemNodeWithMath;+ }+ const projected = {+ checked: node.checked,+ children: node.children.map(child => projectBlockNode(child, cache)),+ };+ cache.set(node, projected);+ return projected;+}++function projectBlockNode(+ node: MarkdownAstBlockContent,+ cache: LegacyProjectionCache,+): BlockNodeWithMath {+ const cached = cache.get(node);+ if (cached != null) {+ return cached as BlockNodeWithMath;+ }+ const metadata = projectRange(node.position);+ let projected: BlockNodeWithMath;+ switch (node.type) {+ case 'heading':+ projected = {+ type: 'heading',+ level: node.depth,+ children: node.children.map(child => projectInlineNode(child, cache)),+ ...metadata,+ };+ break;+ case 'paragraph':+ projected = {+ type: 'paragraph',+ children: node.children.map(child => projectInlineNode(child, cache)),+ ...metadata,+ };+ break;+ case 'code':+ projected = {+ type: 'codeblock',+ language: node.lang ?? 'plaintext',+ content: node.value,+ ...metadata,+ };+ break;+ case 'math':+ projected = {type: 'math', value: node.value, ...metadata};+ break;+ case 'blockquote':+ projected = {+ type: 'blockquote',+ children: node.children.map(child => projectBlockNode(child, cache)),+ ...metadata,+ };+ break;+ case 'list':+ projected = {+ type: 'list',+ ordered: node.ordered,+ start: node.start,+ delimiter: node.delimiter,+ loose: node.spread,+ items: node.children.map(item => projectListItem(item, cache)),+ ...metadata,+ };+ break;+ case 'table': {+ const [header = {type: 'tableRow' as const, children: []}, ...rows] =+ node.children;+ projected = {+ type: 'table',+ headers: header.children.map(cell => projectTableCell(cell, cache)),+ alignments: [...node.align],+ rows: rows.map(row =>+ row.children.map(cell => projectTableCell(cell, cache)),+ ),+ ...metadata,+ };+ break;+ }+ case 'thematicBreak':+ projected = {type: 'hr', ...metadata};+ break;+ case 'image':+ projected = {type: 'image', alt: node.alt, src: node.url, ...metadata};+ break;+ }+ cache.set(node, projected);+ return projected;+}++function projectInlineNodes(+ nodes: ReadonlyArray<MarkdownAstPhrasingContent>,+ cache: LegacyProjectionCache = new WeakMap(),+): InlineNodeWithMath[] {+ return nodes.map(node => projectInlineNode(node, cache));+}++function projectMarkdownRoot(+ root: MarkdownAstRoot,+ cache: LegacyProjectionCache = new WeakMap(),+): BlockNodeWithMath[] {+ return root.children.map(node => projectBlockNode(node, cache));+}+// ---------------------------------------------------------------------------// Parse options// ---------------------------------------------------------------------------@@ -201,7 +402,10 @@ type ResolvedOptions = {readonly linkDefs?: ReadonlyMap<string, string>;};-const EMPTY_OPTS: ResolvedOptions = {sourceIds: undefined, autolink: undefined};+const EMPTY_OPTS: ResolvedOptions = {+ sourceIds: undefined,+ autolink: undefined,+};function resolveOptions(arg: ReadonlySet<string> | RuntimeParseOptions | undefined,@@ -542,7 +746,7 @@ function matchReferenceLink(start: number,linkDefs: ReadonlyMap<string, string>,opts: ResolvedOptions,-): {node: InlineNodeWithMath; end: number} | null {+): {node: MarkdownAstPhrasingContent; end: number} | null {const textClose = text.indexOf(']', start + 1);if (textClose === -1) {return null;@@ -560,7 +764,11 @@ function matchReferenceLink(const href = linkDefs.get(normalizeLinkLabel(label));if (href != null && isSafeUrl(href)) {return {- node: {type: 'link', href, children: parseInlineImpl(linkText, opts)},+ node: {+ type: 'link',+ url: href,+ children: parseInlineImpl(linkText, opts),+ },end: labelClose + 1,};}@@ -577,7 +785,7 @@ function matchReferenceLink(return null;}return {- node: {type: 'link', href, children: parseInlineImpl(linkText, opts)},+ node: {type: 'link', url: href, children: parseInlineImpl(linkText, opts)},end: textClose + 1,};}@@ -587,7 +795,7 @@ function matchReferenceImage(text: string,start: number,linkDefs: ReadonlyMap<string, string>,-): {node: InlineNodeWithMath; end: number} | null {+): {node: MarkdownAstPhrasingContent; end: number} | null {const altClose = text.indexOf(']', start + 2);if (altClose === -1) {return null;@@ -600,7 +808,7 @@ function matchReferenceImage(const label = rawLabel === '' ? alt : rawLabel;const src = linkDefs.get(normalizeLinkLabel(label));if (src != null && isSafeUrl(src)) {- return {node: {type: 'image', src, alt}, end: labelClose + 1};+ return {node: {type: 'image', url: src, alt}, end: labelClose + 1};}// No match — fall back to a shortcut `![alt]`.}@@ -612,7 +820,7 @@ function matchReferenceImage(if (src == null || !isSafeUrl(src)) {return null;}- return {node: {type: 'image', src, alt}, end: altClose + 1};+ return {node: {type: 'image', url: src, alt}, end: altClose + 1};}// ---------------------------------------------------------------------------@@ -791,6 +999,14 @@ export function parseInline(text: string,arg?: ReadonlySet<string> | RuntimeParseOptions,): InlineNodeWithMath[] {+ return projectInlineNodes(parseInlineAst(text, arg));+}++/** @internal Canonical inline parse used by Markdown rendering. */+export function parseInlineAst(+ text: string,+ arg?: ReadonlySet<string> | RuntimeParseOptions,+): MarkdownAstPhrasingContent[] {return parseInlineEntry(text, resolveOptions(arg));}@@ -806,7 +1022,7 @@ export function parseInline(function parseInlineEntry(text: string,opts: ResolvedOptions,-): InlineNodeWithMath[] {+): MarkdownAstPhrasingContent[] {const nodes = parseInlineImpl(text, opts);return opts.autolink === 'gfm' ? transformAutolinks(nodes) : nodes;}@@ -814,14 +1030,14 @@ function parseInlineEntry(function parseInlineImpl(text: string,opts: ResolvedOptions,-): InlineNodeWithMath[] {- const nodes: InlineNodeWithMath[] = [];+): MarkdownAstPhrasingContent[] {+ const nodes: MarkdownAstPhrasingContent[] = [];let i = 0;while (i < text.length) {// --- Escape ---if (text[i] === '\\' && i + 1 < text.length) {- nodes.push({type: 'text', content: text[i + 1]});+ nodes.push({type: 'text', value: text[i + 1]});i += 2;continue;}@@ -832,7 +1048,10 @@ function parseInlineImpl(const openIndex = i + tickCount;const closeIndex = text.indexOf('`'.repeat(tickCount), openIndex);if (closeIndex !== -1) {- nodes.push({type: 'code', content: text.slice(openIndex, closeIndex)});+ nodes.push({+ type: 'inlineCode',+ value: text.slice(openIndex, closeIndex),+ });i = closeIndex + tickCount;continue;}@@ -842,7 +1061,10 @@ function parseInlineImpl(if (opts.math && text[i] === '$') {const closeIndex = findInlineMathEnd(text, i);if (closeIndex !== -1) {- nodes.push({type: 'math', value: text.slice(i + 1, closeIndex)});+ nodes.push({+ type: 'inlineMath',+ value: text.slice(i + 1, closeIndex),+ });i = closeIndex + 1;continue;}@@ -867,11 +1089,11 @@ function parseInlineImpl(const src = text.slice(altClose + 2, srcClose);if (!isSafeUrl(src)) {// Dangerous scheme — emit as plain text.- nodes.push({type: 'text', content: text.slice(i, srcClose + 1)});+ nodes.push({type: 'text', value: text.slice(i, srcClose + 1)});} else {nodes.push({type: 'image',- src,+ url: src,alt: text.slice(i + 2, altClose),});}@@ -910,11 +1132,11 @@ function parseInlineImpl(const href = text.slice(textClose + 2, urlClose);if (!isSafeUrl(href)) {// Dangerous scheme — emit as plain text instead of a link.- nodes.push({type: 'text', content: text.slice(i, urlClose + 1)});+ nodes.push({type: 'text', value: text.slice(i, urlClose + 1)});} else {nodes.push({type: 'link',- href,+ url: href,children: parseInlineImpl(text.slice(i + 1, textClose), opts),});}@@ -950,10 +1172,10 @@ function parseInlineImpl((!isUnderscore || !isWordChar(text[closeIndex + 3]))) {nodes.push({- type: 'bold',+ type: 'strong',children: [{- type: 'italic',+ type: 'emphasis',children: parseInlineImpl(text.slice(i + 3, closeIndex), opts),},],@@ -980,7 +1202,7 @@ function parseInlineImpl((!isUnderscore || !isWordChar(text[closeIndex + 2]))) {nodes.push({- type: 'bold',+ type: 'strong',children: parseInlineImpl(text.slice(i + 2, closeIndex), opts),});i = closeIndex + 2;@@ -994,7 +1216,7 @@ function parseInlineImpl(const closeIndex = text.indexOf('~~', i + 2);if (closeIndex !== -1) {nodes.push({- type: 'strikethrough',+ type: 'delete',children: parseInlineImpl(text.slice(i + 2, closeIndex), opts),});i = closeIndex + 2;@@ -1015,7 +1237,7 @@ function parseInlineImpl((!isUnderscore || !isWordChar(text[closeIndex + 1]))) {nodes.push({- type: 'italic',+ type: 'emphasis',children: parseInlineImpl(text.slice(i + 1, closeIndex), opts),});i = closeIndex + 1;@@ -1043,9 +1265,12 @@ function parseInlineImpl(if (trimmed.length > 0) {const last = nodes[nodes.length - 1];if (last?.type === 'text') {- last.content += trimmed;+ nodes[nodes.length - 1] = {+ ...last,+ value: last.value + trimmed,+ };} else {- nodes.push({type: 'text', content: trimmed});+ nodes.push({type: 'text', value: trimmed});}}nodes.push({type: 'break'});@@ -1056,9 +1281,9 @@ function parseInlineImpl(const last = nodes[nodes.length - 1];if (last?.type === 'text') {- last.content += content;+ nodes[nodes.length - 1] = {...last, value: last.value + content};} else {- nodes.push({type: 'text', content});+ nodes.push({type: 'text', value: content});}i = end;}@@ -1303,26 +1528,26 @@ function scanAutolinksInText(text: string): AutolinkMatch[] {* Split a text-node `content` string into a sequence of text + link nodes* based on autolink matches.*/-function splitTextOnAutolinks(content: string): InlineNodeWithMath[] {+function splitTextOnAutolinks(content: string): MarkdownAstPhrasingContent[] {const matches = scanAutolinksInText(content);if (matches.length === 0) {- return [{type: 'text', content}];+ return [{type: 'text', value: content}];}- const out: InlineNodeWithMath[] = [];+ const out: MarkdownAstPhrasingContent[] = [];let cursor = 0;for (const m of matches) {if (m.start > cursor) {- out.push({type: 'text', content: content.slice(cursor, m.start)});+ out.push({type: 'text', value: content.slice(cursor, m.start)});}out.push({type: 'link',- href: m.href,- children: [{type: 'text', content: m.display}],+ url: m.href,+ children: [{type: 'text', value: m.display}],});cursor = m.end;}if (cursor < content.length) {- out.push({type: 'text', content: content.slice(cursor)});+ out.push({type: 'text', value: content.slice(cursor)});}return out;}@@ -1335,23 +1560,25 @@ function splitTextOnAutolinks(content: string): InlineNodeWithMath[] {* `image` alt text, `citation`, or `break`. Runs only on the outermost* block's inline tree (see `parseInlineEntry`).*/-function transformAutolinks(nodes: InlineNodeWithMath[]): InlineNodeWithMath[] {- const out: InlineNodeWithMath[] = [];+function transformAutolinks(+ nodes: ReadonlyArray<MarkdownAstPhrasingContent>,+): MarkdownAstPhrasingContent[] {+ const out: MarkdownAstPhrasingContent[] = [];for (const node of nodes) {if (node.type === 'text') {- const split = splitTextOnAutolinks(node.content);+ const split = splitTextOnAutolinks(node.value);for (const seg of split) {const last = out[out.length - 1];if (seg.type === 'text' && last?.type === 'text') {- last.content += seg.content;+ out[out.length - 1] = {...last, value: last.value + seg.value};} else {out.push(seg);}}} else if (- node.type === 'bold' ||- node.type === 'italic' ||- node.type === 'strikethrough'+ node.type === 'strong' ||+ node.type === 'emphasis' ||+ node.type === 'delete') {out.push({...node, children: transformAutolinks(node.children)});} else {@@ -1510,9 +1737,9 @@ function parseTable(lines: string[],lineIndex: number,opts: ResolvedOptions,-): {node: BlockNodeWithMath; nextIndex: number} {- const headers: TableCellNodeWithMath[] = splitTableRow(lines[lineIndex]).map(- cell => ({children: parseInlineEntry(cell, opts)}),+): {node: MarkdownAstBlockContent; nextIndex: number} {+ const headers: MarkdownAstTableCell[] = splitTableRow(lines[lineIndex]).map(+ cell => ({type: 'tableCell', children: parseInlineEntry(cell, opts)}),);const alignments: TableAlignment[] = splitTableRow(lines[lineIndex + 1]).map(cell => {@@ -1528,22 +1755,28 @@ function parseTable(: null;},);- const rows: TableCellNodeWithMath[][] = [];+ const rows: MarkdownAstTableRow[] = [];let rowIndex = lineIndex + 2;while (rowIndex < lines.length &&lines[rowIndex].includes('|') &&lines[rowIndex].trim() !== '') {- rows.push(- splitTableRow(lines[rowIndex]).map(cell => ({+ rows.push({+ type: 'tableRow',+ children: splitTableRow(lines[rowIndex]).map(cell => ({+ type: 'tableCell',children: parseInlineEntry(cell, opts),})),- );+ });rowIndex++;}return {- node: {type: 'table', headers, alignments, rows},+ node: {+ type: 'table',+ align: alignments,+ children: [{type: 'tableRow', children: headers}, ...rows],+ },nextIndex: rowIndex,};}@@ -1553,8 +1786,8 @@ function parseList(startIndex: number,ordered: boolean,opts: ResolvedOptions,-): {node: BlockNodeWithMath; nextIndex: number} {- const items: ListItemNodeWithMath[] = [];+): {node: MarkdownAstBlockContent; nextIndex: number} {+ const items: MarkdownAstListItem[] = [];const baseIndent = getIndent(lines[startIndex]);// Ordered lists may use either '.' or ')' as the marker delimiter// (CommonMark 5.2). Capture which one this list starts with so its items@@ -1608,7 +1841,11 @@ function parseList(itemText += '\n' + deindented.join('\n');}- items.push({checked, children: parseMarkdownImpl(itemText, nested(opts))});+ items.push({+ type: 'listItem',+ checked,+ children: parseMarkdownImpl(itemText, nested(opts)),+ });// CommonMark loose list: blank line(s) between items of the same style// and indent still form one list. Skip the blanks and continue if the@@ -1632,8 +1869,8 @@ function parseList(ordered,start,delimiter: ordered ? (delim as '.' | ')') : undefined,- loose: loose || undefined,- items,+ spread: loose || undefined,+ children: items,},nextIndex: index,};@@ -1659,13 +1896,24 @@ export function parseMarkdown(input: string,arg?: ReadonlySet<string> | RuntimeParseOptions,): BlockNodeWithMath[] {- return parseMarkdownImpl(input, resolveOptions(arg));+ return projectMarkdownRoot(parseMarkdownAst(input, arg));+}++/** @internal Canonical block parse used by Markdown and Outline rendering. */+export function parseMarkdownAst(+ input: string,+ arg?: ReadonlySet<string> | RuntimeParseOptions,+): MarkdownAstRoot {+ return {+ type: 'root',+ children: parseMarkdownImpl(input, resolveOptions(arg)),+ };}function parseMarkdownImpl(input: string,baseOpts: ResolvedOptions,-): BlockNodeWithMath[] {+): MarkdownAstBlockContent[] {// Collect this input's link reference definitions and strip their lines,// then merge them with any definitions inherited from an enclosing parse// (the incremental parser passes the whole document's definitions in; a@@ -1686,7 +1934,7 @@ function parseMarkdownImpl(const opts: ResolvedOptions =linkDefs != null ? {...baseOpts, linkDefs} : baseOpts;const lines = cleaned.split('\n');- const blocks: BlockNodeWithMath[] = [];+ const blocks: MarkdownAstBlockContent[] = [];// The line each block started on, parallel to `blocks`. Only collected when// ranges were asked for; a block's end is resolved after the loop, since the// branch that produced it has already moved `index` past whatever it read.@@ -1697,7 +1945,7 @@ function parseMarkdownImpl(? []: null;let blockStartLine = 0;- const pushBlock = (node: BlockNodeWithMath, endLine?: number) => {+ const pushBlock = (node: MarkdownAstBlockContent, endLine?: number) => {blocks.push(node);blockStartLines?.push(blockStartLine);blockEndLines?.push(endLine);@@ -1716,7 +1964,7 @@ function parseMarkdownImpl(const fenceMatch = line.match(/^(`{3,}|~{3,})(\w*)/);if (fenceMatch) {const fence = fenceMatch[1];- const language = fenceMatch[2] || 'plaintext';+ const language = fenceMatch[2] || null;const codeLines: string[] = [];index++;while (index < lines.length && !lines[index].startsWith(fence)) {@@ -1728,7 +1976,7 @@ function parseMarkdownImpl(// can end on them, so it states its own end rather than letting the// positional derivation trim them off.pushBlock(- {type: 'codeblock', language, content: codeLines.join('\n')},+ {type: 'code', lang: language, value: codeLines.join('\n')},Math.min(index, lines.length) - 1,);continue;@@ -1752,7 +2000,7 @@ function parseMarkdownImpl(if (headingMatch) {pushBlock({type: 'heading',- level: headingMatch[1].length as 1 | 2 | 3 | 4 | 5 | 6,+ depth: headingMatch[1].length as 1 | 2 | 3 | 4 | 5 | 6,children: parseInlineEntry(headingMatch[2], opts),});index++;@@ -1761,7 +2009,7 @@ function parseMarkdownImpl(// --- HR (must precede list check to handle `- - -`, `* * *`, `_ _ _`) ---if (isHorizontalRule(line)) {- pushBlock({type: 'hr'});+ pushBlock({type: 'thematicBreak'});index++;continue;}@@ -1775,7 +2023,7 @@ function parseMarkdownImpl(line.trim() === imageMatch[0] &&isSafeUrl(imageMatch[2])) {- pushBlock({type: 'image', alt: imageMatch[1], src: imageMatch[2]});+ pushBlock({type: 'image', alt: imageMatch[1], url: imageMatch[2]});index++;continue;}@@ -1866,7 +2114,7 @@ function parseMarkdownImpl(* `lineMap` says which input line each surviving line came from.*/function stampSourceRanges(- blocks: BlockNodeWithMath[],+ blocks: MarkdownAstBlockContent[],blockStartLines: number[],blockEndLines: (number | undefined)[],lines: string[],@@ -1903,7 +2151,13 @@ function stampSourceRanges(// range that dropped it would slice to something that re-parses// differently.const end = lineStart(endLine) + lines[endLine].length;- blocks[i] = {...blocks[i], range: {start: lineStart(startLine), end}};+ blocks[i] = {+ ...blocks[i],+ position: {+ start: {offset: lineStart(startLine)},+ end: {offset: end},+ },+ };}}@@ -1968,6 +2222,13 @@ type IncrementalCache = {/** The effective document-global definitions used by slice parses. */linkDefs: ReadonlyMap<string, string>;linkDefsKey: string;+ /** Canonical settled blocks shared by rendering and compatibility projection. */+ settledAstBlocks: MarkdownAstBlockContent[];+ settledRevision: number;+ projectedRevision: number;+ projectedSettledBlocks: BlockNodeWithMath[];+ /** Preserves released settled-node identity across projected snapshots. */+ projectionCache: LegacyProjectionCache;work: IncrementalWork;};@@ -1986,6 +2247,11 @@ function makeIncrementalCache(tailLinkDefs: new Map(),linkDefs: defs,linkDefsKey: linkDefsSignature(defs),+ settledAstBlocks: [],+ settledRevision: 0,+ projectedRevision: -1,+ projectedSettledBlocks: [],+ projectionCache: new WeakMap(),work: {splitCharacters: 0,boundaryLines: 0,@@ -2481,9 +2747,9 @@ function atOffset(opts: ResolvedOptions, offset: number): ResolvedOptions {* lists even though the full-text parser joins them per CommonMark §5.3.*/function mergeSettledBlocks(- prev: BlockNodeWithMath[],- delta: BlockNodeWithMath[],-): BlockNodeWithMath[] {+ prev: MarkdownAstBlockContent[],+ delta: MarkdownAstBlockContent[],+): MarkdownAstBlockContent[] {if (prev.length === 0 || delta.length === 0) {return [...prev, ...delta];}@@ -2495,17 +2761,21 @@ function mergeSettledBlocks(prevLast.ordered === deltaFirst.ordered &&prevLast.delimiter === deltaFirst.delimiter) {- const merged: BlockNodeWithMath = {+ const merged: MarkdownAstBlockContent = {type: 'list',ordered: prevLast.ordered,start: prevLast.start,delimiter: prevLast.delimiter,- loose: true,- items: [...prevLast.items, ...deltaFirst.items],- // One list now, so one range: from where the first half started to- // where the second half ended.- ...(prevLast.range != null && deltaFirst.range != null- ? {range: {start: prevLast.range.start, end: deltaFirst.range.end}}+ spread: true,+ children: [...prevLast.children, ...deltaFirst.children],+ // One list now, so one position spans both halves.+ ...(prevLast.position != null && deltaFirst.position != null+ ? {+ position: {+ start: prevLast.position.start,+ end: deltaFirst.position.end,+ },+ }: null),};return [...prev.slice(0, -1), merged, ...delta.slice(1)];@@ -2515,15 +2785,15 @@ function mergeSettledBlocks(/** Append a newly-settled slice without copying the already-settled prefix. */function appendSettledBlocks(- prev: BlockNodeWithMath[],- delta: BlockNodeWithMath[],-): void {+ prev: MarkdownAstBlockContent[],+ delta: MarkdownAstBlockContent[],+): boolean {if (delta.length === 0) {- return;+ return false;}if (prev.length === 0) {prev.push(...delta);- return;+ return false;}const prevLast = prev[prev.length - 1];const deltaFirst = delta[0];@@ -2538,16 +2808,22 @@ function appendSettledBlocks(ordered: prevLast.ordered,start: prevLast.start,delimiter: prevLast.delimiter,- loose: true,- items: [...prevLast.items, ...deltaFirst.items],- ...(prevLast.range != null && deltaFirst.range != null- ? {range: {start: prevLast.range.start, end: deltaFirst.range.end}}+ spread: true,+ children: [...prevLast.children, ...deltaFirst.children],+ ...(prevLast.position != null && deltaFirst.position != null+ ? {+ position: {+ start: prevLast.position.start,+ end: deltaFirst.position.end,+ },+ }: null),};prev.push(...delta.slice(1));- return;+ return true;}prev.push(...delta);+ return false;}function sameUnsettledDefinitions(@@ -2583,6 +2859,11 @@ function resetIncrementalCache(cache.tailLinkDefs = new Map();cache.linkDefs = new Map();cache.linkDefsKey = '';+ cache.settledAstBlocks = [];+ cache.settledRevision++;+ cache.projectedRevision = -1;+ cache.projectedSettledBlocks = [];+ cache.projectionCache = new WeakMap();cache.work = {splitCharacters: 0,boundaryLines: 0,@@ -2622,6 +2903,36 @@ export function parseMarkdownIncremental(state: IncrementalState<boolean>,arg?: ReadonlySet<string> | RuntimeParseOptions,): BlockNodeWithMath[] {+ const root = parseMarkdownAstIncremental(input, state, arg);+ const cache = incrementalCaches.get(state) ?? makeIncrementalCache(state);+ const projected = projectMarkdownRoot(root, cache.projectionCache);+ if (cache.projectedRevision !== cache.settledRevision) {+ cache.projectedSettledBlocks = cache.settledAstBlocks.map(block =>+ projectBlockNode(block, cache.projectionCache),+ );+ cache.projectedRevision = cache.settledRevision;+ }+ state.settledBlocks = cache.projectedSettledBlocks;+ return projected;+}++/** @internal Canonical incremental parse used by Markdown rendering. */+export function parseMarkdownAstIncremental(+ input: string,+ state: IncrementalState<boolean>,+ arg?: ReadonlySet<string> | RuntimeParseOptions,+): MarkdownAstRoot {+ return {+ type: 'root',+ children: parseMarkdownIncrementalAstBlocks(input, state, arg),+ };+}++function parseMarkdownIncrementalAstBlocks(+ input: string,+ state: IncrementalState<boolean>,+ arg?: ReadonlySet<string> | RuntimeParseOptions,+): MarkdownAstBlockContent[] {const opts = resolveOptions(arg);const cache = incrementalCaches.get(state) ?? makeIncrementalCache(state);let reparseSettled = false;@@ -2746,17 +3057,39 @@ export function parseMarkdownIncremental(let parsedSettledBlocks = 0;if (reparseSettled) {- state.settledBlocks = state.settledText+ cache.settledAstBlocks = state.settledText? parseMarkdownImpl(state.settledText, parseOpts): [];- parsedSettledBlocks = state.settledBlocks.length;+ parsedSettledBlocks = cache.settledAstBlocks.length;+ cache.settledRevision++;+ cache.projectedSettledBlocks = cache.settledAstBlocks.map(block =>+ projectBlockNode(block, cache.projectionCache),+ );+ cache.projectedRevision = cache.settledRevision;} else if (settledDelta !== '') {const deltaBlocks = parseMarkdownImpl(settledDelta,atOffset(parseOpts, oldSettledEnd),);- appendSettledBlocks(state.settledBlocks, deltaBlocks);+ const mergeIndex = cache.settledAstBlocks.length - 1;+ const mergedList = appendSettledBlocks(cache.settledAstBlocks, deltaBlocks);parsedSettledBlocks = deltaBlocks.length;+ cache.settledRevision++;+ const projectedDelta = deltaBlocks.map(block =>+ projectBlockNode(block, cache.projectionCache),+ );+ if (mergedList) {+ const canonicalMerged = cache.settledAstBlocks[mergeIndex];+ const projectedMerged = projectBlockNode(+ canonicalMerged,+ cache.projectionCache,+ );+ cache.projectedSettledBlocks[mergeIndex] = projectedMerged;+ cache.projectedSettledBlocks.push(...projectedDelta.slice(1));+ } else {+ cache.projectedSettledBlocks.push(...projectedDelta);+ }+ cache.projectedRevision = cache.settledRevision;}// The unsettled tail is trimmed before parsing, so its offset in the@@ -2790,7 +3123,7 @@ export function parseMarkdownIncremental(definitionCharacters: settledDelta.length + unsettledInput.length,renderedBlocks: parsedSettledBlocks + unsettledBlocks.length,};- return mergeSettledBlocks(state.settledBlocks, unsettledBlocks);+ return mergeSettledBlocks(cache.settledAstBlocks, unsettledBlocks);}// ---------------------------------------------------------------------------@@ -2801,30 +3134,6 @@ export function parseMarkdownIncremental(// derives its item ids from the same functions, so outline hash links always// resolve to a rendered heading by construction.-/** Flatten inline nodes into their plain text content. */-export function inlineText(nodes: ReadonlyArray<InlineNodeWithMath>): string {- return nodes- .map(node => {- switch (node.type) {- case 'text':- case 'code':- case 'math':- return node.type === 'math' ? node.value : node.content;- case 'bold':- case 'italic':- case 'strikethrough':- case 'link':- return inlineText(node.children);- case 'image':- return node.alt;- case 'citation':- case 'break':- return '';- }- })- .join('');-}-/** Turn heading text into a URL-safe slug (lowercase, hyphen-separated). */export function slugify(value: string): string {return valuediff --git a/packages/core/src/Outline/parseOutlineFromMarkdown.ts b/packages/core/src/Outline/parseOutlineFromMarkdown.tsindex 5d4e02aae0..00c5c2f36f 100644--- a/packages/core/src/Outline/parseOutlineFromMarkdown.ts+++ b/packages/core/src/Outline/parseOutlineFromMarkdown.ts@@ -2,22 +2,17 @@/*** @file parseOutlineFromMarkdown.ts- * @input Uses Markdown parser internals (parseMarkdown + heading slug- * helpers) and OutlineItem type+ * @input Uses Markdown's canonical AST parser and heading slug helpers* @output Exports parseOutlineFromMarkdown for extracting heading outlines from Markdown- * @position Pure utility; consumed by useOutlineFromMarkdown and public exports+ * @position Pure compatibility utility; consumed by useOutlineFromMarkdown and public exports** SYNC: When modified, update these files to stay in sync:* - /packages/core/src/Outline/Outline.doc.mjs* - /packages/core/src/Outline/index.ts*/-import {- parseMarkdown,- inlineText,- slugify,- uniqueSlug,-} from '../Markdown/parser';+import {parseMarkdownAst, slugify, uniqueSlug} from '../Markdown/parser';+import {markdownAstText} from '../Markdown/ast';import type {OutlineItem} from './types';/**@@ -30,14 +25,14 @@ import type {OutlineItem} from './types';*/export function parseOutlineFromMarkdown(markdown: string): OutlineItem[] {const counts = new Map<string, number>();- return parseMarkdown(markdown)- .filter(block => block.type === 'heading')+ return parseMarkdownAst(markdown)+ .children.filter(block => block.type === 'heading').map(block => {- const label = inlineText(block.children).trim();+ const label = markdownAstText(block.children).trim();return {id: uniqueSlug(slugify(label), counts),label,- level: block.level,+ level: block.depth,};});}