Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

refactor(markdown): use canonical mdast tree (#6339)

Cindy Zhangd4cc23232be0
diff --git a/packages/core/src/Markdown/Markdown.test.tsx b/packages/core/src/Markdown/Markdown.test.tsx
index 017cfc441a..98b9d2a8ff 100644
--- a/packages/core/src/Markdown/Markdown.test.tsx
+++ b/packages/core/src/Markdown/Markdown.test.tsx
@@ -80,6 +80,17 @@ describe('Markdown', () => {
}
});
+ it('keeps citation markers out of released heading ids', () => {
+ render(
+ <Markdown sources={{cite: {title: 'Citation'}}}>
+ {'# Before [cite] after'}
+ </Markdown>,
+ );
+ expect(
+ screen.getByRole('heading', {name: /Before.*after/}),
+ ).toHaveAttribute('id', 'before-after');
+ });
+
it('passes the generated id to a custom heading component', () => {
const received: (string | undefined)[] = [];
render(
diff --git a/packages/core/src/Markdown/Markdown.tsx b/packages/core/src/Markdown/Markdown.tsx
index 75e68af7c7..0d6fe93638 100644
--- a/packages/core/src/Markdown/Markdown.tsx
+++ b/packages/core/src/Markdown/Markdown.tsx
@@ -48,22 +48,21 @@ import type {CitationSource} from '../Citation/Citation';
import {useLinkComponent} from '../Link/useLinkComponent';
import type {LinkComponentType} from '../Link/types';
import {
- parseInline,
- parseMarkdown,
- parseMarkdownIncremental,
+ parseInlineAst,
+ parseMarkdownAst,
+ parseMarkdownAstIncremental,
createIncrementalState,
trimStreamingArtifacts,
- inlineText,
slugify,
uniqueSlug,
} from './parser';
+import type {IncrementalState, MathParseOptions, ParseOptions} from './parser';
+import {markdownAstText} from './ast';
import type {
- BlockNodeWithMath,
- InlineNodeWithMath,
- IncrementalState,
- MathParseOptions,
- ParseOptions,
-} from './parser';
+ MarkdownAstBlockContent,
+ MarkdownAstPhrasingContent,
+ MarkdownAstTable,
+} from './ast';
import {themeProps} from '../utils/themeProps';
import {useTranslator, type TranslatorFn} from '../i18n';
@@ -487,25 +486,27 @@ interface StreamingCursor {
* Count the total text characters in inline nodes without rendering.
* Used to advance the cursor past a block that will be faded as a whole unit.
*/
-function countInlineTextLength(nodes: InlineNodeWithMath[]): number {
+function countInlineTextLength(
+ nodes: ReadonlyArray<MarkdownAstPhrasingContent>,
+): number {
let len = 0;
for (const node of nodes) {
switch (node.type) {
case 'text':
- len += node.content.length;
+ len += node.value.length;
break;
- case 'code':
- len += node.content.length;
+ case 'inlineCode':
+ len += node.value.length;
break;
- case 'math':
+ case 'inlineMath':
len += node.value.length;
break;
case 'image':
len += node.alt.length;
break;
- case 'bold':
- case 'italic':
- case 'strikethrough':
+ case 'strong':
+ case 'emphasis':
+ case 'delete':
case 'link':
len += countInlineTextLength(node.children);
break;
@@ -523,7 +524,9 @@ function countInlineTextLength(nodes: InlineNodeWithMath[]): number {
/**
* Count total text characters in a block node tree.
*/
-function countBlockTextLength(nodes: BlockNodeWithMath[]): number {
+function countBlockTextLength(
+ nodes: ReadonlyArray<MarkdownAstBlockContent>,
+): number {
let len = 0;
for (const node of nodes) {
switch (node.type) {
@@ -531,8 +534,8 @@ function countBlockTextLength(nodes: BlockNodeWithMath[]): number {
case 'paragraph':
len += countInlineTextLength(node.children);
break;
- case 'codeblock':
- len += node.content.length;
+ case 'code':
+ len += node.value.length;
break;
case 'math':
len += node.value.length;
@@ -541,21 +544,18 @@ function countBlockTextLength(nodes: BlockNodeWithMath[]): number {
len += countBlockTextLength(node.children);
break;
case 'list':
- for (const item of node.items) {
+ for (const item of node.children) {
len += countBlockTextLength(item.children);
}
break;
case 'table':
- for (const h of node.headers) {
- len += countInlineTextLength(h.children);
- }
- for (const row of node.rows) {
- for (const cell of row) {
+ for (const row of node.children) {
+ for (const cell of row.children) {
len += countInlineTextLength(cell.children);
}
}
break;
- case 'hr':
+ case 'thematicBreak':
break;
case 'image':
len += node.alt.length;
@@ -768,7 +768,7 @@ function getCitationNumber(ctx: CitationContext, sourceId: string): number {
}
function renderInline(
- node: InlineNodeWithMath,
+ node: MarkdownAstPhrasingContent,
index: number,
onLinkClick: MarkdownProps['onLinkClick'] | undefined,
cursor: StreamingCursor,
@@ -780,7 +780,7 @@ function renderInline(
switch (node.type) {
case 'text': {
if (inlinePlugins && inlinePlugins.length > 0) {
- const segments = applyInlinePlugins(node.content, inlinePlugins);
+ const segments = applyInlinePlugins(node.value, inlinePlugins);
// If no plugin matched, fall through to the normal path
// O(1) guard: applyInlinePlugins returns a single text segment when
// nothing matched — skip the plugin path entirely in that case.
@@ -803,9 +803,9 @@ function renderInline(
return <Fragment key={index}>{result}</Fragment>;
}
}
- return wrapTextWithFade(node.content, cursor, index);
+ return wrapTextWithFade(node.value, cursor, index);
}
- case 'bold':
+ case 'strong':
return (
<strong key={index} {...stylex.props(styles.bold)}>
{node.children.map((c, i) =>
@@ -822,7 +822,7 @@ function renderInline(
)}
</strong>
);
- case 'italic':
+ case 'emphasis':
return (
<em key={index}>
{node.children.map((c, i) =>
@@ -839,7 +839,7 @@ function renderInline(
)}
</em>
);
- case 'strikethrough':
+ case 'delete':
return (
<del key={index} {...stylex.props(styles.strikethrough)}>
{node.children.map((c, i) =>
@@ -856,18 +856,18 @@ function renderInline(
)}
</del>
);
- case 'code': {
+ case 'inlineCode': {
// Track code content length for cursor but don't split inside code
- cursor.offset += node.content.length;
+ cursor.offset += node.value.length;
const InlineCodeComp = components?.inlineCode;
const codeEl = InlineCodeComp ? (
- <InlineCodeComp key={index}>{node.content}</InlineCodeComp>
+ <InlineCodeComp key={index}>{node.value}</InlineCodeComp>
) : (
- <Code key={index}>{node.content}</Code>
+ <Code key={index}>{node.value}</Code>
);
return codeEl;
}
- case 'math': {
+ case 'inlineMath': {
const MathComp = components?.math;
if (MathComp == null) {
return wrapTextWithFade(`$${node.value}$`, cursor, index);
@@ -876,7 +876,7 @@ function renderInline(
return <MathComp key={index} value={node.value} display="inline" />;
}
case 'link': {
- const safeHref = sanitizeUrl(node.href);
+ const safeHref = sanitizeUrl(node.url);
if (safeHref == null) {
// Unsafe URL — render as plain text
return (
@@ -954,7 +954,7 @@ function renderInline(
);
}
case 'image': {
- const safeSrc = sanitizeUrl(node.src);
+ const safeSrc = sanitizeUrl(node.url);
if (safeSrc == null) {
return <span key={index}>[{node.alt}]</span>;
}
@@ -1012,13 +1012,13 @@ function renderInline(
// ---------------------------------------------------------------------------
function getElementSpacing(
- node: BlockNodeWithMath,
+ node: MarkdownAstBlockContent,
density: 'default' | 'compact',
): StyleXStyles {
const compact = density === 'compact';
switch (node.type) {
case 'heading':
- return node.level <= 3
+ return node.depth <= 3
? compact
? styles.spacingHeadingMajorCompact
: styles.spacingHeadingMajorDefault
@@ -1029,7 +1029,7 @@ function getElementSpacing(
return compact
? styles.spacingParagraphCompact
: styles.spacingParagraphDefault;
- case 'codeblock':
+ case 'code':
case 'math':
return compact
? styles.spacingCodeblockCompact
@@ -1042,7 +1042,7 @@ function getElementSpacing(
return compact ? styles.spacingListCompact : styles.spacingListDefault;
case 'table':
return compact ? styles.spacingTableCompact : styles.spacingTableDefault;
- case 'hr':
+ case 'thematicBreak':
return compact ? styles.spacingHrCompact : styles.spacingHrDefault;
case 'image':
return compact ? styles.spacingImageCompact : styles.spacingImageDefault;
@@ -1057,15 +1057,17 @@ function getElementSpacing(
* Compute per-column min-widths from table AST content.
* Buckets: ≤6 chars → 60px, 7–15 → 80px, >15 → 120px.
*/
-function computeTableColumnMinWidths(node: {
- headers: {children: InlineNodeWithMath[]}[];
- rows: {children: InlineNodeWithMath[]}[][];
-}): number[] {
- return node.headers.map((h, colIdx) => {
- let maxLen = countInlineTextLength(h.children);
- for (const row of node.rows) {
- if (row[colIdx]) {
- const len = countInlineTextLength(row[colIdx].children);
+function computeTableColumnMinWidths(node: MarkdownAstTable): number[] {
+ const [header, ...rows] = node.children;
+ if (header == null) {
+ return [];
+ }
+ return header.children.map((cell, colIdx) => {
+ let maxLen = countInlineTextLength(cell.children);
+ for (const row of rows) {
+ const rowCell = row.children[colIdx];
+ if (rowCell != null) {
+ const len = countInlineTextLength(rowCell.children);
if (len > maxLen) {
maxLen = len;
}
@@ -1076,7 +1078,7 @@ function computeTableColumnMinWidths(node: {
}
function renderBlock(
- node: BlockNodeWithMath,
+ node: MarkdownAstBlockContent,
index: number,
blockCount: number,
density: 'default' | 'compact',
@@ -1090,7 +1092,7 @@ function renderBlock(
inlinePlugins: MarkdownInlinePlugin[] | undefined,
components: Partial<MarkdownComponents> | undefined,
t: TranslatorFn,
- headingIdMap?: ReadonlyMap<BlockNodeWithMath, string>,
+ headingIdMap?: ReadonlyMap<MarkdownAstBlockContent, string>,
): SyncReactNode {
const blockAlignMargin = BLOCK_ALIGN_MARGIN[contentAlign];
const blockAlignStyle =
@@ -1103,7 +1105,7 @@ function renderBlock(
switch (node.type) {
case 'heading': {
- const level = Math.min(node.level + headingLevelStart - 1, 6) as
+ const level = Math.min(node.depth + headingLevelStart - 1, 6) as
1 | 2 | 3 | 4 | 5 | 6;
const headingChildren = node.children.map((c, i) =>
renderInline(
@@ -1203,16 +1205,16 @@ function renderBlock(
</div>
);
}
- case 'codeblock': {
+ case 'code': {
// Track codeblock content in cursor for accurate character counting
- cursor.offset += node.content.length;
+ cursor.offset += node.value.length;
const CodeBlockComp = components?.code;
if (CodeBlockComp) {
return (
<CodeBlockComp
key={index}
- code={node.content}
- language={node.language}
+ code={node.value}
+ language={node.lang ?? 'plaintext'}
/>
);
}
@@ -1229,8 +1231,8 @@ function renderBlock(
),
)}>
<CodeBlock
- code={node.content}
- language={node.language}
+ code={node.value}
+ language={node.lang ?? 'plaintext'}
isCollapsible
xstyle={[
contentWidthValue != null
@@ -1316,11 +1318,12 @@ function renderBlock(
case 'list': {
// Detect task lists: all items have a checked state
const isTaskList =
- node.items.length > 0 && node.items.every(item => item.checked != null);
+ node.children.length > 0 &&
+ node.children.every(item => item.checked != null);
if (isTaskList) {
// Extract labels from task items — render as rich inline content
- const checkedValues = node.items
+ const checkedValues = node.children
.map((item, i) => ({item, key: `task-${i}`}))
.filter(({item}) => item.checked)
.map(({key}) => key);
@@ -1343,7 +1346,7 @@ function renderBlock(
xstyle={styles.blockIndent}
isReadOnly
density="compact">
- {node.items.map((item, i) => {
+ {node.children.map((item, i) => {
const firstChild = item.children[0];
const isInline =
item.children.length === 1 &&
@@ -1423,7 +1426,7 @@ function renderBlock(
density="compact"
start={node.ordered ? node.start : undefined}
xstyle={styles.blockIndent}>
- {node.items.map((item, i) => {
+ {node.children.map((item, i) => {
const firstChild = item.children[0];
const isInline =
item.children.length === 1 && firstChild?.type === 'paragraph';
@@ -1483,6 +1486,7 @@ function renderBlock(
}
case 'table': {
const colMinWidths = computeTableColumnMinWidths(node);
+ const [header, ...rows] = node.children;
return (
<div
@@ -1512,14 +1516,14 @@ function renderBlock(
<Table dividers="rows" textOverflow="wrap">
<TableHeader>
<TableRow>
- {node.headers.map((h, i) => (
+ {header?.children.map((h, i) => (
<TableHeaderCell
// eslint-disable-next-line @eslint-react/no-array-index-key -- markdown table columns are positional by definition
key={i}
xstyle={[
dynamicStyles.cellMinWidth(`${colMinWidths[i]}px`),
- node.alignments[i] === 'center' && cellAlignStyles.center,
- node.alignments[i] === 'right' && cellAlignStyles.end,
+ node.align[i] === 'center' && cellAlignStyles.center,
+ node.align[i] === 'right' && cellAlignStyles.end,
]}>
{h.children.map((c, j) =>
renderInline(
@@ -1538,14 +1542,14 @@ function renderBlock(
</TableRow>
</TableHeader>
<TableBody>
- {node.rows.map((row, i) => {
- const cells = row.map((cell, j) => (
+ {rows.map((row, i) => {
+ const cells = row.children.map((cell, j) => (
<TableCell
// eslint-disable-next-line @eslint-react/no-array-index-key -- markdown table cells are positional by row and column
key={j}
xstyle={[
- node.alignments[j] === 'center' && cellAlignStyles.center,
- node.alignments[j] === 'right' && cellAlignStyles.end,
+ node.align[j] === 'center' && cellAlignStyles.center,
+ node.align[j] === 'right' && cellAlignStyles.end,
]}>
{cell.children.map((c, k) =>
renderInline(
@@ -1574,7 +1578,7 @@ function renderBlock(
</div>
);
}
- case 'hr': {
+ case 'thematicBreak': {
const HrComp = components?.hr;
if (HrComp) {
return <HrComp key={index} />;
@@ -1595,7 +1599,7 @@ function renderBlock(
);
}
case 'image': {
- const safeSrc = sanitizeUrl(node.src);
+ const safeSrc = sanitizeUrl(node.url);
if (safeSrc == null) {
return (
<div
@@ -1719,21 +1723,25 @@ export function Markdown({
}
const options = hasMathRenderer ? mathParseOptions : legacyParseOptions;
const input = trimStreamingArtifacts(smoothedText, options);
- return hasMathRenderer
- ? parseMarkdownIncremental(
- input,
- incrementalStateRef.current as IncrementalState<true>,
- mathParseOptions,
- )
- : parseMarkdownIncremental(
- input,
- incrementalStateRef.current as IncrementalState<false>,
- legacyParseOptions,
- );
+ return (
+ hasMathRenderer
+ ? parseMarkdownAstIncremental(
+ input,
+ incrementalStateRef.current,
+ mathParseOptions,
+ )
+ : parseMarkdownAstIncremental(
+ input,
+ incrementalStateRef.current,
+ legacyParseOptions,
+ )
+ ).children;
}
- return hasMathRenderer
- ? parseMarkdown(children, mathParseOptions)
- : parseMarkdown(children, legacyParseOptions);
+ return (
+ hasMathRenderer
+ ? parseMarkdownAst(children, mathParseOptions)
+ : parseMarkdownAst(children, legacyParseOptions)
+ ).children;
}, [
display,
smoothedText,
@@ -1754,11 +1762,11 @@ export function Markdown({
if (display === 'inline' || blocks.length === 0) {
return undefined;
}
- const map = new Map<BlockNodeWithMath, string>();
+ const map = new Map<MarkdownAstBlockContent, string>();
const counts = new Map<string, number>();
for (const block of blocks) {
if (block.type === 'heading') {
- const label = inlineText(block.children).trim();
+ const label = markdownAstText(block.children).trim();
map.set(block, uniqueSlug(slugify(label), counts));
}
}
@@ -1774,8 +1782,8 @@ export function Markdown({
? trimStreamingArtifacts(smoothedText, options)
: children;
return hasMathRenderer
- ? parseInline(input, mathParseOptions)
- : parseInline(input, legacyParseOptions);
+ ? parseInlineAst(input, mathParseOptions)
+ : parseInlineAst(input, legacyParseOptions);
}, [
display,
smoothedText,
@@ -1800,8 +1808,10 @@ export function Markdown({
return Math.min(Math.ceil(duration / tickMs), 12);
}, [token]);
- const prevBlocksRef = useRef<BlockNodeWithMath[]>([]);
- const prevInlineNodesRef = useRef<InlineNodeWithMath[]>([]);
+ const prevBlocksRef = useRef<ReadonlyArray<MarkdownAstBlockContent>>([]);
+ const prevInlineNodesRef = useRef<ReadonlyArray<MarkdownAstPhrasingContent>>(
+ [],
+ );
const boundariesRef = useRef<number[]>([]);
const smoothedLen = smoothedText.length;
const boundaries = useMemo(() => {
diff --git a/packages/core/src/Markdown/ast.test.ts b/packages/core/src/Markdown/ast.test.ts
new file mode 100644
index 0000000000..206cf82b1a
--- /dev/null
+++ b/packages/core/src/Markdown/ast.test.ts
@@ -0,0 +1,169 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, expect, it} from 'vitest';
+import {
+ createIncrementalState,
+ parseMarkdown,
+ parseMarkdownAst,
+} from './parser';
+
+describe('canonical Markdown AST', () => {
+ it('uses MDAST-aligned names and fields for every built-in structure', () => {
+ const ast = parseMarkdownAst(
+ '# Title\n\n**bold** and *emphasis* with `code` and [link](/docs).\n\n- item\n\n| A | B |\n| - | - |\n| 1 | 2 |\n\n```ts\nconst value = 1;\n```\n\n---',
+ );
+
+ expect(ast).toMatchObject({
+ type: 'root',
+ children: [
+ {
+ type: 'heading',
+ depth: 1,
+ children: [{type: 'text', value: 'Title'}],
+ },
+ {
+ type: 'paragraph',
+ children: [
+ {type: 'strong'},
+ {type: 'text'},
+ {type: 'emphasis'},
+ {type: 'text'},
+ {type: 'inlineCode', value: 'code'},
+ {type: 'text'},
+ {type: 'link', url: '/docs'},
+ {type: 'text', value: '.'},
+ ],
+ },
+ {
+ type: 'list',
+ ordered: false,
+ children: [{type: 'listItem'}],
+ },
+ {
+ type: 'table',
+ align: [null, null],
+ children: [
+ {
+ type: 'tableRow',
+ children: [{type: 'tableCell'}, {type: 'tableCell'}],
+ },
+ {
+ type: 'tableRow',
+ children: [{type: 'tableCell'}, {type: 'tableCell'}],
+ },
+ ],
+ },
+ {type: 'code', lang: 'ts', value: 'const value = 1;'},
+ {type: 'thematicBreak'},
+ ],
+ });
+ });
+
+ it('represents remaining built-ins without legacy aliases', () => {
+ const ast = parseMarkdownAst(
+ '> quote\n\n![Block](/block.png)\n\nParagraph ![Inline](/inline.png) [src] $x$ \nnext\n\n$$\ny\n$$',
+ {math: true, sourceIds: new Set(['src'])},
+ );
+
+ expect(ast.children).toMatchObject([
+ {
+ type: 'blockquote',
+ children: [
+ {type: 'paragraph', children: [{type: 'text', value: 'quote'}]},
+ ],
+ },
+ {type: 'image', url: '/block.png', alt: 'Block'},
+ {
+ type: 'paragraph',
+ children: [
+ {type: 'text', value: 'Paragraph '},
+ {type: 'image', url: '/inline.png', alt: 'Inline'},
+ {type: 'text', value: ' '},
+ {type: 'citation', sourceId: 'src'},
+ {type: 'text', value: ' '},
+ {type: 'inlineMath', value: 'x'},
+ {type: 'break'},
+ {type: 'text', value: 'next'},
+ ],
+ },
+ {type: 'math', value: 'y'},
+ ]);
+ expect(parseMarkdownAst('```\nplain\n```').children[0]).toEqual({
+ type: 'code',
+ lang: null,
+ value: 'plain',
+ });
+ expect(parseMarkdown('```\nplain\n```')[0]).toEqual({
+ type: 'codeblock',
+ language: 'plaintext',
+ content: 'plain',
+ });
+ });
+
+ it('keeps source provenance canonical while projecting the released range', () => {
+ const source = '# Heading\n\nParagraph';
+ const ast = parseMarkdownAst(source, {sourceRanges: true});
+ const legacy = parseMarkdown(source, {sourceRanges: true});
+
+ expect(ast.children[0].position).toEqual({
+ start: {offset: 0},
+ end: {offset: 9},
+ });
+ expect(legacy[0]).toEqual({
+ type: 'heading',
+ level: 1,
+ children: [{type: 'text', content: 'Heading'}],
+ range: {start: 0, end: 9},
+ });
+ expect('position' in legacy[0]).toBe(false);
+ });
+
+ it('preserves optional legacy keys and insertion order exactly', () => {
+ const [image, list] = parseMarkdown('![Alt](/image.png)\n\n- item');
+
+ expect(Object.keys(image)).toEqual(['type', 'alt', 'src']);
+ expect(Object.keys(list)).toEqual([
+ 'type',
+ 'ordered',
+ 'start',
+ 'delimiter',
+ 'loose',
+ 'items',
+ ]);
+ if (list.type !== 'list') {
+ throw new Error('Expected list');
+ }
+ expect(Object.keys(list.items[0])).toEqual(['checked', 'children']);
+ expect(list).toStrictEqual({
+ type: 'list',
+ ordered: false,
+ start: undefined,
+ delimiter: undefined,
+ loose: undefined,
+ items: [
+ {
+ checked: undefined,
+ children: [
+ {
+ type: 'paragraph',
+ children: [{type: 'text', content: 'item'}],
+ },
+ ],
+ },
+ ],
+ });
+ });
+
+ it('does not expose canonical fields through released parser results', () => {
+ const state = createIncrementalState();
+ const blocks = parseMarkdown('# Heading\n\nParagraph');
+
+ expect(blocks[0]).toEqual({
+ type: 'heading',
+ level: 1,
+ children: [{type: 'text', content: 'Heading'}],
+ });
+ expect('depth' in blocks[0]).toBe(false);
+ expect(state.settledBlocks).toEqual([]);
+ });
+});
diff --git a/packages/core/src/Markdown/ast.ts b/packages/core/src/Markdown/ast.ts
new file mode 100644
index 0000000000..adb8f23c9c
--- /dev/null
+++ b/packages/core/src/Markdown/ast.ts
@@ -0,0 +1,231 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file ast.ts
+ * @input Parsed Markdown structure and optional Astryx extension nodes
+ * @output Canonical immutable MDAST-aligned Markdown node types and text projection
+ * @position Internal canonical Markdown tree shared by parsing, rendering, and Outline
+ */
+
+export interface MarkdownAstPoint {
+ readonly line?: number;
+ readonly column?: number;
+ readonly offset?: number;
+}
+
+export interface MarkdownAstPosition {
+ readonly start: MarkdownAstPoint;
+ readonly end: MarkdownAstPoint;
+}
+
+export interface MarkdownAstNodeBase {
+ readonly position?: MarkdownAstPosition;
+}
+
+export interface MarkdownAstExtensionNode<
+ Plugin extends string = string,
+ Name extends string = string,
+ Data = Readonly<Record<string, unknown>>,
+ Display extends 'inline' | 'block' = 'inline' | 'block',
+> extends MarkdownAstNodeBase {
+ readonly type: 'extension';
+ readonly plugin: Plugin;
+ readonly name: Name;
+ readonly data: Data;
+ readonly display: Display;
+ readonly source?: string;
+}
+
+export interface MarkdownAstText extends MarkdownAstNodeBase {
+ readonly type: 'text';
+ readonly value: string;
+}
+
+export interface MarkdownAstParent<
+ Type extends 'strong' | 'emphasis' | 'delete',
+ Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+ readonly type: Type;
+ readonly children: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>;
+}
+
+export interface MarkdownAstInlineCode extends MarkdownAstNodeBase {
+ readonly type: 'inlineCode';
+ readonly value: string;
+}
+
+export interface MarkdownAstInlineMath extends MarkdownAstNodeBase {
+ readonly type: 'inlineMath';
+ readonly value: string;
+}
+
+export interface MarkdownAstLink<
+ Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+ readonly type: 'link';
+ readonly url: string;
+ readonly children: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>;
+}
+
+export interface MarkdownAstImage extends MarkdownAstNodeBase {
+ readonly type: 'image';
+ readonly url: string;
+ readonly alt: string;
+}
+
+export interface MarkdownAstCitation extends MarkdownAstNodeBase {
+ readonly type: 'citation';
+ readonly sourceId: string;
+}
+
+export interface MarkdownAstBreak extends MarkdownAstNodeBase {
+ readonly type: 'break';
+}
+
+export type MarkdownAstPhrasingContent<
+ Extension extends MarkdownAstExtensionNode = never,
+> =
+ | MarkdownAstText
+ | MarkdownAstParent<'strong', Extension>
+ | MarkdownAstParent<'emphasis', Extension>
+ | MarkdownAstParent<'delete', Extension>
+ | MarkdownAstInlineCode
+ | MarkdownAstInlineMath
+ | MarkdownAstLink<Extension>
+ | MarkdownAstImage
+ | MarkdownAstCitation
+ | MarkdownAstBreak
+ | (Extension & {readonly display: 'inline'});
+
+export interface MarkdownAstHeading<
+ Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+ readonly type: 'heading';
+ readonly depth: 1 | 2 | 3 | 4 | 5 | 6;
+ readonly children: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>;
+}
+
+export interface MarkdownAstParagraph<
+ Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+ readonly type: 'paragraph';
+ readonly children: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>;
+}
+
+export interface MarkdownAstCode extends MarkdownAstNodeBase {
+ readonly type: 'code';
+ readonly lang: string | null;
+ readonly value: string;
+}
+
+export interface MarkdownAstMath extends MarkdownAstNodeBase {
+ readonly type: 'math';
+ readonly value: string;
+}
+
+export interface MarkdownAstBlockquote<
+ Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+ readonly type: 'blockquote';
+ readonly children: ReadonlyArray<MarkdownAstBlockContent<Extension>>;
+}
+
+export interface MarkdownAstListItem<
+ Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+ readonly type: 'listItem';
+ readonly checked?: boolean;
+ readonly children: ReadonlyArray<MarkdownAstBlockContent<Extension>>;
+}
+
+export interface MarkdownAstList<
+ Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+ readonly type: 'list';
+ readonly ordered: boolean;
+ readonly start?: number;
+ readonly spread?: boolean;
+ /** Astryx-preserved ordered-list marker delimiter. */
+ readonly delimiter?: '.' | ')';
+ readonly children: ReadonlyArray<MarkdownAstListItem<Extension>>;
+}
+
+export type MarkdownAstTableAlignment = 'left' | 'center' | 'right' | null;
+
+export interface MarkdownAstTableCell<
+ Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+ readonly type: 'tableCell';
+ readonly children: ReadonlyArray<MarkdownAstPhrasingContent<Extension>>;
+}
+
+export interface MarkdownAstTableRow<
+ Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+ readonly type: 'tableRow';
+ readonly children: ReadonlyArray<MarkdownAstTableCell<Extension>>;
+}
+
+export interface MarkdownAstTable<
+ Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+ readonly type: 'table';
+ readonly align: ReadonlyArray<MarkdownAstTableAlignment>;
+ readonly children: ReadonlyArray<MarkdownAstTableRow<Extension>>;
+}
+
+export interface MarkdownAstThematicBreak extends MarkdownAstNodeBase {
+ readonly type: 'thematicBreak';
+}
+
+export type MarkdownAstBlockContent<
+ Extension extends MarkdownAstExtensionNode = never,
+> =
+ | MarkdownAstHeading<Extension>
+ | MarkdownAstParagraph<Extension>
+ | MarkdownAstCode
+ | MarkdownAstMath
+ | MarkdownAstBlockquote<Extension>
+ | MarkdownAstList<Extension>
+ | MarkdownAstTable<Extension>
+ | MarkdownAstThematicBreak
+ | MarkdownAstImage
+ | (Extension & {readonly display: 'block'});
+
+export interface MarkdownAstRoot<
+ Extension extends MarkdownAstExtensionNode = never,
+> extends MarkdownAstNodeBase {
+ readonly type: 'root';
+ readonly children: ReadonlyArray<MarkdownAstBlockContent<Extension>>;
+}
+
+export function markdownAstText(
+ nodes: ReadonlyArray<MarkdownAstPhrasingContent>,
+): string {
+ let text = '';
+ for (const node of nodes) {
+ switch (node.type) {
+ case 'text':
+ case 'inlineCode':
+ case 'inlineMath':
+ text += node.value;
+ break;
+ case 'strong':
+ case 'emphasis':
+ case 'delete':
+ case 'link':
+ text += markdownAstText(node.children);
+ break;
+ case 'image':
+ text += node.alt;
+ break;
+ case 'citation':
+ case 'break':
+ break;
+ default: {
+ node satisfies never;
+ }
+ }
+ }
+ return text;
+}
diff --git a/packages/core/src/Markdown/incremental.test.ts b/packages/core/src/Markdown/incremental.test.ts
index 18b2df4578..77e2eb1bef 100644
--- a/packages/core/src/Markdown/incremental.test.ts
+++ b/packages/core/src/Markdown/incremental.test.ts
@@ -47,6 +47,18 @@ describe('parseMarkdownIncremental', () => {
}
});
+ it('keeps projected settled-list identity when a following block settles too', () => {
+ const state = createIncrementalState();
+ parseMarkdownIncremental('1. a\n\nTail', state);
+ const result = parseMarkdownIncremental(
+ '1. a\n\n1. b\n\nAfter\n\nTail',
+ state,
+ );
+
+ expect(state.settledBlocks).toEqual(parseMarkdown(state.settledText));
+ expect(result[0]).toBe(state.settledBlocks[0]);
+ });
+
it('joins loose ) ordered list across streamed chunks', () => {
const text = '1) apple\n\n1) banana\n\n1) cherry\n';
const {final} = simulateStreaming(text, 5);
diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.ts
index 6dee3b2eb9..af2e82e7a4 100644
--- a/packages/core/src/Markdown/parser.ts
+++ b/packages/core/src/Markdown/parser.ts
@@ -2,14 +2,22 @@
/**
* @file parser.ts
- * @input Markdown string
- * @output Array of MarkdownNode AST nodes, including opt-in math; heading slug
- * helpers
- * (inlineText, slugify, uniqueSlug) shared by Markdown rendering and
- * Outline's parseOutlineFromMarkdown
- * @position Core parser; consumed by Markdown.tsx and Outline
+ * @input Markdown string and released parse options
+ * @output Canonical MDAST-aligned nodes for internal consumers plus unchanged
+ * released parser-node projections; shared heading slug helpers
+ * @position Core parser and compatibility boundary; consumed by Markdown and Outline
*/
+import type {
+ MarkdownAstBlockContent,
+ MarkdownAstListItem,
+ MarkdownAstPhrasingContent,
+ MarkdownAstPosition,
+ MarkdownAstRoot,
+ MarkdownAstTableCell,
+ MarkdownAstTableRow,
+} from './ast';
+
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
@@ -130,6 +138,199 @@ export type TableCellNode = {children: InlineNode[]};
type TableCellNodeWithMath = {children: InlineNodeWithMath[]};
export type TableAlignment = 'left' | 'center' | 'right' | null;
+type LegacyProjectionCache = WeakMap<object, object>;
+
+function projectRange(
+ position: MarkdownAstPosition | undefined,
+): BlockMetadata {
+ const start = position?.start.offset;
+ const end = position?.end.offset;
+ return start == null || end == null ? {} : {range: {start, end}};
+}
+
+function projectInlineNode(
+ node: MarkdownAstPhrasingContent,
+ cache: LegacyProjectionCache,
+): InlineNodeWithMath {
+ const cached = cache.get(node);
+ if (cached != null) {
+ return cached as InlineNodeWithMath;
+ }
+ let projected: InlineNodeWithMath;
+ switch (node.type) {
+ case 'text':
+ projected = {type: 'text', content: node.value};
+ break;
+ case 'strong':
+ projected = {
+ type: 'bold',
+ children: node.children.map(child => projectInlineNode(child, cache)),
+ };
+ break;
+ case 'emphasis':
+ projected = {
+ type: 'italic',
+ children: node.children.map(child => projectInlineNode(child, cache)),
+ };
+ break;
+ case 'delete':
+ projected = {
+ type: 'strikethrough',
+ children: node.children.map(child => projectInlineNode(child, cache)),
+ };
+ break;
+ case 'inlineCode':
+ projected = {type: 'code', content: node.value};
+ break;
+ case 'inlineMath':
+ projected = {type: 'math', value: node.value};
+ break;
+ case 'link':
+ projected = {
+ type: 'link',
+ href: node.url,
+ children: node.children.map(child => projectInlineNode(child, cache)),
+ };
+ break;
+ case 'image':
+ projected = {type: 'image', src: node.url, alt: node.alt};
+ break;
+ case 'citation':
+ projected = {type: 'citation', sourceId: node.sourceId};
+ break;
+ case 'break':
+ projected = {type: 'break'};
+ break;
+ }
+ cache.set(node, projected);
+ return projected;
+}
+
+function projectTableCell(
+ node: MarkdownAstTableCell,
+ cache: LegacyProjectionCache,
+): TableCellNodeWithMath {
+ const cached = cache.get(node);
+ if (cached != null) {
+ return cached as TableCellNodeWithMath;
+ }
+ const projected = {
+ children: node.children.map(child => projectInlineNode(child, cache)),
+ };
+ cache.set(node, projected);
+ return projected;
+}
+
+function projectListItem(
+ node: MarkdownAstListItem,
+ cache: LegacyProjectionCache,
+): ListItemNodeWithMath {
+ const cached = cache.get(node);
+ if (cached != null) {
+ return cached as ListItemNodeWithMath;
+ }
+ const projected = {
+ checked: node.checked,
+ children: node.children.map(child => projectBlockNode(child, cache)),
+ };
+ cache.set(node, projected);
+ return projected;
+}
+
+function projectBlockNode(
+ node: MarkdownAstBlockContent,
+ cache: LegacyProjectionCache,
+): BlockNodeWithMath {
+ const cached = cache.get(node);
+ if (cached != null) {
+ return cached as BlockNodeWithMath;
+ }
+ const metadata = projectRange(node.position);
+ let projected: BlockNodeWithMath;
+ switch (node.type) {
+ case 'heading':
+ projected = {
+ type: 'heading',
+ level: node.depth,
+ children: node.children.map(child => projectInlineNode(child, cache)),
+ ...metadata,
+ };
+ break;
+ case 'paragraph':
+ projected = {
+ type: 'paragraph',
+ children: node.children.map(child => projectInlineNode(child, cache)),
+ ...metadata,
+ };
+ break;
+ case 'code':
+ projected = {
+ type: 'codeblock',
+ language: node.lang ?? 'plaintext',
+ content: node.value,
+ ...metadata,
+ };
+ break;
+ case 'math':
+ projected = {type: 'math', value: node.value, ...metadata};
+ break;
+ case 'blockquote':
+ projected = {
+ type: 'blockquote',
+ children: node.children.map(child => projectBlockNode(child, cache)),
+ ...metadata,
+ };
+ break;
+ case 'list':
+ projected = {
+ type: 'list',
+ ordered: node.ordered,
+ start: node.start,
+ delimiter: node.delimiter,
+ loose: node.spread,
+ items: node.children.map(item => projectListItem(item, cache)),
+ ...metadata,
+ };
+ break;
+ case 'table': {
+ const [header = {type: 'tableRow' as const, children: []}, ...rows] =
+ node.children;
+ projected = {
+ type: 'table',
+ headers: header.children.map(cell => projectTableCell(cell, cache)),
+ alignments: [...node.align],
+ rows: rows.map(row =>
+ row.children.map(cell => projectTableCell(cell, cache)),
+ ),
+ ...metadata,
+ };
+ break;
+ }
+ case 'thematicBreak':
+ projected = {type: 'hr', ...metadata};
+ break;
+ case 'image':
+ projected = {type: 'image', alt: node.alt, src: node.url, ...metadata};
+ break;
+ }
+ cache.set(node, projected);
+ return projected;
+}
+
+function projectInlineNodes(
+ nodes: ReadonlyArray<MarkdownAstPhrasingContent>,
+ cache: LegacyProjectionCache = new WeakMap(),
+): InlineNodeWithMath[] {
+ return nodes.map(node => projectInlineNode(node, cache));
+}
+
+function projectMarkdownRoot(
+ root: MarkdownAstRoot,
+ cache: LegacyProjectionCache = new WeakMap(),
+): BlockNodeWithMath[] {
+ return root.children.map(node => projectBlockNode(node, cache));
+}
+
// ---------------------------------------------------------------------------
// Parse options
// ---------------------------------------------------------------------------
@@ -201,7 +402,10 @@ type ResolvedOptions = {
readonly linkDefs?: ReadonlyMap<string, string>;
};
-const EMPTY_OPTS: ResolvedOptions = {sourceIds: undefined, autolink: undefined};
+const EMPTY_OPTS: ResolvedOptions = {
+ sourceIds: undefined,
+ autolink: undefined,
+};
function resolveOptions(
arg: ReadonlySet<string> | RuntimeParseOptions | undefined,
@@ -542,7 +746,7 @@ function matchReferenceLink(
start: number,
linkDefs: ReadonlyMap<string, string>,
opts: ResolvedOptions,
-): {node: InlineNodeWithMath; end: number} | null {
+): {node: MarkdownAstPhrasingContent; end: number} | null {
const textClose = text.indexOf(']', start + 1);
if (textClose === -1) {
return null;
@@ -560,7 +764,11 @@ function matchReferenceLink(
const href = linkDefs.get(normalizeLinkLabel(label));
if (href != null && isSafeUrl(href)) {
return {
- node: {type: 'link', href, children: parseInlineImpl(linkText, opts)},
+ node: {
+ type: 'link',
+ url: href,
+ children: parseInlineImpl(linkText, opts),
+ },
end: labelClose + 1,
};
}
@@ -577,7 +785,7 @@ function matchReferenceLink(
return null;
}
return {
- node: {type: 'link', href, children: parseInlineImpl(linkText, opts)},
+ node: {type: 'link', url: href, children: parseInlineImpl(linkText, opts)},
end: textClose + 1,
};
}
@@ -587,7 +795,7 @@ function matchReferenceImage(
text: string,
start: number,
linkDefs: ReadonlyMap<string, string>,
-): {node: InlineNodeWithMath; end: number} | null {
+): {node: MarkdownAstPhrasingContent; end: number} | null {
const altClose = text.indexOf(']', start + 2);
if (altClose === -1) {
return null;
@@ -600,7 +808,7 @@ function matchReferenceImage(
const label = rawLabel === '' ? alt : rawLabel;
const src = linkDefs.get(normalizeLinkLabel(label));
if (src != null && isSafeUrl(src)) {
- return {node: {type: 'image', src, alt}, end: labelClose + 1};
+ return {node: {type: 'image', url: src, alt}, end: labelClose + 1};
}
// No match — fall back to a shortcut `![alt]`.
}
@@ -612,7 +820,7 @@ function matchReferenceImage(
if (src == null || !isSafeUrl(src)) {
return null;
}
- return {node: {type: 'image', src, alt}, end: altClose + 1};
+ return {node: {type: 'image', url: src, alt}, end: altClose + 1};
}
// ---------------------------------------------------------------------------
@@ -791,6 +999,14 @@ export function parseInline(
text: string,
arg?: ReadonlySet<string> | RuntimeParseOptions,
): InlineNodeWithMath[] {
+ return projectInlineNodes(parseInlineAst(text, arg));
+}
+
+/** @internal Canonical inline parse used by Markdown rendering. */
+export function parseInlineAst(
+ text: string,
+ arg?: ReadonlySet<string> | RuntimeParseOptions,
+): MarkdownAstPhrasingContent[] {
return parseInlineEntry(text, resolveOptions(arg));
}
@@ -806,7 +1022,7 @@ export function parseInline(
function parseInlineEntry(
text: string,
opts: ResolvedOptions,
-): InlineNodeWithMath[] {
+): MarkdownAstPhrasingContent[] {
const nodes = parseInlineImpl(text, opts);
return opts.autolink === 'gfm' ? transformAutolinks(nodes) : nodes;
}
@@ -814,14 +1030,14 @@ function parseInlineEntry(
function parseInlineImpl(
text: string,
opts: ResolvedOptions,
-): InlineNodeWithMath[] {
- const nodes: InlineNodeWithMath[] = [];
+): MarkdownAstPhrasingContent[] {
+ const nodes: MarkdownAstPhrasingContent[] = [];
let i = 0;
while (i < text.length) {
// --- Escape ---
if (text[i] === '\\' && i + 1 < text.length) {
- nodes.push({type: 'text', content: text[i + 1]});
+ nodes.push({type: 'text', value: text[i + 1]});
i += 2;
continue;
}
@@ -832,7 +1048,10 @@ function parseInlineImpl(
const openIndex = i + tickCount;
const closeIndex = text.indexOf('`'.repeat(tickCount), openIndex);
if (closeIndex !== -1) {
- nodes.push({type: 'code', content: text.slice(openIndex, closeIndex)});
+ nodes.push({
+ type: 'inlineCode',
+ value: text.slice(openIndex, closeIndex),
+ });
i = closeIndex + tickCount;
continue;
}
@@ -842,7 +1061,10 @@ function parseInlineImpl(
if (opts.math && text[i] === '$') {
const closeIndex = findInlineMathEnd(text, i);
if (closeIndex !== -1) {
- nodes.push({type: 'math', value: text.slice(i + 1, closeIndex)});
+ nodes.push({
+ type: 'inlineMath',
+ value: text.slice(i + 1, closeIndex),
+ });
i = closeIndex + 1;
continue;
}
@@ -867,11 +1089,11 @@ function parseInlineImpl(
const src = text.slice(altClose + 2, srcClose);
if (!isSafeUrl(src)) {
// Dangerous scheme — emit as plain text.
- nodes.push({type: 'text', content: text.slice(i, srcClose + 1)});
+ nodes.push({type: 'text', value: text.slice(i, srcClose + 1)});
} else {
nodes.push({
type: 'image',
- src,
+ url: src,
alt: text.slice(i + 2, altClose),
});
}
@@ -910,11 +1132,11 @@ function parseInlineImpl(
const href = text.slice(textClose + 2, urlClose);
if (!isSafeUrl(href)) {
// Dangerous scheme — emit as plain text instead of a link.
- nodes.push({type: 'text', content: text.slice(i, urlClose + 1)});
+ nodes.push({type: 'text', value: text.slice(i, urlClose + 1)});
} else {
nodes.push({
type: 'link',
- href,
+ url: href,
children: parseInlineImpl(text.slice(i + 1, textClose), opts),
});
}
@@ -950,10 +1172,10 @@ function parseInlineImpl(
(!isUnderscore || !isWordChar(text[closeIndex + 3]))
) {
nodes.push({
- type: 'bold',
+ type: 'strong',
children: [
{
- type: 'italic',
+ type: 'emphasis',
children: parseInlineImpl(text.slice(i + 3, closeIndex), opts),
},
],
@@ -980,7 +1202,7 @@ function parseInlineImpl(
(!isUnderscore || !isWordChar(text[closeIndex + 2]))
) {
nodes.push({
- type: 'bold',
+ type: 'strong',
children: parseInlineImpl(text.slice(i + 2, closeIndex), opts),
});
i = closeIndex + 2;
@@ -994,7 +1216,7 @@ function parseInlineImpl(
const closeIndex = text.indexOf('~~', i + 2);
if (closeIndex !== -1) {
nodes.push({
- type: 'strikethrough',
+ type: 'delete',
children: parseInlineImpl(text.slice(i + 2, closeIndex), opts),
});
i = closeIndex + 2;
@@ -1015,7 +1237,7 @@ function parseInlineImpl(
(!isUnderscore || !isWordChar(text[closeIndex + 1]))
) {
nodes.push({
- type: 'italic',
+ type: 'emphasis',
children: parseInlineImpl(text.slice(i + 1, closeIndex), opts),
});
i = closeIndex + 1;
@@ -1043,9 +1265,12 @@ function parseInlineImpl(
if (trimmed.length > 0) {
const last = nodes[nodes.length - 1];
if (last?.type === 'text') {
- last.content += trimmed;
+ nodes[nodes.length - 1] = {
+ ...last,
+ value: last.value + trimmed,
+ };
} else {
- nodes.push({type: 'text', content: trimmed});
+ nodes.push({type: 'text', value: trimmed});
}
}
nodes.push({type: 'break'});
@@ -1056,9 +1281,9 @@ function parseInlineImpl(
const last = nodes[nodes.length - 1];
if (last?.type === 'text') {
- last.content += content;
+ nodes[nodes.length - 1] = {...last, value: last.value + content};
} else {
- nodes.push({type: 'text', content});
+ nodes.push({type: 'text', value: content});
}
i = end;
}
@@ -1303,26 +1528,26 @@ function scanAutolinksInText(text: string): AutolinkMatch[] {
* Split a text-node `content` string into a sequence of text + link nodes
* based on autolink matches.
*/
-function splitTextOnAutolinks(content: string): InlineNodeWithMath[] {
+function splitTextOnAutolinks(content: string): MarkdownAstPhrasingContent[] {
const matches = scanAutolinksInText(content);
if (matches.length === 0) {
- return [{type: 'text', content}];
+ return [{type: 'text', value: content}];
}
- const out: InlineNodeWithMath[] = [];
+ const out: MarkdownAstPhrasingContent[] = [];
let cursor = 0;
for (const m of matches) {
if (m.start > cursor) {
- out.push({type: 'text', content: content.slice(cursor, m.start)});
+ out.push({type: 'text', value: content.slice(cursor, m.start)});
}
out.push({
type: 'link',
- href: m.href,
- children: [{type: 'text', content: m.display}],
+ url: m.href,
+ children: [{type: 'text', value: m.display}],
});
cursor = m.end;
}
if (cursor < content.length) {
- out.push({type: 'text', content: content.slice(cursor)});
+ out.push({type: 'text', value: content.slice(cursor)});
}
return out;
}
@@ -1335,23 +1560,25 @@ function splitTextOnAutolinks(content: string): InlineNodeWithMath[] {
* `image` alt text, `citation`, or `break`. Runs only on the outermost
* block's inline tree (see `parseInlineEntry`).
*/
-function transformAutolinks(nodes: InlineNodeWithMath[]): InlineNodeWithMath[] {
- const out: InlineNodeWithMath[] = [];
+function transformAutolinks(
+ nodes: ReadonlyArray<MarkdownAstPhrasingContent>,
+): MarkdownAstPhrasingContent[] {
+ const out: MarkdownAstPhrasingContent[] = [];
for (const node of nodes) {
if (node.type === 'text') {
- const split = splitTextOnAutolinks(node.content);
+ const split = splitTextOnAutolinks(node.value);
for (const seg of split) {
const last = out[out.length - 1];
if (seg.type === 'text' && last?.type === 'text') {
- last.content += seg.content;
+ out[out.length - 1] = {...last, value: last.value + seg.value};
} else {
out.push(seg);
}
}
} else if (
- node.type === 'bold' ||
- node.type === 'italic' ||
- node.type === 'strikethrough'
+ node.type === 'strong' ||
+ node.type === 'emphasis' ||
+ node.type === 'delete'
) {
out.push({...node, children: transformAutolinks(node.children)});
} else {
@@ -1510,9 +1737,9 @@ function parseTable(
lines: string[],
lineIndex: number,
opts: ResolvedOptions,
-): {node: BlockNodeWithMath; nextIndex: number} {
- const headers: TableCellNodeWithMath[] = splitTableRow(lines[lineIndex]).map(
- cell => ({children: parseInlineEntry(cell, opts)}),
+): {node: MarkdownAstBlockContent; nextIndex: number} {
+ const headers: MarkdownAstTableCell[] = splitTableRow(lines[lineIndex]).map(
+ cell => ({type: 'tableCell', children: parseInlineEntry(cell, opts)}),
);
const alignments: TableAlignment[] = splitTableRow(lines[lineIndex + 1]).map(
cell => {
@@ -1528,22 +1755,28 @@ function parseTable(
: null;
},
);
- const rows: TableCellNodeWithMath[][] = [];
+ const rows: MarkdownAstTableRow[] = [];
let rowIndex = lineIndex + 2;
while (
rowIndex < lines.length &&
lines[rowIndex].includes('|') &&
lines[rowIndex].trim() !== ''
) {
- rows.push(
- splitTableRow(lines[rowIndex]).map(cell => ({
+ rows.push({
+ type: 'tableRow',
+ children: splitTableRow(lines[rowIndex]).map(cell => ({
+ type: 'tableCell',
children: parseInlineEntry(cell, opts),
})),
- );
+ });
rowIndex++;
}
return {
- node: {type: 'table', headers, alignments, rows},
+ node: {
+ type: 'table',
+ align: alignments,
+ children: [{type: 'tableRow', children: headers}, ...rows],
+ },
nextIndex: rowIndex,
};
}
@@ -1553,8 +1786,8 @@ function parseList(
startIndex: number,
ordered: boolean,
opts: ResolvedOptions,
-): {node: BlockNodeWithMath; nextIndex: number} {
- const items: ListItemNodeWithMath[] = [];
+): {node: MarkdownAstBlockContent; nextIndex: number} {
+ const items: MarkdownAstListItem[] = [];
const baseIndent = getIndent(lines[startIndex]);
// Ordered lists may use either '.' or ')' as the marker delimiter
// (CommonMark 5.2). Capture which one this list starts with so its items
@@ -1608,7 +1841,11 @@ function parseList(
itemText += '\n' + deindented.join('\n');
}
- items.push({checked, children: parseMarkdownImpl(itemText, nested(opts))});
+ items.push({
+ type: 'listItem',
+ checked,
+ children: parseMarkdownImpl(itemText, nested(opts)),
+ });
// CommonMark loose list: blank line(s) between items of the same style
// and indent still form one list. Skip the blanks and continue if the
@@ -1632,8 +1869,8 @@ function parseList(
ordered,
start,
delimiter: ordered ? (delim as '.' | ')') : undefined,
- loose: loose || undefined,
- items,
+ spread: loose || undefined,
+ children: items,
},
nextIndex: index,
};
@@ -1659,13 +1896,24 @@ export function parseMarkdown(
input: string,
arg?: ReadonlySet<string> | RuntimeParseOptions,
): BlockNodeWithMath[] {
- return parseMarkdownImpl(input, resolveOptions(arg));
+ return projectMarkdownRoot(parseMarkdownAst(input, arg));
+}
+
+/** @internal Canonical block parse used by Markdown and Outline rendering. */
+export function parseMarkdownAst(
+ input: string,
+ arg?: ReadonlySet<string> | RuntimeParseOptions,
+): MarkdownAstRoot {
+ return {
+ type: 'root',
+ children: parseMarkdownImpl(input, resolveOptions(arg)),
+ };
}
function parseMarkdownImpl(
input: string,
baseOpts: ResolvedOptions,
-): BlockNodeWithMath[] {
+): MarkdownAstBlockContent[] {
// Collect this input's link reference definitions and strip their lines,
// then merge them with any definitions inherited from an enclosing parse
// (the incremental parser passes the whole document's definitions in; a
@@ -1686,7 +1934,7 @@ function parseMarkdownImpl(
const opts: ResolvedOptions =
linkDefs != null ? {...baseOpts, linkDefs} : baseOpts;
const lines = cleaned.split('\n');
- const blocks: BlockNodeWithMath[] = [];
+ const blocks: MarkdownAstBlockContent[] = [];
// The line each block started on, parallel to `blocks`. Only collected when
// ranges were asked for; a block's end is resolved after the loop, since the
// branch that produced it has already moved `index` past whatever it read.
@@ -1697,7 +1945,7 @@ function parseMarkdownImpl(
? []
: null;
let blockStartLine = 0;
- const pushBlock = (node: BlockNodeWithMath, endLine?: number) => {
+ const pushBlock = (node: MarkdownAstBlockContent, endLine?: number) => {
blocks.push(node);
blockStartLines?.push(blockStartLine);
blockEndLines?.push(endLine);
@@ -1716,7 +1964,7 @@ function parseMarkdownImpl(
const fenceMatch = line.match(/^(`{3,}|~{3,})(\w*)/);
if (fenceMatch) {
const fence = fenceMatch[1];
- const language = fenceMatch[2] || 'plaintext';
+ const language = fenceMatch[2] || null;
const codeLines: string[] = [];
index++;
while (index < lines.length && !lines[index].startsWith(fence)) {
@@ -1728,7 +1976,7 @@ function parseMarkdownImpl(
// can end on them, so it states its own end rather than letting the
// positional derivation trim them off.
pushBlock(
- {type: 'codeblock', language, content: codeLines.join('\n')},
+ {type: 'code', lang: language, value: codeLines.join('\n')},
Math.min(index, lines.length) - 1,
);
continue;
@@ -1752,7 +2000,7 @@ function parseMarkdownImpl(
if (headingMatch) {
pushBlock({
type: 'heading',
- level: headingMatch[1].length as 1 | 2 | 3 | 4 | 5 | 6,
+ depth: headingMatch[1].length as 1 | 2 | 3 | 4 | 5 | 6,
children: parseInlineEntry(headingMatch[2], opts),
});
index++;
@@ -1761,7 +2009,7 @@ function parseMarkdownImpl(
// --- HR (must precede list check to handle `- - -`, `* * *`, `_ _ _`) ---
if (isHorizontalRule(line)) {
- pushBlock({type: 'hr'});
+ pushBlock({type: 'thematicBreak'});
index++;
continue;
}
@@ -1775,7 +2023,7 @@ function parseMarkdownImpl(
line.trim() === imageMatch[0] &&
isSafeUrl(imageMatch[2])
) {
- pushBlock({type: 'image', alt: imageMatch[1], src: imageMatch[2]});
+ pushBlock({type: 'image', alt: imageMatch[1], url: imageMatch[2]});
index++;
continue;
}
@@ -1866,7 +2114,7 @@ function parseMarkdownImpl(
* `lineMap` says which input line each surviving line came from.
*/
function stampSourceRanges(
- blocks: BlockNodeWithMath[],
+ blocks: MarkdownAstBlockContent[],
blockStartLines: number[],
blockEndLines: (number | undefined)[],
lines: string[],
@@ -1903,7 +2151,13 @@ function stampSourceRanges(
// range that dropped it would slice to something that re-parses
// differently.
const end = lineStart(endLine) + lines[endLine].length;
- blocks[i] = {...blocks[i], range: {start: lineStart(startLine), end}};
+ blocks[i] = {
+ ...blocks[i],
+ position: {
+ start: {offset: lineStart(startLine)},
+ end: {offset: end},
+ },
+ };
}
}
@@ -1968,6 +2222,13 @@ type IncrementalCache = {
/** The effective document-global definitions used by slice parses. */
linkDefs: ReadonlyMap<string, string>;
linkDefsKey: string;
+ /** Canonical settled blocks shared by rendering and compatibility projection. */
+ settledAstBlocks: MarkdownAstBlockContent[];
+ settledRevision: number;
+ projectedRevision: number;
+ projectedSettledBlocks: BlockNodeWithMath[];
+ /** Preserves released settled-node identity across projected snapshots. */
+ projectionCache: LegacyProjectionCache;
work: IncrementalWork;
};
@@ -1986,6 +2247,11 @@ function makeIncrementalCache(
tailLinkDefs: new Map(),
linkDefs: defs,
linkDefsKey: linkDefsSignature(defs),
+ settledAstBlocks: [],
+ settledRevision: 0,
+ projectedRevision: -1,
+ projectedSettledBlocks: [],
+ projectionCache: new WeakMap(),
work: {
splitCharacters: 0,
boundaryLines: 0,
@@ -2481,9 +2747,9 @@ function atOffset(opts: ResolvedOptions, offset: number): ResolvedOptions {
* lists even though the full-text parser joins them per CommonMark §5.3.
*/
function mergeSettledBlocks(
- prev: BlockNodeWithMath[],
- delta: BlockNodeWithMath[],
-): BlockNodeWithMath[] {
+ prev: MarkdownAstBlockContent[],
+ delta: MarkdownAstBlockContent[],
+): MarkdownAstBlockContent[] {
if (prev.length === 0 || delta.length === 0) {
return [...prev, ...delta];
}
@@ -2495,17 +2761,21 @@ function mergeSettledBlocks(
prevLast.ordered === deltaFirst.ordered &&
prevLast.delimiter === deltaFirst.delimiter
) {
- const merged: BlockNodeWithMath = {
+ const merged: MarkdownAstBlockContent = {
type: 'list',
ordered: prevLast.ordered,
start: prevLast.start,
delimiter: prevLast.delimiter,
- loose: true,
- items: [...prevLast.items, ...deltaFirst.items],
- // One list now, so one range: from where the first half started to
- // where the second half ended.
- ...(prevLast.range != null && deltaFirst.range != null
- ? {range: {start: prevLast.range.start, end: deltaFirst.range.end}}
+ spread: true,
+ children: [...prevLast.children, ...deltaFirst.children],
+ // One list now, so one position spans both halves.
+ ...(prevLast.position != null && deltaFirst.position != null
+ ? {
+ position: {
+ start: prevLast.position.start,
+ end: deltaFirst.position.end,
+ },
+ }
: null),
};
return [...prev.slice(0, -1), merged, ...delta.slice(1)];
@@ -2515,15 +2785,15 @@ function mergeSettledBlocks(
/** Append a newly-settled slice without copying the already-settled prefix. */
function appendSettledBlocks(
- prev: BlockNodeWithMath[],
- delta: BlockNodeWithMath[],
-): void {
+ prev: MarkdownAstBlockContent[],
+ delta: MarkdownAstBlockContent[],
+): boolean {
if (delta.length === 0) {
- return;
+ return false;
}
if (prev.length === 0) {
prev.push(...delta);
- return;
+ return false;
}
const prevLast = prev[prev.length - 1];
const deltaFirst = delta[0];
@@ -2538,16 +2808,22 @@ function appendSettledBlocks(
ordered: prevLast.ordered,
start: prevLast.start,
delimiter: prevLast.delimiter,
- loose: true,
- items: [...prevLast.items, ...deltaFirst.items],
- ...(prevLast.range != null && deltaFirst.range != null
- ? {range: {start: prevLast.range.start, end: deltaFirst.range.end}}
+ spread: true,
+ children: [...prevLast.children, ...deltaFirst.children],
+ ...(prevLast.position != null && deltaFirst.position != null
+ ? {
+ position: {
+ start: prevLast.position.start,
+ end: deltaFirst.position.end,
+ },
+ }
: null),
};
prev.push(...delta.slice(1));
- return;
+ return true;
}
prev.push(...delta);
+ return false;
}
function sameUnsettledDefinitions(
@@ -2583,6 +2859,11 @@ function resetIncrementalCache(
cache.tailLinkDefs = new Map();
cache.linkDefs = new Map();
cache.linkDefsKey = '';
+ cache.settledAstBlocks = [];
+ cache.settledRevision++;
+ cache.projectedRevision = -1;
+ cache.projectedSettledBlocks = [];
+ cache.projectionCache = new WeakMap();
cache.work = {
splitCharacters: 0,
boundaryLines: 0,
@@ -2622,6 +2903,36 @@ export function parseMarkdownIncremental(
state: IncrementalState<boolean>,
arg?: ReadonlySet<string> | RuntimeParseOptions,
): BlockNodeWithMath[] {
+ const root = parseMarkdownAstIncremental(input, state, arg);
+ const cache = incrementalCaches.get(state) ?? makeIncrementalCache(state);
+ const projected = projectMarkdownRoot(root, cache.projectionCache);
+ if (cache.projectedRevision !== cache.settledRevision) {
+ cache.projectedSettledBlocks = cache.settledAstBlocks.map(block =>
+ projectBlockNode(block, cache.projectionCache),
+ );
+ cache.projectedRevision = cache.settledRevision;
+ }
+ state.settledBlocks = cache.projectedSettledBlocks;
+ return projected;
+}
+
+/** @internal Canonical incremental parse used by Markdown rendering. */
+export function parseMarkdownAstIncremental(
+ input: string,
+ state: IncrementalState<boolean>,
+ arg?: ReadonlySet<string> | RuntimeParseOptions,
+): MarkdownAstRoot {
+ return {
+ type: 'root',
+ children: parseMarkdownIncrementalAstBlocks(input, state, arg),
+ };
+}
+
+function parseMarkdownIncrementalAstBlocks(
+ input: string,
+ state: IncrementalState<boolean>,
+ arg?: ReadonlySet<string> | RuntimeParseOptions,
+): MarkdownAstBlockContent[] {
const opts = resolveOptions(arg);
const cache = incrementalCaches.get(state) ?? makeIncrementalCache(state);
let reparseSettled = false;
@@ -2746,17 +3057,39 @@ export function parseMarkdownIncremental(
let parsedSettledBlocks = 0;
if (reparseSettled) {
- state.settledBlocks = state.settledText
+ cache.settledAstBlocks = state.settledText
? parseMarkdownImpl(state.settledText, parseOpts)
: [];
- parsedSettledBlocks = state.settledBlocks.length;
+ parsedSettledBlocks = cache.settledAstBlocks.length;
+ cache.settledRevision++;
+ cache.projectedSettledBlocks = cache.settledAstBlocks.map(block =>
+ projectBlockNode(block, cache.projectionCache),
+ );
+ cache.projectedRevision = cache.settledRevision;
} else if (settledDelta !== '') {
const deltaBlocks = parseMarkdownImpl(
settledDelta,
atOffset(parseOpts, oldSettledEnd),
);
- appendSettledBlocks(state.settledBlocks, deltaBlocks);
+ const mergeIndex = cache.settledAstBlocks.length - 1;
+ const mergedList = appendSettledBlocks(cache.settledAstBlocks, deltaBlocks);
parsedSettledBlocks = deltaBlocks.length;
+ cache.settledRevision++;
+ const projectedDelta = deltaBlocks.map(block =>
+ projectBlockNode(block, cache.projectionCache),
+ );
+ if (mergedList) {
+ const canonicalMerged = cache.settledAstBlocks[mergeIndex];
+ const projectedMerged = projectBlockNode(
+ canonicalMerged,
+ cache.projectionCache,
+ );
+ cache.projectedSettledBlocks[mergeIndex] = projectedMerged;
+ cache.projectedSettledBlocks.push(...projectedDelta.slice(1));
+ } else {
+ cache.projectedSettledBlocks.push(...projectedDelta);
+ }
+ cache.projectedRevision = cache.settledRevision;
}
// The unsettled tail is trimmed before parsing, so its offset in the
@@ -2790,7 +3123,7 @@ export function parseMarkdownIncremental(
definitionCharacters: settledDelta.length + unsettledInput.length,
renderedBlocks: parsedSettledBlocks + unsettledBlocks.length,
};
- return mergeSettledBlocks(state.settledBlocks, unsettledBlocks);
+ return mergeSettledBlocks(cache.settledAstBlocks, unsettledBlocks);
}
// ---------------------------------------------------------------------------
@@ -2801,30 +3134,6 @@ export function parseMarkdownIncremental(
// derives its item ids from the same functions, so outline hash links always
// resolve to a rendered heading by construction.
-/** Flatten inline nodes into their plain text content. */
-export function inlineText(nodes: ReadonlyArray<InlineNodeWithMath>): string {
- return nodes
- .map(node => {
- switch (node.type) {
- case 'text':
- case 'code':
- case 'math':
- return node.type === 'math' ? node.value : node.content;
- case 'bold':
- case 'italic':
- case 'strikethrough':
- case 'link':
- return inlineText(node.children);
- case 'image':
- return node.alt;
- case 'citation':
- case 'break':
- return '';
- }
- })
- .join('');
-}
-
/** Turn heading text into a URL-safe slug (lowercase, hyphen-separated). */
export function slugify(value: string): string {
return value
diff --git a/packages/core/src/Outline/parseOutlineFromMarkdown.ts b/packages/core/src/Outline/parseOutlineFromMarkdown.ts
index 5d4e02aae0..00c5c2f36f 100644
--- a/packages/core/src/Outline/parseOutlineFromMarkdown.ts
+++ b/packages/core/src/Outline/parseOutlineFromMarkdown.ts
@@ -2,22 +2,17 @@
/**
* @file parseOutlineFromMarkdown.ts
- * @input Uses Markdown parser internals (parseMarkdown + heading slug
- * helpers) and OutlineItem type
+ * @input Uses Markdown's canonical AST parser and heading slug helpers
* @output Exports parseOutlineFromMarkdown for extracting heading outlines from Markdown
- * @position Pure utility; consumed by useOutlineFromMarkdown and public exports
+ * @position Pure compatibility utility; consumed by useOutlineFromMarkdown and public exports
*
* SYNC: When modified, update these files to stay in sync:
* - /packages/core/src/Outline/Outline.doc.mjs
* - /packages/core/src/Outline/index.ts
*/
-import {
- parseMarkdown,
- inlineText,
- slugify,
- uniqueSlug,
-} from '../Markdown/parser';
+import {parseMarkdownAst, slugify, uniqueSlug} from '../Markdown/parser';
+import {markdownAstText} from '../Markdown/ast';
import type {OutlineItem} from './types';
/**
@@ -30,14 +25,14 @@ import type {OutlineItem} from './types';
*/
export function parseOutlineFromMarkdown(markdown: string): OutlineItem[] {
const counts = new Map<string, number>();
- return parseMarkdown(markdown)
- .filter(block => block.type === 'heading')
+ return parseMarkdownAst(markdown)
+ .children.filter(block => block.type === 'heading')
.map(block => {
- const label = inlineText(block.children).trim();
+ const label = markdownAstText(block.children).trim();
return {
id: uniqueSlug(slugify(label), counts),
label,
- level: block.level,
+ level: block.depth,
};
});
}