Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

feat(markdown): opt-in source ranges on parsed blocks (#5290)

Alexander Blome93acd6225cb
diff --git a/.changeset/markdown-source-ranges.md b/.changeset/markdown-source-ranges.md
new file mode 100644
index 0000000000..4b4c7cce04
--- /dev/null
+++ b/.changeset/markdown-source-ranges.md
@@ -0,0 +1,32 @@
+---
+'@astryxdesign/core': patch
+---
+
+[feat] Markdown: opt-in source ranges on parsed blocks
+
+`parseMarkdown(source, {sourceRanges: true})` now gives every top-level block a
+`range` — the `[start, end)` offsets it occupies in the source that was passed
+in — so a consumer holding that source can slice the original markdown for a
+block instead of reconstructing it from the node (or from the rendered DOM).
+Reconstruction is lossy in ways slicing is not: escapes, the exact emphasis and
+fence characters, heading depth beyond the clamp, and alignment all survive a
+slice unchanged.
+
+Off by default and absent unless asked for, so no existing node, snapshot or
+comparison changes.
+
+Two things the offsets get right that a naive implementation does not: link
+reference definitions are stripped before the block loop runs, and the ranges
+are reported against the string the caller passed rather than the stripped
+text; and `parseMarkdownIncremental` parses slices, so blocks report absolute
+offsets into the whole document as it streams — including a list whose halves
+arrived in separate chunks and were merged.
+
+A range covers a block's own lines verbatim, so slicing it and parsing the
+result gives the same node back.
+
+Blocks nested inside a list item or a blockquote carry no range: their children
+are parsed from text the parser reassembled with markers and `>` prefixes
+removed, so an offset into it would not address the document.
+
+@lexs
diff --git a/packages/core/src/Markdown/index.ts b/packages/core/src/Markdown/index.ts
index 462f2d564f..3aa229ef48 100644
--- a/packages/core/src/Markdown/index.ts
+++ b/packages/core/src/Markdown/index.ts
@@ -25,6 +25,7 @@ export {
export type {
BlockNode,
InlineNode,
+ SourceRange,
ListItemNode,
TableCellNode,
TableAlignment,
diff --git a/packages/core/src/Markdown/parser.test.ts b/packages/core/src/Markdown/parser.test.ts
index 38bb51996f..85daa04fa7 100644
--- a/packages/core/src/Markdown/parser.test.ts
+++ b/packages/core/src/Markdown/parser.test.ts
@@ -1,8 +1,13 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
import {describe, it, expect} from 'vitest';
-import {parseMarkdown, parseInline} from './parser';
-import type {InlineNode} from './parser';
+import {
+ createIncrementalState,
+ parseInline,
+ parseMarkdown,
+ parseMarkdownIncremental,
+} from './parser';
+import type {BlockNode, InlineNode} from './parser';
describe('parseInline', () => {
it('parses plain text', () => {
@@ -1209,3 +1214,129 @@ describe('link reference definitions', () => {
expect(blocks[0].type).toBe('paragraph');
});
});
+
+describe('sourceRanges', () => {
+ const slice = (source: string, block: {range?: readonly [number, number]}) =>
+ block.range == null ? null : source.slice(block.range[0], block.range[1]);
+
+ it('is off by default', () => {
+ const [block] = parseMarkdown('# Title');
+ expect(block.range).toBeUndefined();
+ });
+
+ it('gives every top-level block the source it came from', () => {
+ const source = [
+ '# Title',
+ '',
+ 'A paragraph that',
+ 'wraps onto two lines.',
+ '',
+ '- one',
+ '- two',
+ '',
+ '```js',
+ 'const x = 1;',
+ '```',
+ '',
+ '| a | b |',
+ '| --- | --- |',
+ '| 1 | 2 |',
+ '',
+ '> quoted',
+ '',
+ '---',
+ ].join('\n');
+ const blocks = parseMarkdown(source, {sourceRanges: true});
+ expect(blocks.map(b => slice(source, b))).toEqual([
+ '# Title',
+ 'A paragraph that\nwraps onto two lines.',
+ '- one\n- two',
+ '```js\nconst x = 1;\n```',
+ '| a | b |\n| --- | --- |\n| 1 | 2 |',
+ '> quoted',
+ '---',
+ ]);
+ });
+
+ it('reports offsets into the input, not into the link-definition-stripped text', () => {
+ // The definition lines are removed before the block loop runs, so a naive
+ // offset would drift by their length for everything after them.
+ const source = [
+ '[ref]: https://example.com',
+ '',
+ 'See [the docs][ref].',
+ '',
+ 'Another paragraph.',
+ ].join('\n');
+ const blocks = parseMarkdown(source, {sourceRanges: true});
+ expect(blocks.map(b => slice(source, b))).toEqual([
+ 'See [the docs][ref].',
+ 'Another paragraph.',
+ ]);
+ });
+
+ it('reports absolute offsets when the document is parsed incrementally', () => {
+ const source = ['# Title', '', 'One.', '', 'Two.', '', 'Three.'].join('\n');
+ const state = createIncrementalState();
+ let blocks: BlockNode[] = [];
+ for (let end = 1; end <= source.length; end++) {
+ blocks = parseMarkdownIncremental(source.slice(0, end), state, {
+ sourceRanges: true,
+ });
+ }
+ expect(blocks.map(b => slice(source, b))).toEqual([
+ '# Title',
+ 'One.',
+ 'Two.',
+ 'Three.',
+ ]);
+ });
+
+ it('keeps the blank lines an unterminated fence owns', () => {
+ // Mid-stream the closing fence has not arrived, and the blank lines are
+ // part of the code, not spacing between blocks.
+ const source = '```\ncode\n\n';
+ const [block] = parseMarkdown(source, {sourceRanges: true});
+ // The whole thing: the fence consumed those lines as code.
+ expect(slice(source, block)).toBe(source);
+ });
+
+ it('slices to something that re-parses to the same block', () => {
+ // The property a consumer actually needs, and the one that says the
+ // offsets are right: what the range points at is the block.
+ const check = (source: string) => {
+ for (const block of parseMarkdown(source, {sourceRanges: true})) {
+ const {range: _range, ...node} = block;
+ expect(parseMarkdown(slice(source, block)!)).toEqual([node]);
+ }
+ };
+ check('# Title\n\nA paragraph.\n\n- one\n- two\n\n> quoted');
+ // CRLF: the parser keeps the `\r` in its own content, so the range does
+ // too rather than slicing to text that parses differently.
+ check('# Title\r\n\r\nA paragraph.\r\n');
+ });
+
+ it('re-parses when the caller flips the option on an existing state', () => {
+ const source = 'One.\n\nTwo.\n\nThree.';
+ const state = createIncrementalState();
+ const without = parseMarkdownIncremental(source, state);
+ expect(without.every(b => b.range == null)).toBe(true);
+ const with_ = parseMarkdownIncremental(source, state, {
+ sourceRanges: true,
+ });
+ expect(with_.map(b => slice(source, b))).toEqual(['One.', 'Two.', 'Three.']);
+ const back = parseMarkdownIncremental(source, state);
+ expect(back.every(b => b.range == null)).toBe(true);
+ });
+
+ it('spans both halves of a list the incremental parser merged', () => {
+ const source = '1. one\n\n1. two';
+ const state = createIncrementalState();
+ parseMarkdownIncremental('1. one\n\n', state, {sourceRanges: true});
+ const blocks = parseMarkdownIncremental(source, state, {
+ sourceRanges: true,
+ });
+ expect(blocks).toHaveLength(1);
+ expect(slice(source, blocks[0])).toBe(source);
+ });
+});
diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.ts
index 3ed553d7aa..c11b57f7cc 100644
--- a/packages/core/src/Markdown/parser.ts
+++ b/packages/core/src/Markdown/parser.ts
@@ -24,7 +24,15 @@ export type InlineNode =
| {type: 'citation'; sourceId: string}
| {type: 'break'};
-export type BlockNode =
+export type BlockNode = BlockNodeKind & {
+ /**
+ * Where this block came from in the source, when parsed with the
+ * `sourceRanges` option. Top-level blocks only.
+ */
+ range?: SourceRange;
+};
+
+type BlockNodeKind =
| {type: 'heading'; level: 1 | 2 | 3 | 4 | 5 | 6; children: InlineNode[]}
| {type: 'paragraph'; children: InlineNode[]}
| {type: 'codeblock'; language: string; content: string}
@@ -47,6 +55,12 @@ export type BlockNode =
| {type: 'hr'}
| {type: 'image'; src: string; alt: string};
+/**
+ * Character offsets `[start, end)` of a block in the source string handed to
+ * `parseMarkdown`. `end` excludes the block's trailing blank lines.
+ */
+export type SourceRange = readonly [start: number, end: number];
+
export type ListItemNode = {checked?: boolean; children: BlockNode[]};
export type TableCellNode = {children: InlineNode[]};
export type TableAlignment = 'left' | 'center' | 'right' | null;
@@ -78,11 +92,28 @@ export type ParseOptions = {
* suffix is not rejected (Astryx accepts any plausible TLD shape).
*/
autolink?: 'gfm';
+ /**
+ * When true, every top-level block carries a `range` — the offsets it
+ * occupies in the string passed in. Lets a consumer that still holds the
+ * source slice the original markdown for a block instead of reconstructing
+ * it from the parsed node (or from the rendered DOM). Off by default: the
+ * field is absent unless asked for, so nothing that compares nodes changes.
+ *
+ * Blocks nested inside a list item or a blockquote do not carry one.
+ */
+ sourceRanges?: boolean;
};
type ResolvedOptions = {
readonly sourceIds: ReadonlySet<string> | undefined;
readonly autolink: 'gfm' | undefined;
+ readonly sourceRanges?: boolean;
+ /**
+ * Offset of this parse's input within the document the ranges are reported
+ * against. Internal only — the incremental parser parses slices and needs
+ * their blocks' ranges to come out absolute.
+ */
+ readonly baseOffset?: number;
/**
* Link reference definitions (`[label]: url`) collected from the whole
* document, keyed by normalized label. Internal only — populated by the
@@ -109,7 +140,11 @@ function resolveOptions(
return {sourceIds: arg as ReadonlySet<string>, autolink: undefined};
}
const opts = arg as ParseOptions;
- return {sourceIds: opts.sourceIds, autolink: opts.autolink};
+ return {
+ sourceIds: opts.sourceIds,
+ autolink: opts.autolink,
+ sourceRanges: opts.sourceRanges,
+ };
}
// ---------------------------------------------------------------------------
@@ -174,6 +209,11 @@ function matchLinkDefinition(
function extractLinkDefinitions(input: string): {
defs: ReadonlyMap<string, string>;
cleaned: string;
+ /**
+ * For each line of `cleaned`, the line of `input` it came from. Undefined
+ * when nothing was stripped and the two are the same text.
+ */
+ lineMap?: number[];
} {
const lines = input.split('\n');
const defs = new Map<string, string>();
@@ -235,8 +275,14 @@ function extractLinkDefinitions(input: string): {
if (defs.size === 0) {
return {defs, cleaned: input};
}
- const cleaned = lines.filter((_, index) => keep[index]).join('\n');
- return {defs, cleaned};
+ const lineMap: number[] = [];
+ for (let index = 0; index < lines.length; index++) {
+ if (keep[index]) {
+ lineMap.push(index);
+ }
+ }
+ const cleaned = lineMap.map(index => lines[index]).join('\n');
+ return {defs, cleaned, lineMap};
}
/** Order-independent signature of a link-definition set, for cache checks. */
@@ -1042,6 +1088,16 @@ function isTableSeparator(line: string): boolean {
return nonEmpty.length > 0 && nonEmpty.every(cell => /^:?-+:?$/.test(cell));
}
+/**
+ * The same options for content parsed out of an enclosing block. Ranges are a
+ * top-level contract: a list item's or a blockquote's children are parsed from
+ * text the caller reassembled (markers and `>` prefixes stripped), so an
+ * offset into it would not address the document.
+ */
+function nested(opts: ResolvedOptions): ResolvedOptions {
+ return opts.sourceRanges ? {...opts, sourceRanges: false} : opts;
+}
+
/**
* Returns true when a line could start a new block — used to stop paragraph
* continuation. Every regex here uses bounded or single-class quantifiers
@@ -1213,7 +1269,7 @@ function parseList(
itemText += '\n' + deindented.join('\n');
}
- items.push({checked, children: parseMarkdownImpl(itemText, opts)});
+ items.push({checked, children: parseMarkdownImpl(itemText, nested(opts))});
// CommonMark loose list: blank line(s) between items of the same style
// and indent still form one list. Skip the blanks and continue if the
@@ -1274,7 +1330,7 @@ function parseMarkdownImpl(
// definitions win on conflict, matching CommonMark's first-definition-wins
// in document order; locally-nested definitions still resolve within this
// parse.
- const {defs, cleaned} = extractLinkDefinitions(input);
+ const {defs, cleaned, lineMap} = extractLinkDefinitions(input);
const inherited = baseOpts.linkDefs;
let linkDefs: ReadonlyMap<string, string> | undefined;
if (defs.size === 0) {
@@ -1288,9 +1344,25 @@ function parseMarkdownImpl(
linkDefs != null ? {...baseOpts, linkDefs} : baseOpts;
const lines = cleaned.split('\n');
const blocks: BlockNode[] = [];
+ // The line each block started on, parallel to `blocks`. Only collected when
+ // ranges were asked for; a block's end is resolved after the loop, since the
+ // branch that produced it has already moved `index` past whatever it read.
+ const blockStartLines: number[] | null = opts.sourceRanges ? [] : null;
+ // Set only by a block that consumes blank lines as content, where the
+ // positional end derivation would trim them away.
+ const blockEndLines: (number | undefined)[] | null = opts.sourceRanges
+ ? []
+ : null;
+ let blockStartLine = 0;
+ const pushBlock = (node: BlockNode, endLine?: number) => {
+ blocks.push(node);
+ blockStartLines?.push(blockStartLine);
+ blockEndLines?.push(endLine);
+ };
let index = 0;
while (index < lines.length) {
+ blockStartLine = index;
const line = lines[index];
if (line.trim() === '') {
index++;
@@ -1309,14 +1381,20 @@ function parseMarkdownImpl(
index++;
}
index++; // skip closing fence
- blocks.push({type: 'codeblock', language, content: codeLines.join('\n')});
+ // A fence owns its blank lines, and an unterminated one (mid-stream)
+ // can end on them, so it states its own end rather than letting the
+ // positional derivation trim them off.
+ pushBlock(
+ {type: 'codeblock', language, content: codeLines.join('\n')},
+ Math.min(index, lines.length) - 1,
+ );
continue;
}
// --- Heading ---
const headingMatch = line.match(/^(#{1,6}) +(.*)/);
if (headingMatch) {
- blocks.push({
+ pushBlock({
type: 'heading',
level: headingMatch[1].length as 1 | 2 | 3 | 4 | 5 | 6,
children: parseInlineEntry(headingMatch[2], opts),
@@ -1327,7 +1405,7 @@ function parseMarkdownImpl(
// --- HR (must precede list check to handle `- - -`, `* * *`, `_ _ _`) ---
if (isHorizontalRule(line)) {
- blocks.push({type: 'hr'});
+ pushBlock({type: 'hr'});
index++;
continue;
}
@@ -1335,7 +1413,7 @@ function parseMarkdownImpl(
// --- Standalone image ---
const imageMatch = line.match(/^!\[([^\]]*)\]\(([^)]+)\)/);
if (imageMatch && line.trim() === imageMatch[0]) {
- blocks.push({type: 'image', alt: imageMatch[1], src: imageMatch[2]});
+ pushBlock({type: 'image', alt: imageMatch[1], src: imageMatch[2]});
index++;
continue;
}
@@ -1347,7 +1425,7 @@ function parseMarkdownImpl(
isTableSeparator(lines[index + 1])
) {
const tableResult = parseTable(lines, index, opts);
- blocks.push(tableResult.node);
+ pushBlock(tableResult.node);
index = tableResult.nextIndex;
continue;
}
@@ -1362,9 +1440,9 @@ function parseMarkdownImpl(
quoteLines.push(lines[index].replace(/^> ?/, ''));
index++;
}
- blocks.push({
+ pushBlock({
type: 'blockquote',
- children: parseMarkdownImpl(quoteLines.join('\n'), opts),
+ children: parseMarkdownImpl(quoteLines.join('\n'), nested(opts)),
});
continue;
}
@@ -1372,7 +1450,7 @@ function parseMarkdownImpl(
// --- Unordered list ---
if (/^ {0,9}[-*+] /.test(line)) {
const listResult = parseList(lines, index, false, opts);
- blocks.push(listResult.node);
+ pushBlock(listResult.node);
index = listResult.nextIndex;
continue;
}
@@ -1380,7 +1458,7 @@ function parseMarkdownImpl(
// --- Ordered list ---
if (/^ {0,9}\d+[.)] /.test(line)) {
const listResult = parseList(lines, index, true, opts);
- blocks.push(listResult.node);
+ pushBlock(listResult.node);
index = listResult.nextIndex;
continue;
}
@@ -1396,14 +1474,76 @@ function parseMarkdownImpl(
paraLines.push(lines[index]);
index++;
}
- blocks.push({
+ pushBlock({
type: 'paragraph',
children: parseInlineEntry(paraLines.join('\n'), opts),
});
}
+ if (blockStartLines != null) {
+ stampSourceRanges(
+ blocks,
+ blockStartLines,
+ blockEndLines ?? [],
+ lines,
+ lineMap,
+ input,
+ opts,
+ );
+ }
return blocks;
}
+/**
+ * Give each block the offsets it occupies in the original input.
+ *
+ * Blocks are contiguous and in source order, so a block runs from its own
+ * first line to the line before the next block starts, minus the blank lines
+ * between them. Offsets are computed against the *input*, not the text the
+ * block loop saw: link reference definitions are stripped before parsing, and
+ * `lineMap` says which input line each surviving line came from.
+ */
+function stampSourceRanges(
+ blocks: BlockNode[],
+ blockStartLines: number[],
+ blockEndLines: (number | undefined)[],
+ lines: string[],
+ lineMap: number[] | undefined,
+ input: string,
+ opts: ResolvedOptions,
+): void {
+ const base = opts.baseOffset ?? 0;
+ // Offset of the first character of every line of the input.
+ const inputLineStarts = [0];
+ for (let i = 0; i < input.length; i++) {
+ if (input[i] === '\n') {
+ inputLineStarts.push(i + 1);
+ }
+ }
+ // Stripping removes whole lines and never edits one, so a parsed line's
+ // length is its input line's length.
+ const lineStart = (line: number): number =>
+ base + inputLineStarts[lineMap != null ? lineMap[line] : line];
+
+ for (let i = 0; i < blocks.length; i++) {
+ const startLine = blockStartLines[i];
+ let endLine = blockEndLines[i];
+ if (endLine == null) {
+ const nextStart =
+ i + 1 < blocks.length ? blockStartLines[i + 1] : lines.length;
+ endLine = nextStart - 1;
+ while (endLine > startLine && lines[endLine].trim() === '') {
+ endLine--;
+ }
+ }
+ // Exactly the block's own lines, verbatim — a CRLF document's trailing
+ // `\r` included, since the parser reads it as part of the line too and a
+ // range that dropped it would slice to something that re-parses
+ // differently.
+ const end = lineStart(endLine) + lines[endLine].length;
+ blocks[i] = {...blocks[i], range: [lineStart(startLine), end]};
+ }
+}
+
// ---------------------------------------------------------------------------
// Incremental parsing
// ---------------------------------------------------------------------------
@@ -1420,6 +1560,12 @@ export interface IncrementalState {
* with newly-arriving content.
*/
autolink?: 'gfm';
+ /**
+ * The `sourceRanges` option the cached `settledBlocks` were parsed with.
+ * Flipping it invalidates them the same way `autolink` does: they either
+ * lack the ranges the caller now asks for, or carry ones it did not.
+ */
+ sourceRanges?: boolean;
/**
* Signature of the link reference definitions the cached `settledBlocks`
* were parsed with. Definitions are document-global and typically arrive
@@ -1689,6 +1835,15 @@ function trimUnsettledStructural(text: string): string {
return lines.join('\n');
}
+/**
+ * The same options, for parsing a slice that starts at `offset` of the
+ * document — so the slice's blocks report ranges into the whole document
+ * rather than into the slice.
+ */
+function atOffset(opts: ResolvedOptions, offset: number): ResolvedOptions {
+ return opts.sourceRanges ? {...opts, baseOffset: offset} : opts;
+}
+
/**
* Concatenate freshly-parsed delta blocks with previously-settled blocks,
* merging adjacent same-style lists into a single loose list. The boundary
@@ -1718,6 +1873,11 @@ function mergeSettledBlocks(
delimiter: prevLast.delimiter,
loose: true,
items: [...prevLast.items, ...deltaFirst.items],
+ // One list now, so one range: from where the first half started to
+ // where the second half ended.
+ ...(prevLast.range != null && deltaFirst.range != null
+ ? {range: [prevLast.range[0], deltaFirst.range[1]] as SourceRange}
+ : null),
};
return [...prev.slice(0, -1), merged, ...delta.slice(1)];
}
@@ -1740,15 +1900,19 @@ export function parseMarkdownIncremental(
arg?: ReadonlySet<string> | ParseOptions,
): BlockNode[] {
const opts = resolveOptions(arg);
- // Invalidate cache when the autolink option flips — cached settled blocks
- // were parsed with the previous setting and would otherwise be reused
- // unchanged.
- if (state.autolink !== opts.autolink) {
+ // Invalidate cache when the autolink or sourceRanges option flips — cached
+ // settled blocks were parsed with the previous setting and would otherwise
+ // be reused unchanged.
+ if (
+ state.autolink !== opts.autolink ||
+ Boolean(state.sourceRanges) !== Boolean(opts.sourceRanges)
+ ) {
state.prevInput = '';
state.settledText = '';
state.settledBlocks = [];
state.settledUpTo = 0;
state.autolink = opts.autolink;
+ state.sourceRanges = opts.sourceRanges;
}
if (input === '') {
state.prevInput = '';
@@ -1799,15 +1963,32 @@ export function parseMarkdownIncremental(
) {
// Settled portion grew — parse only the new delta
const delta = settledText.slice(state.settledText.length);
- const deltaBlocks = parseMarkdownImpl(delta, parseOpts);
+ const deltaBlocks = parseMarkdownImpl(
+ delta,
+ atOffset(parseOpts, state.settledText.length),
+ );
settledBlocks = mergeSettledBlocks(state.settledBlocks, deltaBlocks);
} else {
// Content before the boundary changed — full re-parse of settled portion
settledBlocks = parseMarkdownImpl(settledText, parseOpts);
}
+ // The unsettled tail is trimmed before parsing, so its offset in the
+ // document is where that trimmed text actually starts — not the boundary,
+ // which is a line index. If it somehow can't be located, parse it without
+ // ranges rather than report wrong ones. Only worth searching for when
+ // ranges were asked for: this runs on every streamed chunk.
+ const unsettledStart =
+ unsettledText && opts.sourceRanges
+ ? input.indexOf(unsettledText, settledText.length)
+ : -1;
const unsettledBlocks = unsettledText
- ? parseMarkdownImpl(unsettledText, parseOpts)
+ ? parseMarkdownImpl(
+ unsettledText,
+ unsettledStart >= 0
+ ? atOffset(parseOpts, unsettledStart)
+ : nested(parseOpts),
+ )
: [];
state.settledText = settledText;