An open source design system that's fully customizable and agent ready
TypeScript
feat(markdown): opt-in source ranges on parsed blocks (#5290)
Alexander Blome93acd6225cb
diff --git a/.changeset/markdown-source-ranges.md b/.changeset/markdown-source-ranges.mdnew file mode 100644index 0000000000..4b4c7cce04--- /dev/null+++ b/.changeset/markdown-source-ranges.md@@ -0,0 +1,32 @@+---+'@astryxdesign/core': patch+---++[feat] Markdown: opt-in source ranges on parsed blocks++`parseMarkdown(source, {sourceRanges: true})` now gives every top-level block a+`range` — the `[start, end)` offsets it occupies in the source that was passed+in — so a consumer holding that source can slice the original markdown for a+block instead of reconstructing it from the node (or from the rendered DOM).+Reconstruction is lossy in ways slicing is not: escapes, the exact emphasis and+fence characters, heading depth beyond the clamp, and alignment all survive a+slice unchanged.++Off by default and absent unless asked for, so no existing node, snapshot or+comparison changes.++Two things the offsets get right that a naive implementation does not: link+reference definitions are stripped before the block loop runs, and the ranges+are reported against the string the caller passed rather than the stripped+text; and `parseMarkdownIncremental` parses slices, so blocks report absolute+offsets into the whole document as it streams — including a list whose halves+arrived in separate chunks and were merged.++A range covers a block's own lines verbatim, so slicing it and parsing the+result gives the same node back.++Blocks nested inside a list item or a blockquote carry no range: their children+are parsed from text the parser reassembled with markers and `>` prefixes+removed, so an offset into it would not address the document.++@lexsdiff --git a/packages/core/src/Markdown/index.ts b/packages/core/src/Markdown/index.tsindex 462f2d564f..3aa229ef48 100644--- a/packages/core/src/Markdown/index.ts+++ b/packages/core/src/Markdown/index.ts@@ -25,6 +25,7 @@ export {export type {BlockNode,InlineNode,+ SourceRange,ListItemNode,TableCellNode,TableAlignment,diff --git a/packages/core/src/Markdown/parser.test.ts b/packages/core/src/Markdown/parser.test.tsindex 38bb51996f..85daa04fa7 100644--- a/packages/core/src/Markdown/parser.test.ts+++ b/packages/core/src/Markdown/parser.test.ts@@ -1,8 +1,13 @@// Copyright (c) Meta Platforms, Inc. and affiliates.import {describe, it, expect} from 'vitest';-import {parseMarkdown, parseInline} from './parser';-import type {InlineNode} from './parser';+import {+ createIncrementalState,+ parseInline,+ parseMarkdown,+ parseMarkdownIncremental,+} from './parser';+import type {BlockNode, InlineNode} from './parser';describe('parseInline', () => {it('parses plain text', () => {@@ -1209,3 +1214,129 @@ describe('link reference definitions', () => {expect(blocks[0].type).toBe('paragraph');});});++describe('sourceRanges', () => {+ const slice = (source: string, block: {range?: readonly [number, number]}) =>+ block.range == null ? null : source.slice(block.range[0], block.range[1]);++ it('is off by default', () => {+ const [block] = parseMarkdown('# Title');+ expect(block.range).toBeUndefined();+ });++ it('gives every top-level block the source it came from', () => {+ const source = [+ '# Title',+ '',+ 'A paragraph that',+ 'wraps onto two lines.',+ '',+ '- one',+ '- two',+ '',+ '```js',+ 'const x = 1;',+ '```',+ '',+ '| a | b |',+ '| --- | --- |',+ '| 1 | 2 |',+ '',+ '> quoted',+ '',+ '---',+ ].join('\n');+ const blocks = parseMarkdown(source, {sourceRanges: true});+ expect(blocks.map(b => slice(source, b))).toEqual([+ '# Title',+ 'A paragraph that\nwraps onto two lines.',+ '- one\n- two',+ '```js\nconst x = 1;\n```',+ '| a | b |\n| --- | --- |\n| 1 | 2 |',+ '> quoted',+ '---',+ ]);+ });++ it('reports offsets into the input, not into the link-definition-stripped text', () => {+ // The definition lines are removed before the block loop runs, so a naive+ // offset would drift by their length for everything after them.+ const source = [+ '[ref]: https://example.com',+ '',+ 'See [the docs][ref].',+ '',+ 'Another paragraph.',+ ].join('\n');+ const blocks = parseMarkdown(source, {sourceRanges: true});+ expect(blocks.map(b => slice(source, b))).toEqual([+ 'See [the docs][ref].',+ 'Another paragraph.',+ ]);+ });++ it('reports absolute offsets when the document is parsed incrementally', () => {+ const source = ['# Title', '', 'One.', '', 'Two.', '', 'Three.'].join('\n');+ const state = createIncrementalState();+ let blocks: BlockNode[] = [];+ for (let end = 1; end <= source.length; end++) {+ blocks = parseMarkdownIncremental(source.slice(0, end), state, {+ sourceRanges: true,+ });+ }+ expect(blocks.map(b => slice(source, b))).toEqual([+ '# Title',+ 'One.',+ 'Two.',+ 'Three.',+ ]);+ });++ it('keeps the blank lines an unterminated fence owns', () => {+ // Mid-stream the closing fence has not arrived, and the blank lines are+ // part of the code, not spacing between blocks.+ const source = '```\ncode\n\n';+ const [block] = parseMarkdown(source, {sourceRanges: true});+ // The whole thing: the fence consumed those lines as code.+ expect(slice(source, block)).toBe(source);+ });++ it('slices to something that re-parses to the same block', () => {+ // The property a consumer actually needs, and the one that says the+ // offsets are right: what the range points at is the block.+ const check = (source: string) => {+ for (const block of parseMarkdown(source, {sourceRanges: true})) {+ const {range: _range, ...node} = block;+ expect(parseMarkdown(slice(source, block)!)).toEqual([node]);+ }+ };+ check('# Title\n\nA paragraph.\n\n- one\n- two\n\n> quoted');+ // CRLF: the parser keeps the `\r` in its own content, so the range does+ // too rather than slicing to text that parses differently.+ check('# Title\r\n\r\nA paragraph.\r\n');+ });++ it('re-parses when the caller flips the option on an existing state', () => {+ const source = 'One.\n\nTwo.\n\nThree.';+ const state = createIncrementalState();+ const without = parseMarkdownIncremental(source, state);+ expect(without.every(b => b.range == null)).toBe(true);+ const with_ = parseMarkdownIncremental(source, state, {+ sourceRanges: true,+ });+ expect(with_.map(b => slice(source, b))).toEqual(['One.', 'Two.', 'Three.']);+ const back = parseMarkdownIncremental(source, state);+ expect(back.every(b => b.range == null)).toBe(true);+ });++ it('spans both halves of a list the incremental parser merged', () => {+ const source = '1. one\n\n1. two';+ const state = createIncrementalState();+ parseMarkdownIncremental('1. one\n\n', state, {sourceRanges: true});+ const blocks = parseMarkdownIncremental(source, state, {+ sourceRanges: true,+ });+ expect(blocks).toHaveLength(1);+ expect(slice(source, blocks[0])).toBe(source);+ });+});diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.tsindex 3ed553d7aa..c11b57f7cc 100644--- a/packages/core/src/Markdown/parser.ts+++ b/packages/core/src/Markdown/parser.ts@@ -24,7 +24,15 @@ export type InlineNode =| {type: 'citation'; sourceId: string}| {type: 'break'};-export type BlockNode =+export type BlockNode = BlockNodeKind & {+ /**+ * Where this block came from in the source, when parsed with the+ * `sourceRanges` option. Top-level blocks only.+ */+ range?: SourceRange;+};++type BlockNodeKind =| {type: 'heading'; level: 1 | 2 | 3 | 4 | 5 | 6; children: InlineNode[]}| {type: 'paragraph'; children: InlineNode[]}| {type: 'codeblock'; language: string; content: string}@@ -47,6 +55,12 @@ export type BlockNode =| {type: 'hr'}| {type: 'image'; src: string; alt: string};+/**+ * Character offsets `[start, end)` of a block in the source string handed to+ * `parseMarkdown`. `end` excludes the block's trailing blank lines.+ */+export type SourceRange = readonly [start: number, end: number];+export type ListItemNode = {checked?: boolean; children: BlockNode[]};export type TableCellNode = {children: InlineNode[]};export type TableAlignment = 'left' | 'center' | 'right' | null;@@ -78,11 +92,28 @@ export type ParseOptions = {* suffix is not rejected (Astryx accepts any plausible TLD shape).*/autolink?: 'gfm';+ /**+ * When true, every top-level block carries a `range` — the offsets it+ * occupies in the string passed in. Lets a consumer that still holds the+ * source slice the original markdown for a block instead of reconstructing+ * it from the parsed node (or from the rendered DOM). Off by default: the+ * field is absent unless asked for, so nothing that compares nodes changes.+ *+ * Blocks nested inside a list item or a blockquote do not carry one.+ */+ sourceRanges?: boolean;};type ResolvedOptions = {readonly sourceIds: ReadonlySet<string> | undefined;readonly autolink: 'gfm' | undefined;+ readonly sourceRanges?: boolean;+ /**+ * Offset of this parse's input within the document the ranges are reported+ * against. Internal only — the incremental parser parses slices and needs+ * their blocks' ranges to come out absolute.+ */+ readonly baseOffset?: number;/*** Link reference definitions (`[label]: url`) collected from the whole* document, keyed by normalized label. Internal only — populated by the@@ -109,7 +140,11 @@ function resolveOptions(return {sourceIds: arg as ReadonlySet<string>, autolink: undefined};}const opts = arg as ParseOptions;- return {sourceIds: opts.sourceIds, autolink: opts.autolink};+ return {+ sourceIds: opts.sourceIds,+ autolink: opts.autolink,+ sourceRanges: opts.sourceRanges,+ };}// ---------------------------------------------------------------------------@@ -174,6 +209,11 @@ function matchLinkDefinition(function extractLinkDefinitions(input: string): {defs: ReadonlyMap<string, string>;cleaned: string;+ /**+ * For each line of `cleaned`, the line of `input` it came from. Undefined+ * when nothing was stripped and the two are the same text.+ */+ lineMap?: number[];} {const lines = input.split('\n');const defs = new Map<string, string>();@@ -235,8 +275,14 @@ function extractLinkDefinitions(input: string): {if (defs.size === 0) {return {defs, cleaned: input};}- const cleaned = lines.filter((_, index) => keep[index]).join('\n');- return {defs, cleaned};+ const lineMap: number[] = [];+ for (let index = 0; index < lines.length; index++) {+ if (keep[index]) {+ lineMap.push(index);+ }+ }+ const cleaned = lineMap.map(index => lines[index]).join('\n');+ return {defs, cleaned, lineMap};}/** Order-independent signature of a link-definition set, for cache checks. */@@ -1042,6 +1088,16 @@ function isTableSeparator(line: string): boolean {return nonEmpty.length > 0 && nonEmpty.every(cell => /^:?-+:?$/.test(cell));}+/**+ * The same options for content parsed out of an enclosing block. Ranges are a+ * top-level contract: a list item's or a blockquote's children are parsed from+ * text the caller reassembled (markers and `>` prefixes stripped), so an+ * offset into it would not address the document.+ */+function nested(opts: ResolvedOptions): ResolvedOptions {+ return opts.sourceRanges ? {...opts, sourceRanges: false} : opts;+}+/*** Returns true when a line could start a new block — used to stop paragraph* continuation. Every regex here uses bounded or single-class quantifiers@@ -1213,7 +1269,7 @@ function parseList(itemText += '\n' + deindented.join('\n');}- items.push({checked, children: parseMarkdownImpl(itemText, opts)});+ items.push({checked, children: parseMarkdownImpl(itemText, nested(opts))});// CommonMark loose list: blank line(s) between items of the same style// and indent still form one list. Skip the blanks and continue if the@@ -1274,7 +1330,7 @@ function parseMarkdownImpl(// definitions win on conflict, matching CommonMark's first-definition-wins// in document order; locally-nested definitions still resolve within this// parse.- const {defs, cleaned} = extractLinkDefinitions(input);+ const {defs, cleaned, lineMap} = extractLinkDefinitions(input);const inherited = baseOpts.linkDefs;let linkDefs: ReadonlyMap<string, string> | undefined;if (defs.size === 0) {@@ -1288,9 +1344,25 @@ function parseMarkdownImpl(linkDefs != null ? {...baseOpts, linkDefs} : baseOpts;const lines = cleaned.split('\n');const blocks: BlockNode[] = [];+ // The line each block started on, parallel to `blocks`. Only collected when+ // ranges were asked for; a block's end is resolved after the loop, since the+ // branch that produced it has already moved `index` past whatever it read.+ const blockStartLines: number[] | null = opts.sourceRanges ? [] : null;+ // Set only by a block that consumes blank lines as content, where the+ // positional end derivation would trim them away.+ const blockEndLines: (number | undefined)[] | null = opts.sourceRanges+ ? []+ : null;+ let blockStartLine = 0;+ const pushBlock = (node: BlockNode, endLine?: number) => {+ blocks.push(node);+ blockStartLines?.push(blockStartLine);+ blockEndLines?.push(endLine);+ };let index = 0;while (index < lines.length) {+ blockStartLine = index;const line = lines[index];if (line.trim() === '') {index++;@@ -1309,14 +1381,20 @@ function parseMarkdownImpl(index++;}index++; // skip closing fence- blocks.push({type: 'codeblock', language, content: codeLines.join('\n')});+ // A fence owns its blank lines, and an unterminated one (mid-stream)+ // can end on them, so it states its own end rather than letting the+ // positional derivation trim them off.+ pushBlock(+ {type: 'codeblock', language, content: codeLines.join('\n')},+ Math.min(index, lines.length) - 1,+ );continue;}// --- Heading ---const headingMatch = line.match(/^(#{1,6}) +(.*)/);if (headingMatch) {- blocks.push({+ pushBlock({type: 'heading',level: headingMatch[1].length as 1 | 2 | 3 | 4 | 5 | 6,children: parseInlineEntry(headingMatch[2], opts),@@ -1327,7 +1405,7 @@ function parseMarkdownImpl(// --- HR (must precede list check to handle `- - -`, `* * *`, `_ _ _`) ---if (isHorizontalRule(line)) {- blocks.push({type: 'hr'});+ pushBlock({type: 'hr'});index++;continue;}@@ -1335,7 +1413,7 @@ function parseMarkdownImpl(// --- Standalone image ---const imageMatch = line.match(/^!\[([^\]]*)\]\(([^)]+)\)/);if (imageMatch && line.trim() === imageMatch[0]) {- blocks.push({type: 'image', alt: imageMatch[1], src: imageMatch[2]});+ pushBlock({type: 'image', alt: imageMatch[1], src: imageMatch[2]});index++;continue;}@@ -1347,7 +1425,7 @@ function parseMarkdownImpl(isTableSeparator(lines[index + 1])) {const tableResult = parseTable(lines, index, opts);- blocks.push(tableResult.node);+ pushBlock(tableResult.node);index = tableResult.nextIndex;continue;}@@ -1362,9 +1440,9 @@ function parseMarkdownImpl(quoteLines.push(lines[index].replace(/^> ?/, ''));index++;}- blocks.push({+ pushBlock({type: 'blockquote',- children: parseMarkdownImpl(quoteLines.join('\n'), opts),+ children: parseMarkdownImpl(quoteLines.join('\n'), nested(opts)),});continue;}@@ -1372,7 +1450,7 @@ function parseMarkdownImpl(// --- Unordered list ---if (/^ {0,9}[-*+] /.test(line)) {const listResult = parseList(lines, index, false, opts);- blocks.push(listResult.node);+ pushBlock(listResult.node);index = listResult.nextIndex;continue;}@@ -1380,7 +1458,7 @@ function parseMarkdownImpl(// --- Ordered list ---if (/^ {0,9}\d+[.)] /.test(line)) {const listResult = parseList(lines, index, true, opts);- blocks.push(listResult.node);+ pushBlock(listResult.node);index = listResult.nextIndex;continue;}@@ -1396,14 +1474,76 @@ function parseMarkdownImpl(paraLines.push(lines[index]);index++;}- blocks.push({+ pushBlock({type: 'paragraph',children: parseInlineEntry(paraLines.join('\n'), opts),});}+ if (blockStartLines != null) {+ stampSourceRanges(+ blocks,+ blockStartLines,+ blockEndLines ?? [],+ lines,+ lineMap,+ input,+ opts,+ );+ }return blocks;}+/**+ * Give each block the offsets it occupies in the original input.+ *+ * Blocks are contiguous and in source order, so a block runs from its own+ * first line to the line before the next block starts, minus the blank lines+ * between them. Offsets are computed against the *input*, not the text the+ * block loop saw: link reference definitions are stripped before parsing, and+ * `lineMap` says which input line each surviving line came from.+ */+function stampSourceRanges(+ blocks: BlockNode[],+ blockStartLines: number[],+ blockEndLines: (number | undefined)[],+ lines: string[],+ lineMap: number[] | undefined,+ input: string,+ opts: ResolvedOptions,+): void {+ const base = opts.baseOffset ?? 0;+ // Offset of the first character of every line of the input.+ const inputLineStarts = [0];+ for (let i = 0; i < input.length; i++) {+ if (input[i] === '\n') {+ inputLineStarts.push(i + 1);+ }+ }+ // Stripping removes whole lines and never edits one, so a parsed line's+ // length is its input line's length.+ const lineStart = (line: number): number =>+ base + inputLineStarts[lineMap != null ? lineMap[line] : line];++ for (let i = 0; i < blocks.length; i++) {+ const startLine = blockStartLines[i];+ let endLine = blockEndLines[i];+ if (endLine == null) {+ const nextStart =+ i + 1 < blocks.length ? blockStartLines[i + 1] : lines.length;+ endLine = nextStart - 1;+ while (endLine > startLine && lines[endLine].trim() === '') {+ endLine--;+ }+ }+ // Exactly the block's own lines, verbatim — a CRLF document's trailing+ // `\r` included, since the parser reads it as part of the line too and a+ // range that dropped it would slice to something that re-parses+ // differently.+ const end = lineStart(endLine) + lines[endLine].length;+ blocks[i] = {...blocks[i], range: [lineStart(startLine), end]};+ }+}+// ---------------------------------------------------------------------------// Incremental parsing// ---------------------------------------------------------------------------@@ -1420,6 +1560,12 @@ export interface IncrementalState {* with newly-arriving content.*/autolink?: 'gfm';+ /**+ * The `sourceRanges` option the cached `settledBlocks` were parsed with.+ * Flipping it invalidates them the same way `autolink` does: they either+ * lack the ranges the caller now asks for, or carry ones it did not.+ */+ sourceRanges?: boolean;/*** Signature of the link reference definitions the cached `settledBlocks`* were parsed with. Definitions are document-global and typically arrive@@ -1689,6 +1835,15 @@ function trimUnsettledStructural(text: string): string {return lines.join('\n');}+/**+ * The same options, for parsing a slice that starts at `offset` of the+ * document — so the slice's blocks report ranges into the whole document+ * rather than into the slice.+ */+function atOffset(opts: ResolvedOptions, offset: number): ResolvedOptions {+ return opts.sourceRanges ? {...opts, baseOffset: offset} : opts;+}+/*** Concatenate freshly-parsed delta blocks with previously-settled blocks,* merging adjacent same-style lists into a single loose list. The boundary@@ -1718,6 +1873,11 @@ function mergeSettledBlocks(delimiter: prevLast.delimiter,loose: true,items: [...prevLast.items, ...deltaFirst.items],+ // One list now, so one range: from where the first half started to+ // where the second half ended.+ ...(prevLast.range != null && deltaFirst.range != null+ ? {range: [prevLast.range[0], deltaFirst.range[1]] as SourceRange}+ : null),};return [...prev.slice(0, -1), merged, ...delta.slice(1)];}@@ -1740,15 +1900,19 @@ export function parseMarkdownIncremental(arg?: ReadonlySet<string> | ParseOptions,): BlockNode[] {const opts = resolveOptions(arg);- // Invalidate cache when the autolink option flips — cached settled blocks- // were parsed with the previous setting and would otherwise be reused- // unchanged.- if (state.autolink !== opts.autolink) {+ // Invalidate cache when the autolink or sourceRanges option flips — cached+ // settled blocks were parsed with the previous setting and would otherwise+ // be reused unchanged.+ if (+ state.autolink !== opts.autolink ||+ Boolean(state.sourceRanges) !== Boolean(opts.sourceRanges)+ ) {state.prevInput = '';state.settledText = '';state.settledBlocks = [];state.settledUpTo = 0;state.autolink = opts.autolink;+ state.sourceRanges = opts.sourceRanges;}if (input === '') {state.prevInput = '';@@ -1799,15 +1963,32 @@ export function parseMarkdownIncremental() {// Settled portion grew — parse only the new deltaconst delta = settledText.slice(state.settledText.length);- const deltaBlocks = parseMarkdownImpl(delta, parseOpts);+ const deltaBlocks = parseMarkdownImpl(+ delta,+ atOffset(parseOpts, state.settledText.length),+ );settledBlocks = mergeSettledBlocks(state.settledBlocks, deltaBlocks);} else {// Content before the boundary changed — full re-parse of settled portionsettledBlocks = parseMarkdownImpl(settledText, parseOpts);}+ // The unsettled tail is trimmed before parsing, so its offset in the+ // document is where that trimmed text actually starts — not the boundary,+ // which is a line index. If it somehow can't be located, parse it without+ // ranges rather than report wrong ones. Only worth searching for when+ // ranges were asked for: this runs on every streamed chunk.+ const unsettledStart =+ unsettledText && opts.sourceRanges+ ? input.indexOf(unsettledText, settledText.length)+ : -1;const unsettledBlocks = unsettledText- ? parseMarkdownImpl(unsettledText, parseOpts)+ ? parseMarkdownImpl(+ unsettledText,+ unsettledStart >= 0+ ? atOffset(parseOpts, unsettledStart)+ : nested(parseOpts),+ ): [];state.settledText = settledText;