An open source design system that's fully customizable and agent ready
TypeScript
refactor(markdown): SourceRange as an object, not a tuple (#5366)
Cindy Zhang5d351e42b7ce
diff --git a/.changeset/markdown-source-ranges.md b/.changeset/markdown-source-ranges.mdindex 4b4c7cce04..bb70fb2fcb 100644--- a/.changeset/markdown-source-ranges.md+++ b/.changeset/markdown-source-ranges.md@@ -5,9 +5,10 @@[feat] Markdown: opt-in source ranges on parsed blocks`parseMarkdown(source, {sourceRanges: true})` now gives every top-level block a-`range` — the `[start, end)` offsets it occupies in the source that was passed-in — so a consumer holding that source can slice the original markdown for a-block instead of reconstructing it from the node (or from the rendered DOM).+`range` — `{start, end}`, the character offsets it occupies in the source that+was passed in, with `end` exclusive — so a consumer holding that source can+`source.slice(range.start, range.end)` for a block instead of reconstructing it+from the node (or from the rendered DOM).Reconstruction is lossy in ways slicing is not: escapes, the exact emphasis andfence characters, heading depth beyond the clamp, and alignment all survive aslice unchanged.diff --git a/packages/core/src/Markdown/parser.perf.test.ts b/packages/core/src/Markdown/parser.perf.test.tsindex 273cb310fd..0078feb658 100644--- a/packages/core/src/Markdown/parser.perf.test.ts+++ b/packages/core/src/Markdown/parser.perf.test.ts@@ -6,7 +6,7 @@ import {parseMarkdownIncremental,createIncrementalState,} from './parser';-import type {BlockNode} from './parser';+import type {BlockNode, ParseOptions} from './parser';function generateAIResponse(paragraphs: number): string {const sections: string[] = [];@@ -69,6 +69,42 @@ function simulateStreamingIncremental(return parseMarkdownIncremental(fullText, state);}+/**+ * How many blocks each chunk of a streamed document had to build.+ *+ * A block the parser kept in its settled cache is handed back as the very same+ * object, so a returned block that is not one of the previous chunk's is one+ * this chunk built. That makes the cache countable without reaching into the+ * parser's internals.+ */+function blocksBuiltPerChunk(+ fullText: string,+ chunkSize: number,+ options?: ParseOptions,+): number[] {+ const state = createIncrementalState();+ let previous: ReadonlySet<BlockNode> = new Set();+ const built: number[] = [];+ const step = (text: string) => {+ const blocks =+ options == null+ ? parseMarkdownIncremental(text, state)+ : parseMarkdownIncremental(text, state, options);+ built.push(blocks.reduce((n, b) => (previous.has(b) ? n : n + 1), 0));+ previous = new Set(blocks);+ };+ for (let i = chunkSize; i <= fullText.length; i += chunkSize) {+ step(fullText.slice(0, i));+ }+ step(fullText);+ return built;+}++const median = (values: number[]): number =>+ [...values].sort((a, b) => a - b)[Math.floor((values.length - 1) / 2)];++const sum = (values: number[]): number => values.reduce((a, b) => a + b, 0);+// A single wall-clock measurement is flaky when the whole suite runs in// parallel forks: scheduler contention can inflate a millisecond-scale// sample well past a tight budget (observed 30.9ms against the 20ms Small@@ -204,3 +240,37 @@ describe('parseMarkdown performance', () => {expect(incrElapsed).toBeLessThanOrEqual(fullElapsed * 1.1);});});++// What the incremental parser exists for is a cache, so these assert the cache:+// integers that come out the same on a loaded CI box as on an idle laptop,+// unlike the wall-clock budgets above.+describe('parseMarkdownIncremental cache', () => {+ it('builds a bounded number of blocks per chunk however long the document is', () => {+ const chunkSize = 50;+ const medians = [50, 200, 500].map(paragraphs =>+ median(blocksBuiltPerChunk(generateAIResponse(paragraphs), chunkSize)),+ );+ console.log(+ ` median blocks built per chunk (50/200/500 paragraphs): ${medians.join('/')}`,+ );+ // A chunk re-parses its unsettled tail, not the document — so the typical+ // chunk's cost is a constant, and a 10x longer document does not move it.+ expect(medians).toEqual([2, 2, 2]);+ });++ it('keeps the cache when source ranges are asked for', () => {+ const text = generateAIResponse(200);+ const chunkSize = 50;+ const without = sum(blocksBuiltPerChunk(text, chunkSize));+ const withRanges = sum(+ blocksBuiltPerChunk(text, chunkSize, {sourceRanges: true}),+ );+ console.log(+ ` blocks built streaming ${text.length} chars: ${without} without ranges, ${withRanges} with`,+ );+ // Stamping offsets must not cost a cache entry. An implementation that+ // tracked positions by invalidating the settled cache each chunk would+ // still pass any time budget generous enough not to flake; it fails here.+ expect(withRanges).toBe(without);+ });+});diff --git a/packages/core/src/Markdown/parser.test.ts b/packages/core/src/Markdown/parser.test.tsindex 85daa04fa7..2236923c2e 100644--- a/packages/core/src/Markdown/parser.test.ts+++ b/packages/core/src/Markdown/parser.test.ts@@ -1216,14 +1216,22 @@ describe('link reference definitions', () => {});describe('sourceRanges', () => {- const slice = (source: string, block: {range?: readonly [number, number]}) =>- block.range == null ? null : source.slice(block.range[0], block.range[1]);+ const slice = (source: string, block: BlockNode) =>+ block.range == null+ ? null+ : source.slice(block.range.start, block.range.end);it('is off by default', () => {const [block] = parseMarkdown('# Title');expect(block.range).toBeUndefined();});+ it('addresses a block with named start and end offsets', () => {+ const source = 'One.\n\nTwo.';+ const [, second] = parseMarkdown(source, {sourceRanges: true});+ expect(second.range).toEqual({start: 6, end: 10});+ });+it('gives every top-level block the source it came from', () => {const source = ['# Title',@@ -1324,7 +1332,11 @@ describe('sourceRanges', () => {const with_ = parseMarkdownIncremental(source, state, {sourceRanges: true,});- expect(with_.map(b => slice(source, b))).toEqual(['One.', 'Two.', 'Three.']);+ expect(with_.map(b => slice(source, b))).toEqual([+ 'One.',+ 'Two.',+ 'Three.',+ ]);const back = parseMarkdownIncremental(source, state);expect(back.every(b => b.range == null)).toBe(true);});diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.tsindex c11b57f7cc..e3f4efc9c4 100644--- a/packages/core/src/Markdown/parser.ts+++ b/packages/core/src/Markdown/parser.ts@@ -56,10 +56,15 @@ type BlockNodeKind =| {type: 'image'; src: string; alt: string};/**- * Character offsets `[start, end)` of a block in the source string handed to- * `parseMarkdown`. `end` excludes the block's trailing blank lines.+ * Where a block sits in the source string handed to `parseMarkdown`:+ * `source.slice(start, end)` is the block, and `end` excludes the block's+ * trailing blank lines.+ *+ * An object rather than a `[start, end]` tuple so a second way of addressing+ * the same block — line numbers, once a consumer needs them — can be added as+ * optional fields without breaking anyone.*/-export type SourceRange = readonly [start: number, end: number];+export type SourceRange = {readonly start: number; readonly end: number};export type ListItemNode = {checked?: boolean; children: BlockNode[]};export type TableCellNode = {children: InlineNode[]};@@ -1540,7 +1545,7 @@ function stampSourceRanges(// range that dropped it would slice to something that re-parses// differently.const end = lineStart(endLine) + lines[endLine].length;- blocks[i] = {...blocks[i], range: [lineStart(startLine), end]};+ blocks[i] = {...blocks[i], range: {start: lineStart(startLine), end}};}}@@ -1876,7 +1881,7 @@ function mergeSettledBlocks(// One list now, so one range: from where the first half started to// where the second half ended....(prevLast.range != null && deltaFirst.range != null- ? {range: [prevLast.range[0], deltaFirst.range[1]] as SourceRange}+ ? {range: {start: prevLast.range.start, end: deltaFirst.range.end}}: null),};return [...prev.slice(0, -1), merged, ...delta.slice(1)];