Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

refactor(markdown): SourceRange as an object, not a tuple (#5366)

Cindy Zhang5d351e42b7ce
diff --git a/.changeset/markdown-source-ranges.md b/.changeset/markdown-source-ranges.md
index 4b4c7cce04..bb70fb2fcb 100644
--- a/.changeset/markdown-source-ranges.md
+++ b/.changeset/markdown-source-ranges.md
@@ -5,9 +5,10 @@
[feat] Markdown: opt-in source ranges on parsed blocks
`parseMarkdown(source, {sourceRanges: true})` now gives every top-level block a
-`range` — the `[start, end)` offsets it occupies in the source that was passed
-in — so a consumer holding that source can slice the original markdown for a
-block instead of reconstructing it from the node (or from the rendered DOM).
+`range` — `{start, end}`, the character offsets it occupies in the source that
+was passed in, with `end` exclusive — so a consumer holding that source can
+`source.slice(range.start, range.end)` for a block instead of reconstructing it
+from the node (or from the rendered DOM).
Reconstruction is lossy in ways slicing is not: escapes, the exact emphasis and
fence characters, heading depth beyond the clamp, and alignment all survive a
slice unchanged.
diff --git a/packages/core/src/Markdown/parser.perf.test.ts b/packages/core/src/Markdown/parser.perf.test.ts
index 273cb310fd..0078feb658 100644
--- a/packages/core/src/Markdown/parser.perf.test.ts
+++ b/packages/core/src/Markdown/parser.perf.test.ts
@@ -6,7 +6,7 @@ import {
parseMarkdownIncremental,
createIncrementalState,
} from './parser';
-import type {BlockNode} from './parser';
+import type {BlockNode, ParseOptions} from './parser';
function generateAIResponse(paragraphs: number): string {
const sections: string[] = [];
@@ -69,6 +69,42 @@ function simulateStreamingIncremental(
return parseMarkdownIncremental(fullText, state);
}
+/**
+ * How many blocks each chunk of a streamed document had to build.
+ *
+ * A block the parser kept in its settled cache is handed back as the very same
+ * object, so a returned block that is not one of the previous chunk's is one
+ * this chunk built. That makes the cache countable without reaching into the
+ * parser's internals.
+ */
+function blocksBuiltPerChunk(
+ fullText: string,
+ chunkSize: number,
+ options?: ParseOptions,
+): number[] {
+ const state = createIncrementalState();
+ let previous: ReadonlySet<BlockNode> = new Set();
+ const built: number[] = [];
+ const step = (text: string) => {
+ const blocks =
+ options == null
+ ? parseMarkdownIncremental(text, state)
+ : parseMarkdownIncremental(text, state, options);
+ built.push(blocks.reduce((n, b) => (previous.has(b) ? n : n + 1), 0));
+ previous = new Set(blocks);
+ };
+ for (let i = chunkSize; i <= fullText.length; i += chunkSize) {
+ step(fullText.slice(0, i));
+ }
+ step(fullText);
+ return built;
+}
+
+const median = (values: number[]): number =>
+ [...values].sort((a, b) => a - b)[Math.floor((values.length - 1) / 2)];
+
+const sum = (values: number[]): number => values.reduce((a, b) => a + b, 0);
+
// A single wall-clock measurement is flaky when the whole suite runs in
// parallel forks: scheduler contention can inflate a millisecond-scale
// sample well past a tight budget (observed 30.9ms against the 20ms Small
@@ -204,3 +240,37 @@ describe('parseMarkdown performance', () => {
expect(incrElapsed).toBeLessThanOrEqual(fullElapsed * 1.1);
});
});
+
+// What the incremental parser exists for is a cache, so these assert the cache:
+// integers that come out the same on a loaded CI box as on an idle laptop,
+// unlike the wall-clock budgets above.
+describe('parseMarkdownIncremental cache', () => {
+ it('builds a bounded number of blocks per chunk however long the document is', () => {
+ const chunkSize = 50;
+ const medians = [50, 200, 500].map(paragraphs =>
+ median(blocksBuiltPerChunk(generateAIResponse(paragraphs), chunkSize)),
+ );
+ console.log(
+ ` median blocks built per chunk (50/200/500 paragraphs): ${medians.join('/')}`,
+ );
+ // A chunk re-parses its unsettled tail, not the document — so the typical
+ // chunk's cost is a constant, and a 10x longer document does not move it.
+ expect(medians).toEqual([2, 2, 2]);
+ });
+
+ it('keeps the cache when source ranges are asked for', () => {
+ const text = generateAIResponse(200);
+ const chunkSize = 50;
+ const without = sum(blocksBuiltPerChunk(text, chunkSize));
+ const withRanges = sum(
+ blocksBuiltPerChunk(text, chunkSize, {sourceRanges: true}),
+ );
+ console.log(
+ ` blocks built streaming ${text.length} chars: ${without} without ranges, ${withRanges} with`,
+ );
+ // Stamping offsets must not cost a cache entry. An implementation that
+ // tracked positions by invalidating the settled cache each chunk would
+ // still pass any time budget generous enough not to flake; it fails here.
+ expect(withRanges).toBe(without);
+ });
+});
diff --git a/packages/core/src/Markdown/parser.test.ts b/packages/core/src/Markdown/parser.test.ts
index 85daa04fa7..2236923c2e 100644
--- a/packages/core/src/Markdown/parser.test.ts
+++ b/packages/core/src/Markdown/parser.test.ts
@@ -1216,14 +1216,22 @@ describe('link reference definitions', () => {
});
describe('sourceRanges', () => {
- const slice = (source: string, block: {range?: readonly [number, number]}) =>
- block.range == null ? null : source.slice(block.range[0], block.range[1]);
+ const slice = (source: string, block: BlockNode) =>
+ block.range == null
+ ? null
+ : source.slice(block.range.start, block.range.end);
it('is off by default', () => {
const [block] = parseMarkdown('# Title');
expect(block.range).toBeUndefined();
});
+ it('addresses a block with named start and end offsets', () => {
+ const source = 'One.\n\nTwo.';
+ const [, second] = parseMarkdown(source, {sourceRanges: true});
+ expect(second.range).toEqual({start: 6, end: 10});
+ });
+
it('gives every top-level block the source it came from', () => {
const source = [
'# Title',
@@ -1324,7 +1332,11 @@ describe('sourceRanges', () => {
const with_ = parseMarkdownIncremental(source, state, {
sourceRanges: true,
});
- expect(with_.map(b => slice(source, b))).toEqual(['One.', 'Two.', 'Three.']);
+ expect(with_.map(b => slice(source, b))).toEqual([
+ 'One.',
+ 'Two.',
+ 'Three.',
+ ]);
const back = parseMarkdownIncremental(source, state);
expect(back.every(b => b.range == null)).toBe(true);
});
diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.ts
index c11b57f7cc..e3f4efc9c4 100644
--- a/packages/core/src/Markdown/parser.ts
+++ b/packages/core/src/Markdown/parser.ts
@@ -56,10 +56,15 @@ type BlockNodeKind =
| {type: 'image'; src: string; alt: string};
/**
- * Character offsets `[start, end)` of a block in the source string handed to
- * `parseMarkdown`. `end` excludes the block's trailing blank lines.
+ * Where a block sits in the source string handed to `parseMarkdown`:
+ * `source.slice(start, end)` is the block, and `end` excludes the block's
+ * trailing blank lines.
+ *
+ * An object rather than a `[start, end]` tuple so a second way of addressing
+ * the same block — line numbers, once a consumer needs them — can be added as
+ * optional fields without breaking anyone.
*/
-export type SourceRange = readonly [start: number, end: number];
+export type SourceRange = {readonly start: number; readonly end: number};
export type ListItemNode = {checked?: boolean; children: BlockNode[]};
export type TableCellNode = {children: InlineNode[]};
@@ -1540,7 +1545,7 @@ function stampSourceRanges(
// range that dropped it would slice to something that re-parses
// differently.
const end = lineStart(endLine) + lines[endLine].length;
- blocks[i] = {...blocks[i], range: [lineStart(startLine), end]};
+ blocks[i] = {...blocks[i], range: {start: lineStart(startLine), end}};
}
}
@@ -1876,7 +1881,7 @@ function mergeSettledBlocks(
// One list now, so one range: from where the first half started to
// where the second half ended.
...(prevLast.range != null && deltaFirst.range != null
- ? {range: [prevLast.range[0], deltaFirst.range[1]] as SourceRange}
+ ? {range: {start: prevLast.range.start, end: deltaFirst.range.end}}
: null),
};
return [...prev.slice(0, -1), merged, ...delta.slice(1)];