Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

feat(Markdown): opt-in source-line stamps with markdownSourceLinesPlugin (#7286)

Cindy Zhange8160da030b2
diff --git a/.changeset/markdown-source-lines.md b/.changeset/markdown-source-lines.md
new file mode 100644
index 0000000000..0ff9841b8c
--- /dev/null
+++ b/.changeset/markdown-source-lines.md
@@ -0,0 +1,7 @@
+---
+'@astryxdesign/core': patch
+---
+
+[feat] Add `markdownSourceLinesPlugin` to `@astryxdesign/core/Markdown/plugins`. With it installed, every block `Markdown` renders, nested blocks included, carries `data-source-line` and `data-source-line-end`: the 1-based, inclusive lines it came from. That lets a host map a selection, search hit, or comment back to the exact Markdown lines. `components` block renderers receive a matching `sourceLines` prop, and plugin renderers read lines from `node.position`. The lines come from the parse `Markdown` already runs, and without the plugin nothing changes.
+
+@cixzhang
diff --git a/packages/core/src/Markdown/Markdown.doc.mjs b/packages/core/src/Markdown/Markdown.doc.mjs
index 6d76b35579..1133da331e 100644
--- a/packages/core/src/Markdown/Markdown.doc.mjs
+++ b/packages/core/src/Markdown/Markdown.doc.mjs
@@ -267,6 +267,11 @@ export const docs = {
description:
'Add markdownSoftBreaksPlugin when single line endings are meaningful. It matches remark-breaks for supported Markdown, including multiline link labels, while code and other opaque content stay unchanged.',
},
+ {
+ guidance: true,
+ description:
+ 'Add markdownSourceLinesPlugin when a feature maps the rendered document back to Markdown source lines (find, highlights, review comments). Every block Markdown renders, nested blocks included, then carries data-source-line and data-source-line-end (1-based, inclusive); components block renderers receive sourceLines to place on their own element, and plugin renderers read node.position lines. Inline elements, table rows, and cells carry none. Without the plugin no line work runs.',
+ },
{
guidance: true,
description:
@@ -389,6 +394,18 @@ import {markdownSoftBreaksPlugin} from '@astryxdesign/core/Markdown/plugins';
<Markdown plugins={[markdownSoftBreaksPlugin]}>
{'First line\\nSecond line'}
</Markdown>;
+`,
+ },
+ {
+ label: 'Source-line stamps',
+ code: `
+import {Markdown} from '@astryxdesign/core/Markdown';
+import {markdownSourceLinesPlugin} from '@astryxdesign/core/Markdown/plugins';
+
+// Each rendered block carries data-source-line / data-source-line-end.
+<Markdown plugins={[markdownSourceLinesPlugin]}>
+ {'# Title\\n\\n- one\\n- two'}
+</Markdown>;
`,
},
{
@@ -787,6 +804,11 @@ export const docsZh = {
description:
'Add markdownSoftBreaksPlugin when single line endings are meaningful. It matches remark-breaks for supported Markdown, including multiline link labels, while code and other opaque content stay unchanged.',
},
+ {
+ guidance: true,
+ description:
+ 'Add markdownSourceLinesPlugin when a feature maps the rendered document back to Markdown source lines (find, highlights, review comments). Every block Markdown renders, nested blocks included, then carries data-source-line and data-source-line-end (1-based, inclusive); components block renderers receive sourceLines to place on their own element, and plugin renderers read node.position lines. Inline elements, table rows, and cells carry none. Without the plugin no line work runs.',
+ },
{
guidance: true,
description:
@@ -889,6 +911,11 @@ export const docsDense = {
description:
'Add markdownSoftBreaksPlugin when single line endings are meaningful. It matches remark-breaks for supported Markdown, including multiline link labels, while code and other opaque content stay unchanged.',
},
+ {
+ guidance: true,
+ description:
+ 'Add markdownSourceLinesPlugin when a feature maps the rendered document back to Markdown source lines (find, highlights, review comments). Every block Markdown renders, nested blocks included, then carries data-source-line and data-source-line-end (1-based, inclusive); components block renderers receive sourceLines to place on their own element, and plugin renderers read node.position lines. Inline elements, table rows, and cells carry none. Without the plugin no line work runs.',
+ },
{
guidance: true,
description:
diff --git a/packages/core/src/Markdown/Markdown.public.test.ts b/packages/core/src/Markdown/Markdown.public.test.ts
index da9a5c154c..fba66b9e0f 100644
--- a/packages/core/src/Markdown/Markdown.public.test.ts
+++ b/packages/core/src/Markdown/Markdown.public.test.ts
@@ -24,6 +24,7 @@ import {
createMarkdownTextTransform,
isMarkdownExtensionNode,
markdownSoftBreaksPlugin,
+ markdownSourceLinesPlugin,
} from './plugins';
import type {
MarkdownExtensionNode,
@@ -227,6 +228,9 @@ describe('Markdown public parser types', () => {
expectTypeOf(markdownSoftBreaksPlugin).toEqualTypeOf<
MarkdownPluginEntry<never>
>();
+ expectTypeOf(markdownSourceLinesPlugin).toEqualTypeOf<
+ MarkdownPluginEntry<never>
+ >();
expectTypeOf(isMarkdownExtensionNode).toBeFunction();
function compileOnlyPluginGuards() {
diff --git a/packages/core/src/Markdown/Markdown.test.tsx b/packages/core/src/Markdown/Markdown.test.tsx
index 79dfb8cb3b..5fa385afef 100644
--- a/packages/core/src/Markdown/Markdown.test.tsx
+++ b/packages/core/src/Markdown/Markdown.test.tsx
@@ -13,7 +13,11 @@ import {render, screen, fireEvent} from '@testing-library/react';
import type {ComponentProps, ReactNode} from 'react';
import * as stylex from '@stylexjs/stylex';
import {Markdown} from './Markdown';
-import type {MarkdownComponents, MarkdownInlinePlugin} from './Markdown';
+import type {
+ MarkdownComponents,
+ MarkdownInlinePlugin,
+ MarkdownSourceLines,
+} from './Markdown';
import type {ParseOptions} from './index';
import {stubMatchMedia} from '../__tests__/stubMatchMedia';
import {parseOutlineFromMarkdown} from '../Outline/parseOutlineFromMarkdown';
@@ -977,6 +981,7 @@ describe('Markdown', () => {
expectTypeOf<MathRendererProps>().toEqualTypeOf<{
value: string;
display: 'inline' | 'block';
+ sourceLines?: MarkdownSourceLines;
}>();
expectTypeOf<ParseOptions>().toMatchTypeOf<{math?: boolean}>();
});
diff --git a/packages/core/src/Markdown/Markdown.tsx b/packages/core/src/Markdown/Markdown.tsx
index 927c2b97cf..68888d209d 100644
--- a/packages/core/src/Markdown/Markdown.tsx
+++ b/packages/core/src/Markdown/Markdown.tsx
@@ -60,6 +60,7 @@ import {getMarkdownAstLegacyCodeLanguage, markdownAstText} from './ast';
import type {
MarkdownAstBlockContent,
MarkdownAstPhrasingContent,
+ MarkdownAstPosition,
MarkdownAstTable,
} from './ast';
import {
@@ -146,12 +147,25 @@ export interface MarkdownInlinePlugin {
*/
export type MarkdownSource = CitationSource;
+/**
+ * A block's 1-based, inclusive source lines, given to `components` block
+ * renderers while `markdownSourceLinesPlugin` is installed.
+ */
+export interface MarkdownSourceLines {
+ readonly start: number;
+ readonly end: number;
+}
+
export interface MarkdownComponents {
/**
* Renders every fenced code block that no semantic-fence plugin claims, and
* a claimed fence whose plugin renderer declines, throws, or suspends.
*/
- code?: React.ComponentType<{code: string; language?: string}>;
+ code?: React.ComponentType<{
+ code: string;
+ language?: string;
+ sourceLines?: MarkdownSourceLines;
+ }>;
inlineCode?: React.ComponentType<{children: string}>;
/**
* Renders opt-in `$…$` and `$$…$$` math. Supplying this renderer enables
@@ -160,6 +174,8 @@ export interface MarkdownComponents {
math?: React.ComponentType<{
value: string;
display: 'inline' | 'block';
+ /** Present for block math only, while source lines are recorded. */
+ sourceLines?: MarkdownSourceLines;
}>;
citation?: React.ComponentType<{
source: CitationSource;
@@ -177,11 +193,46 @@ export interface MarkdownComponents {
* Custom renderers own their output and should apply the id they receive.
*/
id?: string;
+ sourceLines?: MarkdownSourceLines;
+ }>;
+ paragraph?: React.ComponentType<{
+ children: React.ReactNode;
+ sourceLines?: MarkdownSourceLines;
+ }>;
+ image?: React.ComponentType<{
+ src: string;
+ alt: string;
+ sourceLines?: MarkdownSourceLines;
+ }>;
+ blockquote?: React.ComponentType<{
+ children: React.ReactNode;
+ sourceLines?: MarkdownSourceLines;
}>;
- paragraph?: React.ComponentType<{children: React.ReactNode}>;
- image?: React.ComponentType<{src: string; alt: string}>;
- blockquote?: React.ComponentType<{children: React.ReactNode}>;
- hr?: React.ComponentType<object>;
+ hr?: React.ComponentType<{sourceLines?: MarkdownSourceLines}>;
+}
+
+/** A block node's recorded source lines, if the parse recorded them. */
+function blockSourceLines(node: {
+ readonly position?: MarkdownAstPosition;
+}): MarkdownSourceLines | undefined {
+ const start = node.position?.start.line;
+ const end = node.position?.end.line;
+ return start == null || end == null ? undefined : {start, end};
+}
+
+/** The stamps a default block element carries for its source lines. */
+function sourceLineAttributes(lines: MarkdownSourceLines | undefined):
+ | {
+ readonly 'data-source-line': string;
+ readonly 'data-source-line-end': string;
+ }
+ | undefined {
+ return lines == null
+ ? undefined
+ : {
+ 'data-source-line': String(lines.start),
+ 'data-source-line-end': String(lines.end),
+ };
}
export interface MarkdownProps<
@@ -1275,6 +1326,12 @@ function renderBlock(
const spacing = getElementSpacing(node, density);
const isFirst = index === 0;
const isLast = index === blockCount - 1;
+ // Source lines exist only while the source-lines plugin is installed
+ // (module:Markdown/sourceLines); otherwise no block reads them.
+ const recordsSourceLines = preparedPlugins?.sourceLines === true;
+ const sourceLines = recordsSourceLines ? blockSourceLines(node) : undefined;
+ const lineStamps = sourceLineAttributes(sourceLines);
+ const lineProp = sourceLines == null ? null : {sourceLines};
switch (node.type) {
case 'heading': {
@@ -1299,7 +1356,7 @@ function renderBlock(
const HeadingComp = components?.heading;
if (HeadingComp) {
return (
- <HeadingComp key={index} level={level} id={headingId}>
+ <HeadingComp key={index} level={level} id={headingId} {...lineProp}>
{headingChildren}
</HeadingComp>
);
@@ -1325,7 +1382,8 @@ function renderBlock(
isFirst && styles.noMarginBlockStart,
isLast && styles.noMarginBlockEnd,
),
- )}>
+ )}
+ {...lineStamps}>
{headingChildren}
</Tag>
);
@@ -1353,7 +1411,8 @@ function renderBlock(
headingLinksHeadingStyle,
headingStyles[level],
),
- )}>
+ )}
+ {...lineStamps}>
{headingChildren}
</Tag>
</HeadingLinksRenderer>
@@ -1375,7 +1434,11 @@ function renderBlock(
);
const ParagraphComp = components?.paragraph;
if (ParagraphComp) {
- return <ParagraphComp key={index}>{paraChildren}</ParagraphComp>;
+ return (
+ <ParagraphComp key={index} {...lineProp}>
+ {paraChildren}
+ </ParagraphComp>
+ );
}
// Markdown paragraphs render as <div>, not <p>: inline content can
// include block-level nodes (images, custom inline components), and a
@@ -1403,7 +1466,8 @@ function renderBlock(
isFirst && styles.noMarginBlockStart,
isLast && styles.noMarginBlockEnd,
),
- )}>
+ )}
+ {...lineStamps}>
{paraChildren}
</div>
);
@@ -1418,7 +1482,12 @@ function renderBlock(
// when the claiming renderer is absent, declines, throws, or suspends.
const CodeBlockComp = components?.code;
const fallback = CodeBlockComp ? (
- <CodeBlockComp key={index} code={node.value} language={language} />
+ <CodeBlockComp
+ key={index}
+ code={node.value}
+ language={language}
+ {...lineProp}
+ />
) : (
<div
key={index}
@@ -1430,7 +1499,8 @@ function renderBlock(
isFirst && styles.noMarginBlockStart,
isLast && styles.noMarginBlockEnd,
),
- )}>
+ )}
+ {...lineStamps}>
<CodeBlock
code={node.value}
language={language}
@@ -1485,7 +1555,14 @@ function renderBlock(
</div>
);
}
- return <MathComp key={index} value={node.value} display="block" />;
+ return (
+ <MathComp
+ key={index}
+ value={node.value}
+ display="block"
+ {...lineProp}
+ />
+ );
}
case 'blockquote': {
const BlockquoteComp = components?.blockquote;
@@ -1511,12 +1588,17 @@ function renderBlock(
listDepth,
),
);
- return <BlockquoteComp key={index}>{bqC}</BlockquoteComp>;
+ return (
+ <BlockquoteComp key={index} {...lineProp}>
+ {bqC}
+ </BlockquoteComp>
+ );
}
return (
<Blockquote
key={index}
{...themeProps('markdown-blockquote', {density})}
+ {...lineStamps}
xstyle={[
spacing,
contentWidthValue != null
@@ -1637,6 +1719,9 @@ function renderBlock(
key={i}
value={`task-${i}`}
label={label}
+ {...(recordsSourceLines
+ ? sourceLineAttributes(blockSourceLines(item))
+ : undefined)}
/>
);
})}
@@ -1751,7 +1836,12 @@ function renderBlock(
]
}
task={task}>
- <ListItem label={label} />
+ <ListItem
+ label={label}
+ {...(recordsSourceLines
+ ? sourceLineAttributes(blockSourceLines(item))
+ : undefined)}
+ />
</ListMarkerScope>
);
})}
@@ -1791,7 +1881,8 @@ function renderBlock(
isFirst && styles.noMarginBlockStart,
isLast && styles.noMarginBlockEnd,
),
- )}>
+ )}
+ {...lineStamps}>
<Table dividers="rows" textOverflow="wrap">
<TableHeader>
<TableRow>
@@ -1892,7 +1983,7 @@ function renderBlock(
case 'thematicBreak': {
const HrComp = components?.hr;
if (HrComp) {
- return <HrComp key={index} />;
+ return <HrComp key={index} {...lineProp} />;
}
return (
<hr
@@ -1906,6 +1997,7 @@ function renderBlock(
isLast && styles.noMarginBlockEnd,
),
)}
+ {...lineStamps}
/>
);
}
@@ -1922,14 +2014,17 @@ function renderBlock(
isFirst && styles.noMarginBlockStart,
isLast && styles.noMarginBlockEnd,
),
- )}>
+ )}
+ {...lineStamps}>
[{node.alt}]
</div>
);
}
const ImageComp = components?.image;
if (ImageComp) {
- return <ImageComp key={index} src={safeSrc} alt={node.alt} />;
+ return (
+ <ImageComp key={index} src={safeSrc} alt={node.alt} {...lineProp} />
+ );
}
return (
<div
@@ -1941,7 +2036,8 @@ function renderBlock(
isFirst && styles.noMarginBlockStart,
isLast && styles.noMarginBlockEnd,
),
- )}>
+ )}
+ {...lineStamps}>
<img src={safeSrc} alt={node.alt} {...stylex.props(styles.image)} />
</div>
);
diff --git a/packages/core/src/Markdown/index.ts b/packages/core/src/Markdown/index.ts
index 0be59d24b3..f9c90c1485 100644
--- a/packages/core/src/Markdown/index.ts
+++ b/packages/core/src/Markdown/index.ts
@@ -47,6 +47,7 @@ export type {
MarkdownSource,
MarkdownComponents,
MarkdownInlinePlugin,
+ MarkdownSourceLines,
} from './Markdown';
export {
diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.ts
index dc3cefc218..dc9886c9da 100644
--- a/packages/core/src/Markdown/parser.ts
+++ b/packages/core/src/Markdown/parser.ts
@@ -582,6 +582,23 @@ type ResolvedOptions = {
};
/** Whether this parse runs inside a lazy-continuation probe. */
readonly probing?: boolean;
+ /**
+ * Whether blocks record their source lines (`module:Markdown/sourceLines`).
+ * Set only by the source-lines plugin; otherwise no line work runs.
+ */
+ readonly sourceLines?: boolean;
+ /**
+ * 0-based document line of this parse's first input line. Internal only —
+ * the incremental parser parses slices whose lines must come out absolute.
+ */
+ readonly baseLine?: number;
+ /**
+ * 0-based document line of each input line of a nested parse. Internal only
+ * — a list item's or a blockquote's content is reassembled one line per
+ * source line, so its lines map back to the document through this table.
+ * A nested parse without it records no lines.
+ */
+ readonly lineOrigin?: ReadonlyArray<number>;
};
/**
@@ -621,6 +638,9 @@ function makeResolvedOptions(
blockDepth: fields.blockDepth ?? 0,
nestedParses: fields.nestedParses,
probing: fields.probing,
+ sourceLines: fields.sourceLines,
+ baseLine: fields.baseLine,
+ lineOrigin: fields.lineOrigin,
};
}
@@ -648,16 +668,18 @@ function resolveOptions(
});
}
const opts = arg as RuntimeParseOptions;
+ const plugins =
+ opts.plugins != null && opts.plugins.length > 0
+ ? prepareMarkdownPlugins(opts.plugins)
+ : undefined;
return makeResolvedOptions({
sourceIds: opts.sourceIds,
autolink: opts.autolink,
math: opts.math === true ? true : undefined,
sourceRanges: opts.sourceRanges,
- plugins:
- opts.plugins != null && opts.plugins.length > 0
- ? prepareMarkdownPlugins(opts.plugins)
- : undefined,
+ plugins,
isFinal: incremental ? opts.isFinal === true : true,
+ sourceLines: plugins?.sourceLines === true ? true : undefined,
});
}
@@ -3082,9 +3104,23 @@ function nested(opts: ResolvedOptions): ResolvedOptions {
sourceRanges: false,
astPositions: false,
blockDepth: opts.blockDepth + 1,
+ baseLine: undefined,
+ lineOrigin: undefined,
};
}
+/**
+ * The same options for content parsed out of an enclosing block whose input
+ * lines came from the document lines `origin` names. Without source lines,
+ * exactly `nested(opts)`.
+ */
+function nestedAt(
+ opts: ResolvedOptions,
+ origin: ReadonlyArray<number> | null,
+): ResolvedOptions {
+ return origin == null ? nested(opts) : {...nested(opts), lineOrigin: origin};
+}
+
function blockExtensionColumn(line: string): number | null {
const indentation = line.length - line.trimStart().length;
return indentation <= 3 ? indentation : null;
@@ -3324,6 +3360,7 @@ function parseList(
ordered: boolean,
opts: ResolvedOptions,
interruptsLazyContinuation: (lineIndex: number) => boolean,
+ documentLineOf: ((line: number) => number) | null,
): {node: MarkdownAstBlockContent<RuntimeExtensionNode>; nextIndex: number} {
const items: MarkdownAstListItem<RuntimeExtensionNode>[] = [];
const baseIndent = getIndent(lines[startIndex]);
@@ -3351,6 +3388,7 @@ function parseList(
const startsItem = (line: string) =>
itemPattern.test(line) && !isHorizontalRule(line);
while (index < lines.length && startsItem(lines[index])) {
+ const itemStart = index;
const content = ordered
? lines[index].replace(new RegExp(`^ *\\d+${escDelim} `), '')
: lines[index].replace(/^ *[-*+] /, '');
@@ -3450,11 +3488,30 @@ function parseList(
const source = listItemSource(itemText, subLines, lazyLineIndexes);
- items.push({
- type: 'listItem',
- checked,
- children: parseMarkdownImpl(source, nested(opts)),
- });
+ if (documentLineOf == null) {
+ items.push({
+ type: 'listItem',
+ checked,
+ children: parseMarkdownImpl(source, nested(opts)),
+ });
+ } else {
+ // The item's source holds one line per line it consumed, from its
+ // marker line to its last content line, so its lines map back
+ // one-to-one; blank lines are consumed only before more content.
+ const origin: number[] = [];
+ for (let line = itemStart; line < index; line++) {
+ origin.push(documentLineOf(line));
+ }
+ items.push({
+ type: 'listItem',
+ checked,
+ children: parseMarkdownImpl(source, nestedAt(opts, origin)),
+ position: {
+ start: {line: origin[0] + 1},
+ end: {line: origin[origin.length - 1] + 1},
+ },
+ });
+ }
// CommonMark loose list: blank line(s) between items of the same style
// and indent still form one list. Skip the blanks and continue if the
@@ -3649,12 +3706,15 @@ function parseMarkdownImpl(
// The line each block started on, parallel to `blocks`. Only collected when
// ranges were asked for; a block's end is resolved after the loop, since the
// branch that produced it has already moved `index` past whatever it read.
+ // A parse records lines only when it can place them in the document: at
+ // the top level, or nested with a table of its lines' origins.
+ const documentLineOf = sourceLineResolver(opts, lineMap);
const blockStartLines: number[] | null =
- opts.astPositions === true ? [] : null;
+ opts.astPositions === true || documentLineOf != null ? [] : null;
// Set only by a block that consumes blank lines as content, where the
// positional end derivation would trim them away.
const blockEndLines: (number | undefined)[] | null =
- opts.astPositions === true ? [] : null;
+ blockStartLines != null ? [] : null;
let blockStartLine = 0;
const pushBlock = (
node: MarkdownAstBlockContent<RuntimeExtensionNode>,
@@ -3836,7 +3896,18 @@ function parseMarkdownImpl(
}
pushBlock({
type: 'blockquote',
- children: parseMarkdownImpl(quoteLines.join('\n'), nested(opts)),
+ children: parseMarkdownImpl(
+ quoteLines.join('\n'),
+ nestedAt(
+ opts,
+ // Each quote line is one source line, from the quote's first on.
+ documentLineOf == null
+ ? null
+ : quoteLines.map((_, offset) =>
+ documentLineOf(blockStartLine + offset),
+ ),
+ ),
+ ),
});
continue;
}
@@ -3849,6 +3920,7 @@ function parseMarkdownImpl(
false,
opts,
interruptsWithBlockExtension,
+ documentLineOf,
);
pushBlock(listResult.node);
index = listResult.nextIndex;
@@ -3863,6 +3935,7 @@ function parseMarkdownImpl(
true,
opts,
interruptsWithBlockExtension,
+ documentLineOf,
);
pushBlock(listResult.node);
index = listResult.nextIndex;
@@ -3946,6 +4019,7 @@ function parseMarkdownImpl(
lineMap,
input,
opts,
+ documentLineOf,
);
}
const nestedParses = baseOpts.nestedParses?.map;
@@ -3972,13 +4046,17 @@ function stampSourceRanges(
lineMap: number[] | undefined,
input: string,
opts: ResolvedOptions,
+ documentLineOf: ((line: number) => number) | null,
): void {
+ const withOffsets = opts.astPositions === true;
const base = opts.baseOffset ?? 0;
// Offset of the first character of every line of the input.
const inputLineStarts = [0];
- for (let i = 0; i < input.length; i++) {
- if (input[i] === '\n') {
- inputLineStarts.push(i + 1);
+ if (withOffsets) {
+ for (let i = 0; i < input.length; i++) {
+ if (input[i] === '\n') {
+ inputLineStarts.push(i + 1);
+ }
}
}
// Stripping removes whole lines and never edits one, so a parsed line's
@@ -4001,15 +4079,60 @@ function stampSourceRanges(
// `\r` included, since the parser reads it as part of the line too and a
// range that dropped it would slice to something that re-parses
// differently.
- const end = lineStart(endLine) + lines[endLine].length;
- blocks[i] = withMarkerShapeOf(blocks[i], {
- ...blocks[i],
- position: {
+ // Each point is built as one literal so a parse without source lines
+ // produces exactly the `{offset}` points it always has.
+ let position: MarkdownAstPosition;
+ if (documentLineOf == null) {
+ position = {
start: {offset: lineStart(startLine)},
- end: {offset: end},
- },
- });
+ end: {offset: lineStart(endLine) + lines[endLine].length},
+ };
+ } else if (withOffsets) {
+ position = {
+ start: {
+ offset: lineStart(startLine),
+ line: documentLineOf(startLine) + 1,
+ },
+ end: {
+ offset: lineStart(endLine) + lines[endLine].length,
+ line: documentLineOf(endLine) + 1,
+ },
+ };
+ } else {
+ position = {
+ start: {line: documentLineOf(startLine) + 1},
+ end: {line: documentLineOf(endLine) + 1},
+ };
+ }
+ blocks[i] = withMarkerShapeOf(blocks[i], {...blocks[i], position});
+ }
+}
+
+/**
+ * Maps a line of a parse's block loop to its 0-based document line, or `null`
+ * when this parse records no lines. Link reference definitions are stripped
+ * before the loop, so a loop line goes through `lineMap` to its input line
+ * first; a nested parse then goes through `lineOrigin` to the document, and a
+ * top-level parse adds the line its slice starts on.
+ */
+function sourceLineResolver(
+ opts: ResolvedOptions,
+ lineMap: number[] | undefined,
+): ((line: number) => number) | null {
+ if (opts.sourceLines !== true) {
+ return null;
+ }
+ const inputLine = (line: number): number =>
+ lineMap != null ? lineMap[line] : line;
+ const origin = opts.lineOrigin;
+ if (origin != null) {
+ return line => origin[inputLine(line)];
+ }
+ if (opts.astPositions !== true) {
+ return null;
}
+ const baseLine = opts.baseLine ?? 0;
+ return line => baseLine + inputLine(line);
}
// ---------------------------------------------------------------------------
@@ -4070,6 +4193,11 @@ type IncrementalWork = {
type IncrementalCache = {
/** Character offset immediately after the immutable settled prefix. */
settledEnd: number;
+ /**
+ * Line breaks before `settledEnd`, kept only while source lines are
+ * recorded; `undefined` when unknown, counted once from the settled text.
+ */
+ settledEndLine: number | undefined;
/** Definitions whose complete block is in the settled prefix. */
settledLinkDefs: Map<string, string>;
/** Definitions still in the mutable tail on the preceding call. */
@@ -4098,6 +4226,7 @@ function makeIncrementalCache(
const {defs} = extractLinkDefinitions(state.settledText, state.math);
const cache: IncrementalCache = {
settledEnd: state.settledText.length,
+ settledEndLine: undefined,
settledLinkDefs: new Map(defs),
tailLinkDefs: new Map(),
linkDefs: defs,
@@ -4579,8 +4708,27 @@ function trimUnsettledStructural(text: string): string {
* document — so the slice's blocks report ranges into the whole document
* rather than into the slice.
*/
-function atOffset(opts: ResolvedOptions, offset: number): ResolvedOptions {
- return opts.astPositions === true ? {...opts, baseOffset: offset} : opts;
+function atOffset(
+ opts: ResolvedOptions,
+ offset: number,
+ line: number,
+): ResolvedOptions {
+ if (opts.astPositions !== true) {
+ return opts;
+ }
+ return opts.sourceLines === true
+ ? {...opts, baseOffset: offset, baseLine: line}
+ : {...opts, baseOffset: offset};
+}
+
+/** Line breaks in `text` from `start` to `end`. */
+function countLineBreaks(text: string, start = 0, end = text.length): number {
+ let count = 0;
+ for (let index = text.indexOf('\n', start); index !== -1 && index < end;) {
+ count++;
+ index = text.indexOf('\n', index + 1);
+ }
+ return count;
}
/**
@@ -4745,6 +4893,7 @@ function resetIncrementalCache(
state.pluginSyntaxIdentity = undefined;
state.math = undefined;
cache.settledEnd = 0;
+ cache.settledEndLine = 0;
cache.settledLinkDefs.clear();
cache.tailLinkDefs = new Map();
cache.linkDefs = new Map();
@@ -4901,6 +5050,11 @@ function parseMarkdownIncrementalAstBlocks(
// fence/boundary detection, definition collection, and block construction.
// An open fence simply keeps the suffix growing until its closing marker.
const oldSettledEnd = cache.settledEnd;
+ // The line the settled prefix ends on, only while lines are recorded.
+ const oldSettledLine =
+ opts.sourceLines === true
+ ? (cache.settledEndLine ??= countLineBreaks(state.settledText))
+ : 0;
const tailRaw = input.slice(oldSettledEnd);
const tailLines = tailRaw.split('\n');
const {boundary, openFence, openMath} = findSettledBoundary(
@@ -4960,6 +5114,10 @@ function parseMarkdownIncrementalAstBlocks(
state.settledUpTo++;
}
cache.settledEnd = nextSettledEnd;
+ cache.settledEndLine =
+ opts.sourceLines === true
+ ? oldSettledLine + countLineBreaks(settledDelta)
+ : undefined;
}
// Blank lines before the tail and whitespace after it carry nothing, but the
@@ -5000,7 +5158,7 @@ function parseMarkdownIncrementalAstBlocks(
} else if (settledDelta !== '') {
const deltaBlocks = parseMarkdownImpl(
settledDelta,
- atOffset(parseOpts, oldSettledEnd),
+ atOffset(parseOpts, oldSettledEnd, oldSettledLine),
);
const mergeIndex = cache.settledAstBlocks.length - 1;
const mergedList = appendSettledBlocks(cache.settledAstBlocks, deltaBlocks);
@@ -5037,7 +5195,14 @@ function parseMarkdownIncrementalAstBlocks(
? parseMarkdownImpl(
unsettledText,
unsettledStart >= 0
- ? atOffset(parseOpts, unsettledStart)
+ ? atOffset(
+ parseOpts,
+ unsettledStart,
+ opts.sourceLines === true
+ ? (cache.settledEndLine ?? 0) +
+ countLineBreaks(input, cache.settledEnd, unsettledStart)
+ : 0,
+ )
: nested(parseOpts),
)
: [];
diff --git a/packages/core/src/Markdown/plugins/index.ts b/packages/core/src/Markdown/plugins/index.ts
index de5aad7dc7..a7ec201140 100644
--- a/packages/core/src/Markdown/plugins/index.ts
+++ b/packages/core/src/Markdown/plugins/index.ts
@@ -33,6 +33,7 @@ export type {
} from './sourceDecoration';
export {createMarkdownFrontmatter} from './frontmatter';
export {markdownSoftBreaksPlugin} from './softBreaks';
+export {markdownSourceLinesPlugin} from './sourceLines';
export {createMarkdownHeadingLinks} from './headingLinks';
export type {MarkdownHeadingLinksOptions} from './headingLinks';
export type {
diff --git a/packages/core/src/Markdown/plugins/protocol.ts b/packages/core/src/Markdown/plugins/protocol.ts
index f9a003b2f5..0578b5a7e0 100644
--- a/packages/core/src/Markdown/plugins/protocol.ts
+++ b/packages/core/src/Markdown/plugins/protocol.ts
@@ -159,6 +159,13 @@ export interface MarkdownTransform<Node extends MarkdownExtensionNode = never> {
}
const markdownTransformClaim = Symbol('MarkdownTransformClaim');
+/**
+ * Definition key of the first-party source-lines plugin
+ * (`module:Markdown/sourceLines`). It is a parse option, not syntax: the
+ * plugin enters parse identity so the parser records block lines during the
+ * parse it already runs, and no other entry can carry it.
+ */
+const markdownSourceLinesDefinition = Symbol('MarkdownSourceLines');
const markdownTransformTrusted = Symbol('MarkdownTransformTrusted');
interface ClaimAwareMarkdownTransform<
@@ -449,6 +456,29 @@ export function createMarkdownPlugin(
}) as unknown as MarkdownPluginEntry;
}
+/**
+ * @internal Creates the first-party source-lines entry. Its transform is a
+ * trusted identity that claims no source, so the transform pipeline skips it;
+ * the entry's only effect is the parse option the definition key carries.
+ */
+export function createMarkdownSourceLinesEntry(
+ name: string,
+): MarkdownPluginEntry<never> {
+ const transform = markMarkdownTransformClaim(
+ markMarkdownTransformTrusted(document => document),
+ () => false,
+ );
+ const definition: MarkdownTransformPluginDefinition<string, never> & {
+ readonly [markdownSourceLinesDefinition]: true;
+ } = {
+ name,
+ apiVersion: 1,
+ transform,
+ [markdownSourceLinesDefinition]: true,
+ };
+ return createMarkdownPlugin(definition);
+}
+
export interface PreparedSyntaxContribution {
readonly pluginName: string;
readonly display: 'inline' | 'block';
@@ -484,6 +514,8 @@ export interface PreparedMarkdownPlugins {
readonly hasUntrustedTransform: boolean;
readonly syntaxIdentity: string;
readonly renderers: ReadonlyMap<string, PreparedRenderer>;
+ /** Whether the source-lines plugin is installed (`module:Markdown/sourceLines`). */
+ readonly sourceLines: boolean;
}
const preparedPluginLists = new WeakMap<
@@ -542,6 +574,7 @@ export function prepareMarkdownPlugins(
const syntaxIdentity: string[] = [];
const entries: InternalMarkdownPluginEntry[] = [];
const syntaxEntries: MarkdownPluginEntry[] = [];
+ let sourceLines = false;
for (const publicEntry of plugins) {
const definition = getMarkdownPluginDefinition(publicEntry);
@@ -573,6 +606,17 @@ export function prepareMarkdownPlugins(
});
}
}
+ if (
+ (definition as {readonly [markdownSourceLinesDefinition]?: true})[
+ markdownSourceLinesDefinition
+ ] === true
+ ) {
+ sourceLines = true;
+ syntaxEntries.push(syntaxOnlyEntry(publicEntry));
+ syntaxIdentity.push(
+ `${definition.name}\0${definition.apiVersion}\0source-lines`,
+ );
+ }
if ('syntax' in definition && definition.syntax != null) {
syntaxEntries.push(syntaxOnlyEntry(publicEntry));
syntaxIdentity.push(
@@ -614,6 +658,7 @@ export function prepareMarkdownPlugins(
hasUntrustedTransform: transforms.some(entry => !entry.trusted),
syntaxIdentity: syntaxIdentity.join('\u0001'),
renderers,
+ sourceLines,
};
preparedPluginLists.set(plugins, prepared);
return prepared;
diff --git a/packages/core/src/Markdown/plugins/semanticFence.ts b/packages/core/src/Markdown/plugins/semanticFence.ts
index b39dc89ae7..460883a55b 100644
--- a/packages/core/src/Markdown/plugins/semanticFence.ts
+++ b/packages/core/src/Markdown/plugins/semanticFence.ts
@@ -11,6 +11,7 @@ import type {
MarkdownAstBlockContent,
MarkdownAstCode,
MarkdownAstListItem,
+ MarkdownAstPosition,
} from '../ast';
import {
getMarkdownHelperOwnership,
@@ -153,6 +154,7 @@ function createProposal(
>,
pluginName: string,
hasRenderer: (nodeName: string) => boolean,
+ sourceLines: MarkdownAstPosition | undefined,
): MarkdownFenceProposal {
if (
node == null ||
@@ -189,6 +191,9 @@ function createProposal(
name: node.name,
display: 'block' as const,
data,
+ // The fence's Core-authored source lines, recorded only while the
+ // source-lines plugin is installed (module:Markdown/sourceLines FR5).
+ ...(sourceLines == null ? null : {position: sourceLines}),
}),
});
}
@@ -254,7 +259,12 @@ function annotateCode(
Object.defineProperty(annotated, markdownFenceProposal, {
configurable: false,
enumerable: true,
- value: createProposal(proposalNode, pluginName, hasRenderer),
+ value: createProposal(
+ proposalNode,
+ pluginName,
+ hasRenderer,
+ node.position?.start.line == null ? undefined : node.position,
+ ),
writable: false,
});
return annotated;
diff --git a/packages/core/src/Markdown/plugins/sourceLines.test.tsx b/packages/core/src/Markdown/plugins/sourceLines.test.tsx
new file mode 100644
index 0000000000..6ef4823438
--- /dev/null
+++ b/packages/core/src/Markdown/plugins/sourceLines.test.tsx
@@ -0,0 +1,594 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file sourceLines.test.tsx
+ * @input markdownSourceLinesPlugin with Markdown, the canonical parsers, and renderers
+ * @output Regression coverage for module:Markdown/sourceLines
+ * @position Focused acceptance tests for opt-in source-line stamps
+ */
+
+import {render, screen} from '@testing-library/react';
+import {renderToStaticMarkup} from 'react-dom/server';
+import {beforeEach, describe, expect, it, vi} from 'vitest';
+import {Markdown} from '../Markdown';
+import type {MarkdownComponents, MarkdownSourceLines} from '../Markdown';
+import {
+ createIncrementalState,
+ parseMarkdownAst,
+ parseMarkdownAstIncremental,
+} from '../parser';
+import {createMarkdownFenceTransform} from './semanticFence';
+import {createMarkdownFrontmatter} from './frontmatter';
+import {createMarkdownPlugin, type MarkdownExtensionNode} from './protocol';
+import {markdownSourceLinesPlugin} from './sourceLines';
+import type * as ParserModule from '../parser';
+
+const parseCalls = vi.hoisted(() => ({full: 0, incremental: 0}));
+
+vi.mock('../parser', async importOriginal => {
+ const actual = await importOriginal<typeof ParserModule>();
+ return {
+ ...actual,
+ parseMarkdownAst: ((
+ ...args: Parameters<typeof actual.parseMarkdownAst>
+ ) => {
+ parseCalls.full++;
+ return actual.parseMarkdownAst(...args);
+ }) as typeof actual.parseMarkdownAst,
+ parseMarkdownAstIncremental: ((
+ ...args: Parameters<typeof actual.parseMarkdownAstIncremental>
+ ) => {
+ parseCalls.incremental++;
+ return actual.parseMarkdownAstIncremental(...args);
+ }) as typeof actual.parseMarkdownAstIncremental,
+ };
+});
+
+const SOURCE = [
+ '# Title', // 1
+ '', // 2
+ 'First paragraph', // 3
+ 'spans two lines.', // 4
+ '', // 5
+ '- one', // 6
+ '- two', // 7
+ '', // 8
+ '> quoted', // 9
+ '', // 10
+ '```ts', // 11
+ 'const x = 1;', // 12
+ '```', // 13
+ '', // 14
+ '| a | b |', // 15
+ '| - | - |', // 16
+ '| 1 | 2 |', // 17
+ '', // 18
+ '---', // 19
+ '', // 20
+ 'Last *words* here.', // 21
+].join('\n');
+
+const NESTED = [
+ '- outer item', // 1
+ ' continues here', // 2
+ '', // 3
+ ' second paragraph', // 4
+ '', // 5
+ ' - inner one', // 6
+ ' - inner two', // 7
+ '', // 8
+ '> quote opens', // 9
+ 'lazy continuation', // 10
+ '>', // 11
+ '> - quoted item', // 12
+ '>', // 13
+ '> ```', // 14
+ '> fenced', // 15
+ '> ```', // 16
+ '', // 17
+ '[ref]: https://example.com', // 18
+ '', // 19
+ 'See [ref].', // 20
+].join('\n');
+
+function stamps(container: ParentNode, selector: string): string[] {
+ return Array.from(container.querySelectorAll(selector)).map(
+ element =>
+ `${element.getAttribute('data-source-line')}-${element.getAttribute(
+ 'data-source-line-end',
+ )}`,
+ );
+}
+
+function renderDocument(source: string, withPlugin: boolean): HTMLElement {
+ return render(
+ <Markdown plugins={withPlugin ? [markdownSourceLinesPlugin] : []}>
+ {source}
+ </Markdown>,
+ ).container;
+}
+
+/** Every node's type and recorded lines, in document order. */
+function lineSpans(value: unknown, out: string[] = []): string[] {
+ if (Array.isArray(value)) {
+ for (const item of value) {
+ lineSpans(item, out);
+ }
+ return out;
+ }
+ if (value == null || typeof value !== 'object') {
+ return out;
+ }
+ const node = value as {
+ type?: string;
+ position?: {start?: {line?: number}; end?: {line?: number}};
+ children?: unknown;
+ };
+ const start = node.position?.start?.line;
+ if (start != null) {
+ out.push(`${node.type}:${start}-${node.position?.end?.line}`);
+ }
+ lineSpans(node.children, out);
+ return out;
+}
+
+function hasAnyLine(value: unknown): boolean {
+ return JSON.stringify(value).includes('"line"');
+}
+
+describe('markdownSourceLinesPlugin', () => {
+ beforeEach(() => {
+ parseCalls.full = 0;
+ parseCalls.incremental = 0;
+ });
+
+ it('stamps every default block kind with its 1-based inclusive lines', () => {
+ const container = renderDocument(SOURCE, true);
+
+ expect(stamps(container, 'h1')).toEqual(['1-1']);
+ expect(stamps(container, '[role="paragraph"]')).toEqual([
+ '3-4',
+ '9-9',
+ '21-21',
+ ]);
+ expect(stamps(container, 'li')).toEqual(['6-6', '7-7']);
+ expect(stamps(container, 'blockquote')).toEqual(['9-9']);
+ expect(stamps(container, '[data-source-line="11"]')).toEqual(['11-13']);
+ expect(
+ container
+ .querySelector('[data-source-line="11"]')
+ ?.querySelector('pre')
+ ?.textContent?.includes('const x = 1;'),
+ ).toBe(true);
+ expect(stamps(container, '[data-source-line="15"]')).toEqual(['15-17']);
+ expect(
+ container.querySelector('[data-source-line="15"] table'),
+ ).not.toBeNull();
+ expect(stamps(container, 'hr')).toEqual(['19-19']);
+ });
+
+ it('stamps no inline element, table row, or table cell', () => {
+ const container = renderDocument(SOURCE, true);
+
+ expect(
+ container.querySelector('em')?.hasAttribute('data-source-line'),
+ ).toBe(false);
+ expect(container.querySelector('tr[data-source-line]')).toBeNull();
+ expect(container.querySelector('td[data-source-line]')).toBeNull();
+ expect(container.querySelector('th[data-source-line]')).toBeNull();
+ });
+
+ it('stamps nested blocks with document lines, past markers, lazy lines, and stripped definitions', () => {
+ const container = renderDocument(NESTED, true);
+
+ expect(stamps(container, 'li')).toEqual(['1-7', '6-6', '7-7', '12-12']);
+ expect(stamps(container, '[role="paragraph"]')).toEqual([
+ '1-2',
+ '4-4',
+ '9-10',
+ '20-20',
+ ]);
+ expect(stamps(container, 'blockquote')).toEqual(['9-16']);
+ expect(stamps(container, 'blockquote [data-source-line="14"]')).toEqual([
+ '14-16',
+ ]);
+ });
+
+ it('counts CRLF line endings and the lines a frontmatter block occupies', () => {
+ const frontmatter = createMarkdownFrontmatter({
+ name: 'metadata',
+ parse: fields => ({title: fields.title ?? ''}),
+ });
+ render(
+ <Markdown plugins={[frontmatter.plugin, markdownSourceLinesPlugin]}>
+ {'---\r\ntitle: Notes\r\n---\r\n# Heading\r\n\r\nBody\r\nline'}
+ </Markdown>,
+ );
+
+ expect(stamps(document, 'h1')).toEqual(['4-4']);
+ expect(stamps(document, '[role="paragraph"]')).toEqual(['6-7']);
+ });
+
+ it('gives each components block renderer sourceLines and adds no wrapper', () => {
+ const received: Record<string, MarkdownSourceLines | undefined> = {};
+ const record =
+ (kind: string) =>
+ (props: {sourceLines?: MarkdownSourceLines}): void => {
+ received[kind] = props.sourceLines;
+ };
+ const components: Partial<MarkdownComponents> = {
+ heading: props => {
+ record('heading')(props);
+ return <h2 data-testid="heading">{props.children}</h2>;
+ },
+ paragraph: props => {
+ record('paragraph')(props);
+ return <p data-testid="paragraph">{props.children}</p>;
+ },
+ code: props => {
+ record('code')(props);
+ return <pre data-testid="code">{props.code}</pre>;
+ },
+ blockquote: props => {
+ record('blockquote')(props);
+ return <blockquote data-testid="quote">{props.children}</blockquote>;
+ },
+ hr: props => {
+ record('hr')(props);
+ return <hr data-testid="hr" />;
+ },
+ image: props => {
+ record('image')(props);
+ return <img data-testid="image" src={props.src} alt={props.alt} />;
+ },
+ math: props => {
+ record(`math-${props.display}`)(props);
+ return <span data-testid={`math-${props.display}`}>{props.value}</span>;
+ },
+ };
+ const source = [
+ '# Heading', // 1
+ '', // 2
+ 'Text with $x$ inline.', // 3
+ '', // 4
+ '```', // 5
+ 'code', // 6
+ '```', // 7
+ '', // 8
+ '> quote', // 9
+ '', // 10
+ '***', // 11
+ '', // 12
+ '![alt](https://example.com/a.png)', // 13
+ '', // 14
+ '$$', // 15
+ 'y', // 16
+ '$$', // 17
+ ].join('\n');
+ const {container} = render(
+ <Markdown plugins={[markdownSourceLinesPlugin]} components={components}>
+ {source}
+ </Markdown>,
+ );
+
+ expect(received).toEqual({
+ heading: {start: 1, end: 1},
+ paragraph: {start: 9, end: 9},
+ 'math-inline': undefined,
+ code: {start: 5, end: 7},
+ blockquote: {start: 9, end: 9},
+ hr: {start: 11, end: 11},
+ image: {start: 13, end: 13},
+ 'math-block': {start: 15, end: 17},
+ });
+ // Renderers own their element: Markdown places no stamp on, or around, it.
+ expect(container.querySelector('[data-source-line]')).toBeNull();
+ expect(screen.getByTestId('heading').parentElement).toBe(
+ container.firstElementChild,
+ );
+ });
+
+ it('gives plugin block renderers and claimed fences their lines through position', () => {
+ type CardNode = MarkdownExtensionNode<
+ 'cards',
+ 'card',
+ Record<string, never>,
+ 'block'
+ >;
+ type ChartNode = MarkdownExtensionNode<
+ 'charts',
+ 'chart',
+ {readonly spec: string},
+ 'block'
+ >;
+ const seen: (string | undefined)[] = [];
+ const cards = createMarkdownPlugin<'cards', CardNode>({
+ name: 'cards',
+ apiVersion: 1,
+ parseKey: 'v1',
+ syntax: {
+ block: [
+ {
+ startsWith: [':::card'],
+ maxSpan: 200,
+ tokenize: ({offset}) => ({
+ status: 'match',
+ end: offset + ':::card'.length,
+ node: {
+ type: 'extension',
+ plugin: 'cards',
+ name: 'card',
+ display: 'block',
+ data: {},
+ },
+ }),
+ },
+ ],
+ },
+ renderers: {
+ card: {
+ render: ({node}) => {
+ seen.push(
+ `card:${node.position?.start.line}-${node.position?.end.line}`,
+ );
+ return <div>card</div>;
+ },
+ toText: () => 'card',
+ },
+ },
+ });
+ const charts = createMarkdownPlugin<'charts', ChartNode>({
+ name: 'charts',
+ apiVersion: 1,
+ transform: createMarkdownFenceTransform({
+ languages: ['chart'],
+ createNode: ({code}) => ({
+ type: 'extension',
+ plugin: 'charts',
+ name: 'chart',
+ display: 'block',
+ data: {spec: code},
+ }),
+ }),
+ renderers: {
+ chart: {
+ render: ({node}) => {
+ seen.push(
+ `chart:${node.position?.start.line}-${node.position?.end.line}`,
+ );
+ return <div>chart</div>;
+ },
+ toText: node => node.data.spec,
+ },
+ },
+ });
+
+ render(
+ <Markdown plugins={[cards, charts, markdownSourceLinesPlugin]}>
+ {'Intro\n\n:::card\n\n- item\n\n ```chart\n bars\n ```'}
+ </Markdown>,
+ );
+
+ expect(seen).toEqual(['card:3-3', 'chart:7-9']);
+ });
+
+ it('keeps the lines when a claimed fence falls back to the host code renderer or the default code block', () => {
+ type DiagramNode = MarkdownExtensionNode<
+ 'diagrams',
+ 'diagram',
+ {readonly code: string},
+ 'block'
+ >;
+ const diagrams = (render: () => null | never) =>
+ createMarkdownPlugin<'diagrams', DiagramNode>({
+ name: 'diagrams',
+ apiVersion: 1,
+ transform: createMarkdownFenceTransform({
+ languages: ['diagram'],
+ createNode: ({code}) => ({
+ type: 'extension',
+ plugin: 'diagrams',
+ name: 'diagram',
+ display: 'block',
+ data: {code},
+ }),
+ }),
+ renderers: {diagram: {render, toText: node => node.data.code}},
+ });
+ const declining = diagrams(() => null);
+ const throwing = diagrams(() => {
+ throw new Error('broken diagram');
+ });
+ const source =
+ 'Intro\n\n```diagram\na --> b\n```\n\n```ts\nconst x = 1;\n```';
+ const hostLines: [string | undefined, MarkdownSourceLines | undefined][] =
+ [];
+ const HostCode: NonNullable<MarkdownComponents['code']> = ({
+ code,
+ language,
+ sourceLines,
+ }) => {
+ hostLines.push([language, sourceLines]);
+ return <pre>{code}</pre>;
+ };
+
+ const {unmount} = render(
+ <Markdown
+ plugins={[declining, markdownSourceLinesPlugin]}
+ components={{code: HostCode}}>
+ {source}
+ </Markdown>,
+ );
+ expect(hostLines).toEqual([
+ ['diagram', {start: 3, end: 5}],
+ ['ts', {start: 7, end: 9}],
+ ]);
+ unmount();
+
+ const warning = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ const {container} = render(
+ <Markdown plugins={[throwing, markdownSourceLinesPlugin]}>
+ {source}
+ </Markdown>,
+ );
+ expect(stamps(container, '[data-source-line="3"]')).toEqual(['3-5']);
+ expect(
+ container.querySelector('[data-source-line="3"] pre')?.textContent,
+ ).toContain('a --> b');
+ warning.mockRestore();
+ });
+
+ it('records lines on canonical block nodes only with the plugin', () => {
+ const withLines = parseMarkdownAst(NESTED, {
+ plugins: [markdownSourceLinesPlugin],
+ });
+ expect(lineSpans(withLines)).toEqual([
+ 'list:1-7',
+ 'listItem:1-7',
+ 'paragraph:1-2',
+ 'paragraph:4-4',
+ 'list:6-7',
+ 'listItem:6-6',
+ 'paragraph:6-6',
+ 'listItem:7-7',
+ 'paragraph:7-7',
+ 'blockquote:9-16',
+ 'paragraph:9-10',
+ 'list:12-12',
+ 'listItem:12-12',
+ 'paragraph:12-12',
+ 'code:14-16',
+ 'paragraph:20-20',
+ ]);
+
+ const without = parseMarkdownAst(NESTED);
+ expect(hasAnyLine(without)).toBe(false);
+ expect(hasAnyLine(parseMarkdownAst(NESTED, {plugins: []}))).toBe(false);
+ });
+
+ it('keeps the released DOM and renderer props byte-identical without the plugin', () => {
+ const plain = renderToStaticMarkup(<Markdown>{SOURCE}</Markdown>);
+ expect(
+ renderToStaticMarkup(<Markdown plugins={[]}>{SOURCE}</Markdown>),
+ ).toBe(plain);
+ expect(plain).not.toContain('data-source-line');
+
+ const paragraphProps: object[] = [];
+ render(
+ <Markdown
+ components={{
+ paragraph: props => {
+ paragraphProps.push(props);
+ return <p>{props.children}</p>;
+ },
+ }}>
+ {SOURCE}
+ </Markdown>,
+ );
+ expect(paragraphProps.length).toBeGreaterThan(0);
+ expect(paragraphProps.every(props => !('sourceLines' in props))).toBe(true);
+ });
+
+ it('runs one parse and tokenizes each source position once, with or without the plugin', () => {
+ let tokenizerCalls = 0;
+ type MarkNode = MarkdownExtensionNode<
+ 'marks',
+ 'mark',
+ Record<string, never>,
+ 'inline'
+ >;
+ const marks = createMarkdownPlugin<'marks', MarkNode>({
+ name: 'marks',
+ apiVersion: 1,
+ parseKey: 'v1',
+ syntax: {
+ inline: [
+ {
+ startsWith: ['!!'],
+ maxSpan: 4,
+ tokenize: ({offset}) => {
+ tokenizerCalls++;
+ return {
+ status: 'match',
+ end: offset + 2,
+ node: {
+ type: 'extension',
+ plugin: 'marks',
+ name: 'mark',
+ display: 'inline',
+ data: {},
+ },
+ };
+ },
+ },
+ ],
+ },
+ renderers: {mark: {render: () => <b>!</b>, toText: () => '!'}},
+ });
+ const source = `${NESTED}\n\nA !! mark, and - !! in a list:\n\n- item !!\n > quote !!`;
+
+ const {unmount} = render(<Markdown plugins={[marks]}>{source}</Markdown>);
+ const baseline = {parses: parseCalls.full, tokens: tokenizerCalls};
+ unmount();
+ parseCalls.full = 0;
+ tokenizerCalls = 0;
+
+ render(
+ <Markdown plugins={[marks, markdownSourceLinesPlugin]}>
+ {source}
+ </Markdown>,
+ );
+
+ expect(baseline.parses).toBeGreaterThan(0);
+ expect(baseline.tokens).toBeGreaterThan(0);
+ expect({parses: parseCalls.full, tokens: tokenizerCalls}).toEqual(baseline);
+ expect(stamps(document, 'li').length).toBeGreaterThan(0);
+ });
+
+ it('streams to the full-parse lines at every character boundary', () => {
+ const options = {plugins: [markdownSourceLinesPlugin]};
+ const state = createIncrementalState();
+ let compared = 0;
+ let last = parseMarkdownAstIncremental('', state, options);
+ for (let end = 1; end <= NESTED.length; end++) {
+ const prefix = NESTED.slice(0, end);
+ last = parseMarkdownAstIncremental(prefix, state, options);
+ const full = parseMarkdownAst(prefix, options);
+ // Where the stream shows the same blocks as a full parse, their lines
+ // must agree too; elsewhere the stream is still withholding a block.
+ if (
+ JSON.stringify(last.children.map(node => node.type)) ===
+ JSON.stringify(full.children.map(node => node.type)) &&
+ lineSpans(last.children).length === lineSpans(full.children).length
+ ) {
+ expect(lineSpans(last.children)).toEqual(lineSpans(full.children));
+ compared++;
+ }
+ }
+ expect(compared).toBeGreaterThan(NESTED.length / 2);
+ expect(lineSpans(last.children)).toEqual(
+ lineSpans(parseMarkdownAst(NESTED, options).children),
+ );
+ });
+
+ it('keeps a settled block and its stamp while more text streams in', () => {
+ const options = {plugins: [markdownSourceLinesPlugin]};
+ const state = createIncrementalState();
+ const first = parseMarkdownAstIncremental(
+ '# Title\n\nBody one.\n\nBody',
+ state,
+ options,
+ );
+ const second = parseMarkdownAstIncremental(
+ '# Title\n\nBody one.\n\nBody two\nwraps.',
+ state,
+ options,
+ );
+ expect(second.children[0]).toBe(first.children[0]);
+ expect(lineSpans(second.children)).toEqual([
+ 'heading:1-1',
+ 'paragraph:3-3',
+ 'paragraph:5-6',
+ ]);
+ });
+});
diff --git a/packages/core/src/Markdown/plugins/sourceLines.ts b/packages/core/src/Markdown/plugins/sourceLines.ts
new file mode 100644
index 0000000000..8675639861
--- /dev/null
+++ b/packages/core/src/Markdown/plugins/sourceLines.ts
@@ -0,0 +1,21 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file sourceLines.ts
+ * @input The parse Markdown already runs
+ * @output A first-party plugin that records each block's source lines
+ * @position Optional Markdown parse option, owned by module:Markdown/sourceLines
+ */
+
+import {createMarkdownSourceLinesEntry} from './protocol';
+
+/**
+ * Records the 1-based, inclusive source lines of every block Markdown renders,
+ * nested blocks included. Rendered blocks carry them as `data-source-line` and
+ * `data-source-line-end`, `components` block renderers receive them as
+ * `sourceLines`, and canonical block nodes carry them as `position` lines.
+ * The lines come from the parse Markdown already runs; without this plugin
+ * no line work happens.
+ */
+export const markdownSourceLinesPlugin =
+ createMarkdownSourceLinesEntry('source-lines');