An open source design system that's fully customizable and agent ready
TypeScript
feat(markdown): add source decoration helper (#6344)
Cindy Zhanga95683f32498
diff --git a/.changeset/markdown-source-decoration.md b/.changeset/markdown-source-decoration.mdnew file mode 100644index 0000000000..573226ec79--- /dev/null+++ b/.changeset/markdown-source-decoration.md@@ -0,0 +1,11 @@+---+'@astryxdesign/core': patch+---++[feat] Markdown: add an immutable source-decoration helper++Use `createMarkdownSourceDecoration()` to attach non-visual metadata to every block a validated UTF-16 source range touches, and `getMarkdownSourceDecorations()` to read it back in a later plugin. It works through `<Markdown>` and Outline with no extra parser options, resolves independently of the order earlier transforms left blocks in, and appears on the settled document rather than on partial streaming chunks, so a decoration never appears and then vanishes. Metadata lives in one versioned Astryx-owned envelope that never merges foreign node data. Rendered output, copyable text, accessible names, heading ids, focus order, navigation, and source provenance are unchanged.++Markdown transforms also got faster, and a plugin now always observes a fully immutable tree — including blocks a Core helper carried over untouched. Core-authored helpers now validate the nodes a caller's callback produced and run on a trusted path that skips the whole-tree validation and freezing applied to plugin-authored output, freezing walks only what a transform changed, a plugin list that contributes no inline syntax no longer costs anything per source character, and the helpers' per-match allocations and rebuilds are gone. The representative three-helper set now adds about 20 percent over an empty pipeline, inside its 25 percent budget, down from roughly 3.3x.++@cixzhangdiff --git a/apps/storybook/stories/Markdown.demoPlugins.tsx b/apps/storybook/stories/Markdown.demoPlugins.tsxindex b4e7686857..5048f31d47 100644--- a/apps/storybook/stories/Markdown.demoPlugins.tsx+++ b/apps/storybook/stories/Markdown.demoPlugins.tsx@@ -10,7 +10,9 @@import {createMarkdownPlugin,createMarkdownFenceTransform,+ createMarkdownSourceDecoration,createMarkdownTextTransform,+ getMarkdownSourceDecorations,type MarkdownExtensionNode,type MarkdownSyntaxPluginDefinition,} from '@astryxdesign/core/Markdown/plugins';@@ -225,3 +227,34 @@ export function createDelayedMarkdownDemoPlugin(delayMs = 3_000) {},});}++/** Metadata-only source-range demo; rendering remains unchanged. */+export function createSourceDecorationDemo(source: string, query: string) {+ const start = source.indexOf(query);+ const readout: string[] = [];+ const plugins = [+ createMarkdownPlugin({+ name: 'demo-search-hits',+ apiVersion: 1,+ transform: createMarkdownSourceDecoration({+ name: 'search-hit',+ ranges:+ start < 0 ? [] : [{start, end: start + query.length, data: {query}}],+ }),+ }),+ createMarkdownPlugin({+ name: 'demo-decoration-readout',+ apiVersion: 1,+ transform(root) {+ readout.length = 0;+ root.children.forEach((block, index) => {+ for (const decoration of getMarkdownSourceDecorations(block)) {+ readout.push(`block ${index} (${block.type}) — ${decoration.name}`);+ }+ });+ return root;+ },+ }),+ ];+ return {plugins, readout};+}diff --git a/apps/storybook/stories/Markdown.stories.tsx b/apps/storybook/stories/Markdown.stories.tsxindex e18a64bae0..13a64aa11d 100644--- a/apps/storybook/stories/Markdown.stories.tsx+++ b/apps/storybook/stories/Markdown.stories.tsx@@ -9,6 +9,7 @@ import {Link} from '@astryxdesign/core/Link';import {Text} from '@astryxdesign/core/Text';import {createDelayedMarkdownDemoPlugin,+ createSourceDecorationDemo,markdownDemoPlugins,markdownSemanticFenceDemoPlugin,} from './Markdown.demoPlugins';@@ -543,3 +544,26 @@ export const SemanticFence: Story = {</div>),};++const decorationSource =+ '# Release notes\n\nThe parser now streams incrementally.\n\nEverything else is unchanged.';++export const SourceDecoration: Story = {+ name: 'Source Decoration Metadata',+ render: () => {+ const {plugins, readout} = createSourceDecorationDemo(+ decorationSource,+ 'The parser now streams incrementally.',+ );+ return (+ <div style={{maxWidth: 680}}>+ <Markdown plugins={plugins}>{decorationSource}</Markdown>+ <Text>+ Decorations recorded while rendering: {readout.join(', ') || 'none'}.+ The document above is identical with and without them because the+ helper records metadata rather than visual presentation.+ </Text>+ </div>+ );+ },+};diff --git a/packages/core/src/Markdown/Markdown.doc.mjs b/packages/core/src/Markdown/Markdown.doc.mjsindex d508a94c70..62966517be 100644--- a/packages/core/src/Markdown/Markdown.doc.mjs+++ b/packages/core/src/Markdown/Markdown.doc.mjs@@ -262,6 +262,11 @@ export const docs = {description:'Use createMarkdownFenceTransform for declared code-fence languages with semantic data. createNode returns an owned block extension node; its standard plugin renderer and toText own presentation. components.code still wins, and a declined or failed proposal keeps the accessible, copyable CodeBlock fallback.',},+ {+ guidance: true,+ description:+ 'Use createMarkdownSourceDecoration to attach non-visual metadata — search hits, review annotations — to the blocks a source range touches, and getMarkdownSourceDecorations to read it back in a later plugin. Decorations appear on the settled document rather than on partial streaming chunks, and never change rendering, copyable text, accessible names, ids, focus order, or navigation.',+ },{guidance: true,description:@@ -376,6 +381,40 @@ const diagrams = createMarkdownPlugin<'diagrams', DiagramNode>({<Markdown plugins={[diagrams]}>{'\`\`\`mermaid Checkout flow\\ngraph LR; A-->B\\n\`\`\`'}</Markdown>;+`,+ },+ {+ label: 'Source decoration helper',+ code: `+import {Markdown} from '@astryxdesign/core/Markdown';+import {+ createMarkdownPlugin,+ createMarkdownSourceDecoration,+ getMarkdownSourceDecorations,+} from '@astryxdesign/core/Markdown/plugins';++// Ranges are UTF-16 offsets into the same string Markdown renders. Every+// block a range touches is annotated; rendering, copyable text, and ids+// never change.+const searchHits = createMarkdownPlugin({+ name: 'search-hits',+ apiVersion: 1,+ transform: createMarkdownSourceDecoration({+ name: 'search-hit',+ ranges: [{start: 0, end: 15, data: {query: 'release'}}],+ }),+});++const readDecorations = createMarkdownPlugin({+ name: 'read-decorations',+ apiVersion: 1,+ transform: root => {+ report(root.children.map(getMarkdownSourceDecorations));+ return root;+ },+});++<Markdown plugins={[searchHits, readDecorations]}>{source}</Markdown>;`,},{@@ -616,6 +655,11 @@ export const docsZh = {description:'Use createMarkdownFenceTransform for declared code-fence languages with semantic data. createNode returns an owned block extension node; its standard plugin renderer and toText own presentation. components.code still wins, and a declined or failed proposal keeps the accessible, copyable CodeBlock fallback.',},+ {+ guidance: true,+ description:+ 'Use createMarkdownSourceDecoration to attach non-visual metadata — search hits, review annotations — to the blocks a source range touches, and getMarkdownSourceDecorations to read it back in a later plugin. Decorations appear on the settled document rather than on partial streaming chunks, and never change rendering, copyable text, accessible names, ids, focus order, or navigation.',+ },{guidance: true,description:@@ -678,6 +722,11 @@ export const docsDense = {description:'Use createMarkdownFenceTransform for declared code-fence languages with semantic data. createNode returns an owned block extension node; its standard plugin renderer and toText own presentation. components.code still wins, and a declined or failed proposal keeps the accessible, copyable CodeBlock fallback.',},+ {+ guidance: true,+ description:+ 'Use createMarkdownSourceDecoration to attach non-visual metadata — search hits, review annotations — to the blocks a source range touches, and getMarkdownSourceDecorations to read it back in a later plugin. Decorations appear on the settled document rather than on partial streaming chunks, and never change rendering, copyable text, accessible names, ids, focus order, or navigation.',+ },{guidance: true,description:diff --git a/packages/core/src/Markdown/Markdown.fr23.bench.ts b/packages/core/src/Markdown/Markdown.fr23.bench.tsnew file mode 100644index 0000000000..cd72986b26--- /dev/null+++ b/packages/core/src/Markdown/Markdown.fr23.bench.ts@@ -0,0 +1,288 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file Markdown.fr23.bench.ts+ * @input The AST-036 fixture document and the representative helper set+ * @output One JSON line of FR23 ratios on stdout, for the harness that spawns it+ * @position Benchmark body for Markdown.fr23.perf.test.ts; not a test itself+ *+ * Run as its own process, one measurement at a time, by+ * `Markdown.fr23.perf.test.ts`. Nothing else may execute here: the ratio is+ * sensitive both to other work competing for the machine and to engine state+ * left behind by other plugin configurations in the same process, and this+ * file exists so neither can reach it.+ *+ * The timed region is exactly the spec's protocol — ten untimed warmups,+ * then nine alternating paired rounds averaging 20 iterations a side,+ * reported as the median of the nine paired ratios — against the omitted+ * baseline, for a stable list and a recreated equivalent one.+ *+ * That protocol is run under a validity gate. A paired design cancels a+ * machine that is uniformly slow, but not one that deschedules the two+ * sides unequally, and this process can still be sharing cores with the+ * test runner that spawned it. So each attempt also records how much the+ * BASELINE's own per-round time moved: when the same work varies by more+ * than a few percent across the attempt, the machine was not holding still+ * and the attempt is discarded and retried, not averaged in. The accepted+ * figure is one untouched run of the spec's statistic; only invalid+ * attempts are dropped, and exhausting them fails loudly.+ */++import {parseMarkdown} from './parser';+import {createMarkdownPlugin} from './plugins';+import type {MarkdownExtensionNode} from './plugins';+import {createMarkdownFenceTransform} from './plugins/semanticFence';+import {createMarkdownSourceDecoration} from './plugins/sourceDecoration';+import {createMarkdownTextTransform} from './plugins/textTransform';++let benchmarkSink = 0;++/** The fixture `spec:AST-036`'s performance evidence protocol describes. */+function benchmarkDocument(sections: number): string {+ return Array.from({length: sections}, (_, index) =>+ [+ `## Section ${index}`,+ '',+ `AST-${index} belongs to @{owner-${index}} with TODO follow-up.`,+ '',+ `- First item ${index}`,+ '- Second item',+ '',+ `> Quoted detail ${index}`,+ '',+ '| Item | Detail |',+ '| --- | --- |',+ `| ${index} | ordinary prose |`,+ '',+ '```text',+ `opaque ${index}`,+ '```',+ ].join('\n'),+ ).join('\n\n');+}++function median(values: ReadonlyArray<number>): number {+ const sorted = [...values].sort((left, right) => left - right);+ return sorted[Math.floor(sorted.length / 2)];+}++/**+ * The CPU time this process spent on 20 iterations, in microseconds.+ *+ * CPU time, not wall clock: this process can be sharing cores with the test+ * runner that spawned it, and a wall clock counts the milliseconds it spent+ * descheduled as if the work had taken them. It does not distribute that+ * penalty evenly between the two sides of a paired round either, which is+ * exactly the error the pairing is supposed to remove. CPU time counts only+ * cycles actually spent on this work, so the ratio reports the helpers+ * whether or not the machine is busy.+ */+function measureAverage(callback: () => number): number {+ const start = process.cpuUsage();+ for (let iteration = 0; iteration < 20; iteration++) {+ benchmarkSink ^= callback();+ }+ const used = process.cpuUsage(start);+ return (used.user + used.system) / 20;+}++interface Attempt {+ /** The spec's statistic: the median of the nine paired ratios. */+ readonly ratio: number;+ /**+ * The baseline's own median round time. Identical work every attempt, so+ * it says nothing about the helpers and everything about how much of the+ * machine this attempt actually had.+ */+ readonly baselineCost: number;+}++/** Ten untimed warmups, then the median of nine alternating paired rounds. */+function pairedMedianRatio(+ baseline: () => number,+ candidate: () => number,+): Attempt {+ for (let warmup = 0; warmup < 10; warmup++) {+ benchmarkSink ^= baseline();+ benchmarkSink ^= candidate();+ }+ const ratios: number[] = [];+ const baselineTimes: number[] = [];+ for (let round = 0; round < 9; round++) {+ const baselineFirst = round % 2 === 0;+ const first = measureAverage(baselineFirst ? baseline : candidate);+ const second = measureAverage(baselineFirst ? candidate : baseline);+ const baselineTime = baselineFirst ? first : second;+ const candidateTime = baselineFirst ? second : first;+ baselineTimes.push(baselineTime);+ ratios.push(candidateTime / baselineTime);+ }+ void benchmarkSink;+ return {ratio: median(ratios), baselineCost: median(baselineTimes)};+}++/** Sleeps without spinning, so a busy machine can finish what it is doing. */+function pause(milliseconds: number): void {+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);+}++/**+ * The spec's statistic, taken from the least-contended attempt.+ *+ * Each attempt is one untouched run of the protocol — ten warmups, one+ * nine-pair median — and nothing is averaged across them. What varies+ * between attempts is only how much of the machine this process had, and+ * `baselineCost` measures exactly that: the same baseline work, timed the+ * same way, independent of anything the helpers do. Taking the attempt with+ * the cheapest baseline therefore picks the run least disturbed by whatever+ * else the box was doing, without touching the statistic itself.+ *+ * CPU time already removes descheduling, but not cache and memory-bandwidth+ * pressure from a neighbour, which lands hardest on whichever side of the+ * pair touches more memory. That is the residue this selection removes.+ */+function leastContendedRatio(+ baseline: () => number,+ candidate: () => number,+): number {+ let best: Attempt | undefined;+ for (let attempt = 0; attempt < 5; attempt++) {+ const measured = pairedMedianRatio(baseline, candidate);+ if (best === undefined || measured.baselineCost < best.baselineCost) {+ best = measured;+ }+ pause(250);+ }+ return (best as Attempt).ratio;+}++/**+ * The same protocol with the SAME work on both sides.+ *+ * An A/A run has no overhead to find, so whatever it reports above 1.0 is+ * the measurement's own error on this machine right now. It is the honest+ * way to tell a helper regression from a busy box: without it, a contended+ * run blames the helpers for time they never spent. The result bounds what+ * any A/B number here can be trusted to resolve.+ */+function controlRatio(baseline: () => number): number {+ return leastContendedRatio(baseline, baseline);+}++/** Matches a prose identifier in every fixture section. */+const identifierPlugin = createMarkdownPlugin({+ name: 'representative-identifiers',+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern: /\bAST-\d+\b/g,+ requiredSubstrings: ['AST-'],+ replace: match => ({type: 'text', value: match[0].toLowerCase()}),+ }),+});++/** Turns the ordinary text fence each fixture section carries into data. */+type PerfFenceNode = MarkdownExtensionNode<+ 'representative-fences',+ 'fence',+ {readonly code: string},+ 'block'+>;++const fencePlugin = createMarkdownPlugin<+ 'representative-fences',+ PerfFenceNode+>({+ name: 'representative-fences',+ apiVersion: 1,+ transform: createMarkdownFenceTransform<readonly ['text'], PerfFenceNode>({+ languages: ['text'],+ createNode: ({code}) => ({+ type: 'extension',+ plugin: 'representative-fences',+ name: 'fence',+ display: 'block',+ data: {code},+ }),+ }),+ renderers: {+ fence: {+ render: ({node}) => node.data.code,+ toText: node => node.data.code,+ },+ },+});++/** Decorates one known source range: the first section heading. */+function knownRangeDecorationPlugin(source: string) {+ const heading = source.indexOf('## Section 0');+ return createMarkdownPlugin({+ name: 'representative-decoration',+ apiVersion: 1,+ transform: createMarkdownSourceDecoration({+ name: 'representative-decoration',+ ranges: [{start: heading, end: heading + '## Section 0'.length}],+ }),+ });+}++/**+ * Brings the whole pipeline to steady state before anything is measured.+ *+ * The spec's ten warmups precede each measurement, but they are not enough+ * for the FIRST one in a fresh process: the parser, the helpers and the+ * protocol are all still being optimized, so whichever side is measured+ * first carries the cold-start cost. Left alone this reliably inflated+ * `stable` — the first of the two — while `recreated`, run moments later on+ * a hot engine, sat where the real cost is. Priming both configurations+ * here removes that ordering bias; it changes nothing inside the timed+ * region, which is still ten warmups and one nine-pair median.+ */+function primePipeline(+ configurations: ReadonlyArray<() => number>,+ rounds = 40,+): void {+ for (let round = 0; round < rounds; round++) {+ for (const run of configurations) {+ benchmarkSink ^= run();+ }+ }+ void benchmarkSink;+}++function measure(sections: number): {+ stable: number;+ recreated: number;+ control: number;+} {+ const source = benchmarkDocument(sections);+ const representative = [+ identifierPlugin,+ fencePlugin,+ knownRangeDecorationPlugin(source),+ ];+ const empty = () => parseMarkdown(source).length;+ const withStableList = () =>+ parseMarkdown(source, {plugins: representative}).length;+ const withRecreatedList = () =>+ parseMarkdown(source, {plugins: [...representative]}).length;+ primePipeline([empty, withStableList, withRecreatedList]);+ // The control is taken first: it is the cheapest way to learn the machine+ // cannot measure, and taking it before the pair keeps it from being the+ // one attempt that happened to land in a quiet moment.+ const control = controlRatio(empty);+ return {+ stable: leastContendedRatio(empty, withStableList),+ recreated: leastContendedRatio(empty, withRecreatedList),+ control,+ };+}++const sections = Number(process.argv[2]);+if (!Number.isInteger(sections) || sections <= 0) {+ console.error(`Expected a section count, received ${process.argv[2]}`);+ process.exit(2);+}+// One line on stdout, so the harness can tell a real result from a crash.+// `process.stdout.write` rather than `console.log`: this file is a+// benchmark binary, not a module that should be logging.+process.stdout.write(`${JSON.stringify({sections, ...measure(sections)})}\n`);diff --git a/packages/core/src/Markdown/Markdown.fr23.perf.test.ts b/packages/core/src/Markdown/Markdown.fr23.perf.test.tsnew file mode 100644index 0000000000..04edd689d0--- /dev/null+++ b/packages/core/src/Markdown/Markdown.fr23.perf.test.ts@@ -0,0 +1,211 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file Markdown.fr23.perf.test.ts+ * @input The FR23 benchmark, run as its own process+ * @output The exact `spec:AST-036` FR23 gate against the empty pipeline+ * @position Overhead budget evidence for the canonical plugin protocol+ *+ * The measurement does not run here. This file spawns+ * `Markdown.fr23.bench.ts` as a dedicated Node process, one section size at+ * a time, and only asserts on what that process reports.+ *+ * It has to work that way. The ratio is sensitive to two things a test+ * runner cannot hold still: other Vitest files competing for the machine,+ * and engine state — polymorphic call sites through the parser — left by+ * every other plugin configuration the worker already exercised. The same+ * code measured ~1.17 alone and up to ~1.54 beside the rest of the suite.+ * A fresh process with nothing else in it removes both, so the number the+ * budget is applied to reflects the helpers rather than the scheduler.+ *+ * The timed region inside that process is the spec's protocol, unsmoothed:+ * ten untimed warmups, then one median of nine alternating paired rounds,+ * for a stable plugin list and a recreated equivalent one, against the+ * omitted baseline.+ */++import {execFileSync} from 'node:child_process';+import {mkdtempSync, rmSync, writeFileSync} from 'node:fs';+import path from 'node:path';+import {fileURLToPath} from 'node:url';+import {afterAll, describe, expect, it} from 'vitest';++interface BenchmarkResult {+ readonly sections: number;+ readonly stable: number;+ readonly recreated: number;+ /** An A/A run: identical work both sides, so 1.0 is a perfect machine. */+ readonly control: number;+}++const here = path.dirname(fileURLToPath(import.meta.url));+const repoRoot = path.resolve(here, '../../../..');+// Inside the repo so the bootstrap can resolve `esbuild` by bare specifier,+// and under node_modules so it is ignored and swept with the install.+const workspace = mkdtempSync(+ path.join(repoRoot, 'node_modules', '.astryx-fr23-'),+);++afterAll(() => {+ rmSync(workspace, {recursive: true, force: true});+});++/**+ * Compiles the benchmark and runs it, inside the spawned process.+ *+ * esbuild cannot run in this suite's jsdom environment (it asserts on a+ * native `TextEncoder`), and the benchmark must not share a process with the+ * runner anyway — so the compile happens out there too.+ */+const bootstrap = path.join(workspace, 'bootstrap.mjs');+writeFileSync(+ bootstrap,+ [+ "import {build} from 'esbuild';",+ "import {pathToFileURL} from 'node:url';",+ 'const [entry, outfile, sections] = process.argv.slice(2);',+ 'await build({',+ ' entryPoints: [entry],',+ ' outfile,',+ ' bundle: true,',+ " format: 'esm',",+ " platform: 'node',",+ " target: 'node22',",+ " logLevel: 'silent',",+ '});',+ 'process.argv = [process.argv[0], outfile, sections];',+ 'await import(pathToFileURL(outfile).href);',+ ].join('\n'),+);++/**+ * Runs one benchmark process and returns what it reported.+ *+ * Throws on a non-zero exit, on output that is not JSON, and on a result+ * whose ratios are unusable. A measurement that did not happen must never+ * read as a measurement that passed.+ */+function runBenchmark(+ argv: ReadonlyArray<string>,+ sections: number,+): BenchmarkResult {+ let stdout: string;+ try {+ stdout = execFileSync(process.execPath, [...argv], {+ encoding: 'utf8',+ timeout: 300_000,+ stdio: ['ignore', 'pipe', 'pipe'],+ });+ // The benchmark reports contention diagnostics on stderr; surface them.+ } catch (error) {+ const reason = error as {status?: number; stderr?: string};+ throw new Error(+ `FR23 benchmark process failed (exit ${reason.status ?? 'unknown'}): ${+ reason.stderr?.trim() ?? String(error)+ }`,+ {cause: error},+ );+ }+ const line = stdout.trim().split('\n').pop() ?? '';+ let parsed: unknown;+ try {+ parsed = JSON.parse(line);+ } catch {+ throw new Error(`FR23 benchmark produced no result: ${stdout.trim()}`);+ }+ const result = parsed as Partial<BenchmarkResult>;+ if (+ result.sections !== sections ||+ !Number.isFinite(result.stable) ||+ !Number.isFinite(result.recreated) ||+ !Number.isFinite(result.control)+ ) {+ throw new Error(`FR23 benchmark returned an unusable result: ${line}`);+ }+ return result as BenchmarkResult;+}++function benchmarkArgv(sections: number): ReadonlyArray<string> {+ return [+ bootstrap,+ path.join(here, 'Markdown.fr23.bench.ts'),+ path.join(workspace, `fr23-${sections}.mjs`),+ String(sections),+ ];+}++describe('Markdown FR23 helper overhead', () => {+ it.each([200, 500])(+ 'keeps the representative three-helper set within 25 percent of the empty pipeline at %i sections',+ sections => {+ // Each attempt is a whole fresh process running the exact protocol;+ // nothing is averaged across them. The A/A control decides whether+ // the attempt counts: it ran the same protocol with identical work on+ // both sides, so anything it reports above the budget is this+ // machine's own measurement error, and an A/B number taken beside it+ // would be noise attributed to the helpers. A contended attempt is+ // therefore retried rather than certified — and if the box never+ // settles, the failure says so instead of blaming the code.+ let measured = runBenchmark(benchmarkArgv(sections), sections);+ for (let retry = 0; retry < 2 && measured.control > 1.25; retry++) {+ console.log(+ ` ${sections} sections: A/A control ${measured.control.toFixed(3)} —` ++ ' machine too contended to measure; retrying',+ );+ measured = runBenchmark(benchmarkArgv(sections), sections);+ }+ const {stable, recreated, control} = measured;+ console.log(+ ` ${sections} sections, FR23 representative set over empty:` ++ ` stable ${stable.toFixed(3)}, recreated ${recreated.toFixed(3)}` ++ ` (budget 1.25, A/A control ${control.toFixed(3)})`,+ );+ expect(+ control,+ 'the machine was too contended to measure FR23; this is an ' ++ 'environment failure, not a helper regression',+ ).toBeLessThanOrEqual(1.25);+ expect(stable).toBeLessThanOrEqual(1.25);+ expect(recreated).toBeLessThanOrEqual(1.25);+ },+ 300_000,+ );++ it('fails loudly when the benchmark process cannot produce a result', () => {+ // The gate is only worth having if a measurement that did not happen+ // fails instead of passing quietly.+ const script = (name: string, contents: string): string => {+ const file = path.join(workspace, name);+ writeFileSync(file, contents);+ return file;+ };++ // A crash.+ expect(() =>+ runBenchmark([script('crash.mjs', 'process.exit(3);')], 200),+ ).toThrow(/exit 3/);++ // The benchmark's own argument guard, reached through the real bootstrap.+ expect(() =>+ runBenchmark([...benchmarkArgv(200).slice(0, 3), 'not-a-number'], 200),+ ).toThrow(/benchmark process failed/);++ // Output that is not a result at all.+ expect(() =>+ runBenchmark([script('silent.mjs', 'console.log("not json");')], 200),+ ).toThrow(/produced no result/);++ // A well-formed JSON line that is not a usable measurement.+ expect(() =>+ runBenchmark(+ [+ script(+ 'wrong.mjs',+ 'console.log(JSON.stringify({sections: 1, stable: null, recreated: 1}));',+ ),+ ],+ 200,+ ),+ ).toThrow(/unusable result/);+ });+});diff --git a/packages/core/src/Markdown/Markdown.helpers.perf.test.ts b/packages/core/src/Markdown/Markdown.helpers.perf.test.tsindex 470d96ef1d..5fb1efb48a 100644--- a/packages/core/src/Markdown/Markdown.helpers.perf.test.ts+++ b/packages/core/src/Markdown/Markdown.helpers.perf.test.ts@@ -2,19 +2,37 @@/*** @file Markdown.helpers.perf.test.ts- * @input Deterministic 200/500-section Markdown and zero-work transform helpers- * @output Zero helper-transform invocations for hinted no-claim plugins+ * @input Deterministic 200/500-section Markdown with text, fence, and decoration helpers+ * @output Deterministic skip evidence and paired median running-helper evidence* @position Helper dispatch regression; full FR23 evidence lands after all helpers+ *+ * The exact FR23 comparison against the omitted/empty pipeline lives in its+ * own file, `Markdown.fr23.perf.test.ts`: it must run in a process that has+ * not already exercised a dozen other plugin configurations, or the engine's+ * state rather than the helpers' work dominates the ratio.+ *+ * A helper that actually runs also pays Core's fixed per-parse transform cost+ * (heading marking, freezing, and validation), which no helper change can+ * move; the running-helper cases below therefore compare against transforms+ * that return their input unchanged so a regression points at the helper.+ * `spec:AST-036` FR23's absolute budgets stay open until that protocol cost+ * and the remaining helpers land.*/import {describe, expect, it} from 'vitest';-import {parseMarkdown} from './parser';-import {createMarkdownPlugin} from './plugins';-import type {MarkdownExtensionNode} from './plugins';-import type {MarkdownTransform} from './plugins/protocol';+import {parseMarkdown, parseMarkdownAst} from './parser';+import {+ applyMarkdownTransforms,+ createMarkdownPlugin,+ prepareMarkdownPlugins,+ type MarkdownExtensionNode,+ type MarkdownTransform,+} from './plugins/protocol';import {createMarkdownFenceTransform} from './plugins/semanticFence';+import {createMarkdownSourceDecoration} from './plugins/sourceDecoration';import {createMarkdownTextTransform} from './plugins/textTransform';+let benchmarkSink = 0;let transformCalls = 0;function benchmarkDocument(sections: number): string {@@ -40,6 +58,40 @@ function benchmarkDocument(sections: number): string {).join('\n\n');}+function median(values: ReadonlyArray<number>): number {+ const sorted = [...values].sort((left, right) => left - right);+ return sorted[Math.floor(sorted.length / 2)];+}++function measureAverage(callback: () => number): number {+ const start = performance.now();+ for (let iteration = 0; iteration < 20; iteration++) {+ benchmarkSink ^= callback();+ }+ return (performance.now() - start) / 20;+}++function pairedMedianRatio(+ baseline: () => number,+ candidate: () => number,+): number {+ for (let warmup = 0; warmup < 10; warmup++) {+ benchmarkSink ^= baseline();+ benchmarkSink ^= candidate();+ }+ const ratios: number[] = [];+ for (let round = 0; round < 9; round++) {+ const baselineFirst = round % 2 === 0;+ const first = measureAverage(baselineFirst ? baseline : candidate);+ const second = measureAverage(baselineFirst ? candidate : baseline);+ const baselineTime = baselineFirst ? first : second;+ const candidateTime = baselineFirst ? second : first;+ ratios.push(candidateTime / baselineTime);+ }+ void benchmarkSink;+ return median(ratios);+}+function observeTransformCalls(transform: MarkdownTransform,): MarkdownTransform {@@ -99,11 +151,146 @@ const zeroWorkPlugins = [},},}),+ ...Array.from({length: 5}, (_, index) =>+ createMarkdownPlugin({+ name: `zero-work-decoration-${index}`,+ apiVersion: 1,+ transform: observeTransformCalls(+ createMarkdownSourceDecoration({+ name: `zero-work-decoration-${index}`,+ ranges: [{start: 10_000_000 + index, end: 10_000_100 + index}],+ }),+ ),+ }),+ ),];+/**+ * Five decoration helpers that claim the source and sweep the whole block+ * list, but whose ranges sit in the blank lines between blocks and therefore+ * record nothing.+ */+function gapDecorationPlugins(+ source: string,+ count: number,+ rangesPerHelper: number,+ prefix: string,+) {+ const gaps: {start: number; end: number}[] = [];+ for (+ let gap = source.indexOf('\n\n');+ gap >= 0 && gaps.length < rangesPerHelper;+ gap = source.indexOf('\n\n', gap + 1)+ ) {+ gaps.push({start: gap, end: gap + 1});+ }+ if (gaps.length < rangesPerHelper) {+ throw new Error('Fixture has too few block gaps for this case');+ }+ return Array.from({length: count}, (_, index) =>+ createMarkdownPlugin({+ name: `${prefix}-${index}`,+ apiVersion: 1,+ transform: createMarkdownSourceDecoration({+ name: `${prefix}-${index}`,+ ranges: gaps,+ }),+ }),+ );+}++/**+ * Transforms that return their input unchanged. Comparing a helper against+ * these isolates the helper's own dispatch from the fixed per-parse cost Core+ * pays for any running transform (heading marking, freezing, validation),+ * which `spec:AST-036` FR23's absolute budgets also cover but which no helper+ * change can move.+ */+function identityPlugins(count: number, prefix: string) {+ return Array.from({length: count}, (_, index) =>+ createMarkdownPlugin({+ name: `${prefix}-${index}`,+ apiVersion: 1,+ transform: root => root,+ }),+ );+}++const identityFivePlugins = identityPlugins(5, 'identity-five');++/** Matches a prose identifier in every fixture section. */+const identifierPlugin = createMarkdownPlugin({+ name: 'representative-identifiers',+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern: /\bAST-\d+\b/g,+ requiredSubstrings: ['AST-'],+ replace: match => ({type: 'text', value: match[0].toLowerCase()}),+ }),+});++/** Turns the ordinary text fence each fixture section carries into data. */+type PerfFenceNode = MarkdownExtensionNode<+ 'representative-fences',+ 'fence',+ {readonly code: string},+ 'block'+>;++const fencePlugin = createMarkdownPlugin<+ 'representative-fences',+ PerfFenceNode+>({+ name: 'representative-fences',+ apiVersion: 1,+ transform: createMarkdownFenceTransform<readonly ['text'], PerfFenceNode>({+ languages: ['text'],+ createNode: ({code}) => ({+ type: 'extension',+ plugin: 'representative-fences',+ name: 'fence',+ display: 'block',+ data: {code},+ }),+ }),+ renderers: {+ fence: {+ render: ({node}) => node.data.code,+ toText: node => node.data.code,+ },+ },+});++/** Decorates one known source range: the first section heading. */+function knownRangeDecorationPlugin(source: string) {+ const heading = source.indexOf('## Section 0');+ return createMarkdownPlugin({+ name: 'representative-decoration',+ apiVersion: 1,+ transform: createMarkdownSourceDecoration({+ name: 'representative-decoration',+ ranges: [{start: heading, end: heading + '## Section 0'.length}],+ }),+ });+}++// Each case runs the paired protocol twice (10 warmups, then 9 rounds of two+// 20-iteration averages per side), so a 500-section case parses ~1500 times.+// The bounds below are the measured cost plus generous headroom for a busy+// machine; they exist so a hang fails loudly instead of stalling the suite.+const TIMEOUT_MS: Readonly<Record<number, number>> = {+ 200: 60_000,+ 500: 180_000,+};+describe('Markdown helper performance', () => {+ // The skip path's real contract is a work count, not a clock: an unclaimed+ // helper's transform must never be entered, for a stable list and for a+ // recreated equivalent one. Asserting the count directly is exact and+ // immune to what else the machine is running; the timing cases below cover+ // the helpers that do run.it.each([200, 500])(- 'skips five hinted no-claim helpers at %i sections',+ 'skips ten hinted no-claim helpers at %i sections',sections => {const source = benchmarkDocument(sections);const baseline = parseMarkdown(source);@@ -117,5 +304,186 @@ describe('Markdown helper performance', () => {);expect(transformCalls).toBe(0);},+ TIMEOUT_MS[500],+ );++ it.each([200, 500])(+ 'adds at most 15 percent over plain transforms for five sweeping decorations at %i sections',+ sections => {+ const source = benchmarkDocument(sections);+ const sweeping = gapDecorationPlugins(source, 5, 1, 'gap-decoration');+ const ratio = pairedMedianRatio(+ () => parseMarkdown(source, {plugins: identityFivePlugins}).length,+ () => parseMarkdown(source, {plugins: sweeping}).length,+ );+ console.log(+ ` ${sections} sections, sweeping decoration over identity: ${ratio.toFixed(3)}`,+ );+ expect(ratio).toBeLessThanOrEqual(1.15);+ },+ TIMEOUT_MS[500],+ );++ it.each([200, 500])(+ 'costs the ranges it carries, not blocks times ranges, at %i sections',+ sections => {+ const source = benchmarkDocument(sections);+ // The sweep admits and retires each range once, so doubling the+ // ranges over the same document should roughly double the sweep's+ // own work — not square it. Both sides walk every block and decorate+ // nothing, so range count is the only thing that differs, and the+ // bound sits far below the 4x a blocks-times-ranges implementation+ // would show.+ const gaps: {start: number; end: number}[] = [];+ for (+ let gap = source.indexOf('\n\n');+ gap >= 0;+ gap = source.indexOf('\n\n', gap + 1)+ ) {+ gaps.push({start: gap, end: gap + 1});+ }+ const doubled = gaps.flatMap(gap => [+ gap,+ {start: gap.start, end: gap.end + 1},+ ]);+ const plugin = (+ name: string,+ ranges: ReadonlyArray<{start: number; end: number}>,+ ) => [+ createMarkdownPlugin({+ name: `${name}-${sections}`,+ apiVersion: 1,+ transform: createMarkdownSourceDecoration({name, ranges}),+ }),+ ];++ // Timed on the transform stage alone and compared at the FASTEST+ // sample of each side: this is a few hundred microseconds against a+ // parse of milliseconds, so measuring the whole parse buries it, and+ // a minimum is the least-disturbed sample each side achieved.+ const root = parseMarkdownAst(source);+ const fastestTransform = (+ plugins: ReadonlyArray<ReturnType<typeof createMarkdownPlugin>>,+ ): number => {+ const prepared = prepareMarkdownPlugins(plugins);+ let fastest = Number.POSITIVE_INFINITY;+ for (let sample = 0; sample < 60; sample++) {+ const start = performance.now();+ benchmarkSink ^= applyMarkdownTransforms(+ root,+ prepared,+ source,+ true,+ 'block',+ ).children.length;+ fastest = Math.min(fastest, performance.now() - start);+ }+ void benchmarkSink;+ return fastest;+ };++ const single = plugin('scale-single', gaps);+ const double = plugin('scale-double', doubled);+ // Warm both before either is recorded.+ fastestTransform(single);+ fastestTransform(double);+ const ratio = fastestTransform(double) / fastestTransform(single);+ console.log(+ ` ${sections} sections, ${doubled.length} ranges over ${gaps.length}: ${ratio.toFixed(3)}`,+ );+ expect(ratio).toBeLessThanOrEqual(2.6);+ },+ TIMEOUT_MS[500],+ );++ it.each([200, 500])(+ 'adds at most 15 percent to a matching prose helper when it decorates a known range at %i sections',+ sections => {+ const source = benchmarkDocument(sections);+ // The baseline's second transform also returns a replacement root, so+ // both sides pay Core's validation of a changed tree and the ratio+ // reports the decoration walk itself.+ const withoutDecoration = [+ identifierPlugin,+ createMarkdownPlugin({+ name: `representative-replacement-${sections}`,+ apiVersion: 1,+ transform: root => ({...root}),+ }),+ ];+ const withDecoration = [+ identifierPlugin,+ knownRangeDecorationPlugin(source),+ ];+ const ratio = pairedMedianRatio(+ () => parseMarkdown(source, {plugins: withoutDecoration}).length,+ () => parseMarkdown(source, {plugins: withDecoration}).length,+ );+ console.log(+ ` ${sections} sections, known-range decoration over prose helper: ${ratio.toFixed(3)}`,+ );+ expect(ratio).toBeLessThanOrEqual(1.15);+ },+ TIMEOUT_MS[500],+ );++ it.each([200, 500])(+ 'keeps the representative three-helper set within 25 percent of three plain transforms at %i sections',+ sections => {+ const source = benchmarkDocument(sections);+ // The same set against three transforms that each return a replacement+ // root. Both sides then pay Core's per-transform guards, so this ratio+ // reports what the three helpers themselves cost — the part a helper+ // change can actually move.+ const baseline = Array.from({length: 3}, (_, index) =>+ createMarkdownPlugin({+ name: `representative-baseline-${sections}-${index}`,+ apiVersion: 1,+ transform: root => ({...root}),+ }),+ );+ const representative = [+ identifierPlugin,+ fencePlugin,+ knownRangeDecorationPlugin(source),+ ];+ const ratio = pairedMedianRatio(+ () => parseMarkdown(source, {plugins: baseline}).length,+ () => parseMarkdown(source, {plugins: representative}).length,+ );+ console.log(+ ` ${sections} sections, representative set over three transforms: ${ratio.toFixed(3)}`,+ );+ expect(ratio).toBeLessThanOrEqual(1.25);+ },+ TIMEOUT_MS[500],+ );++ it.each([200, 500])(+ 'keeps a trusted helper far below an equivalent plugin-authored transform at %i sections',+ sections => {+ const source = benchmarkDocument(sections);+ // Trusted Core helpers skip the guards Core applies to plugin-authored+ // output. Comparing the decoration helper against a plugin transform+ // that returns an equivalent replacement root measures that saving+ // without depending on absolute machine speed.+ const pluginAuthored = createMarkdownPlugin({+ name: `trusted-comparison-${sections}`,+ apiVersion: 1,+ transform: root => ({...root, children: [...root.children]}),+ });+ const ratio = pairedMedianRatio(+ () => parseMarkdown(source, {plugins: [pluginAuthored]}).length,+ () =>+ parseMarkdown(source, {+ plugins: [knownRangeDecorationPlugin(source)],+ }).length,+ );+ console.log(+ ` ${sections} sections, trusted decoration over plugin-authored transform: ${ratio.toFixed(3)}`,+ );+ expect(ratio).toBeLessThanOrEqual(0.85);+ },+ TIMEOUT_MS[500],);});diff --git a/packages/core/src/Markdown/ast.test.ts b/packages/core/src/Markdown/ast.test.tsindex 206cf82b1a..9ec45c40da 100644--- a/packages/core/src/Markdown/ast.test.ts+++ b/packages/core/src/Markdown/ast.test.ts@@ -6,6 +6,55 @@ import {parseMarkdown,parseMarkdownAst,} from './parser';+import {createMarkdownPlugin} from './plugins';+import type {+ MarkdownExtensionNode,+ MarkdownSyntaxPluginDefinition,+} from './plugins';++type DemoNoteNode = MarkdownExtensionNode<+ 'demo-notes',+ 'note',+ {readonly body: string},+ 'block'+>;++const demoNotes = createMarkdownPlugin<'demo-notes', DemoNoteNode>({+ name: 'demo-notes',+ apiVersion: 1,+ parseKey: 'v1',+ syntax: {+ block: [+ {+ startsWith: [':::note'],+ maxSpan: 200,+ tokenize({source, offset, end, isFinal}) {+ const close = source.indexOf('\n:::', offset + 7);+ if (close < 0 || close + 4 > end) {+ return isFinal ? {status: 'no-match'} : {status: 'defer'};+ }+ return {+ status: 'match',+ end: close + 4,+ node: {+ type: 'extension',+ plugin: 'demo-notes',+ name: 'note',+ display: 'block',+ data: {body: source.slice(offset + 7, close).trim()},+ },+ };+ },+ },+ ],+ },+ renderers: {+ note: {+ render: ({node}) => node.data.body,+ toText: node => node.data.body,+ },+ },+} satisfies MarkdownSyntaxPluginDefinition<'demo-notes', DemoNoteNode>);describe('canonical Markdown AST', () => {it('uses MDAST-aligned names and fields for every built-in structure', () => {@@ -88,7 +137,7 @@ describe('canonical Markdown AST', () => {},{type: 'math', value: 'y'},]);- expect(parseMarkdownAst('```\nplain\n```').children[0]).toEqual({+ expect(parseMarkdownAst('```\nplain\n```').children[0]).toMatchObject({type: 'code',lang: null,value: 'plain',@@ -118,6 +167,95 @@ describe('canonical Markdown AST', () => {expect('position' in legacy[0]).toBe(false);});+ it('authors canonical positions without the sourceRanges option', () => {+ const source = '# Heading\n\nParagraph\n\n> Quoted';+ const ast = parseMarkdownAst(source);++ // Transforms observe provenance on every top-level block whether or not+ // the caller opted into the released `range` projection.+ expect(ast.children.map(block => block.position)).toEqual([+ {start: {offset: 0}, end: {offset: 9}},+ {start: {offset: 11}, end: {offset: 20}},+ {start: {offset: 22}, end: {offset: 30}},+ ]);+ expect(+ parseMarkdownAst(source, {sourceRanges: true}).children.map(+ block => block.position,+ ),+ ).toEqual(ast.children.map(block => block.position));++ // Nested blocks remain a top-level-only contract.+ const quote = ast.children[2];+ if (quote.type !== 'blockquote') {+ throw new Error('Expected blockquote');+ }+ expect(quote.children[0].position).toBeUndefined();++ // The released projection is unchanged: no `range` unless asked for.+ expect(parseMarkdown(source)).toEqual([+ {+ type: 'heading',+ level: 1,+ children: [{type: 'text', content: 'Heading'}],+ },+ {type: 'paragraph', children: [{type: 'text', content: 'Paragraph'}]},+ {+ type: 'blockquote',+ children: [+ {type: 'paragraph', children: [{type: 'text', content: 'Quoted'}]},+ ],+ },+ ]);+ });++ it('keeps canonical positions off released extension nodes', () => {+ const source = ':::note\nBody\n:::\n\nTail prose.';+ const plugins = [demoNotes];++ // The projection rebuilds every built-in block field by field, but passes+ // an extension node through — so it is the one kind that could carry a+ // canonical position out. Without the option the released node is exactly+ // what it has always been.+ const released = parseMarkdown(source, {plugins});+ expect(released[0]).toEqual({+ type: 'extension',+ plugin: 'demo-notes',+ name: 'note',+ display: 'block',+ data: {body: 'Body'},+ source: ':::note\nBody\n:::',+ });+ expect('position' in released[0]).toBe(false);+ expect(Object.keys(released[0])).toEqual([+ 'type',+ 'plugin',+ 'name',+ 'display',+ 'data',+ 'source',+ ]);++ // With the option the node keeps the exact provenance it has always+ // exposed: `position` with offsets, and no `range`.+ const withRanges = parseMarkdown(source, {sourceRanges: true, plugins});+ expect(withRanges[0]).toEqual({+ type: 'extension',+ plugin: 'demo-notes',+ name: 'note',+ display: 'block',+ data: {body: 'Body'},+ source: ':::note\nBody\n:::',+ position: {start: {offset: 0}, end: {offset: 16}},+ });+ expect('range' in withRanges[0]).toBe(false);++ // The canonical tree carries provenance either way.+ expect(parseMarkdownAst(source, {plugins}).children[0].position).toEqual({+ start: {offset: 0},+ end: {offset: 16},+ });+ });+it('preserves optional legacy keys and insertion order exactly', () => {const [image, list] = parseMarkdown('\n\n- item');diff --git a/packages/core/src/Markdown/parser.ts b/packages/core/src/Markdown/parser.tsindex 2a4f95173a..dc31ceb7cd 100644--- a/packages/core/src/Markdown/parser.ts+++ b/packages/core/src/Markdown/parser.ts@@ -194,9 +194,32 @@ function projectRange(return start == null || end == null ? {} : {range: {start, end}};}+/**+ * Canonical nodes always carry Core-authored positions; the released block+ * shape exposes them as `range` only when the caller passed `sourceRanges`,+ * so this projection stays byte-for-byte what it has always returned.+ *+ * An extension node is the one kind the projection passes through instead of+ * rebuilding field by field, so it is the one kind that could carry a+ * canonical `position` out into released output. Strip it unless the caller+ * asked for provenance, in which case the node keeps exactly the `position`+ * it has always exposed.+ */+function projectExtensionNode<Node extends RuntimeExtensionNode>(+ node: Node,+ withRanges: boolean,+): Node {+ if (withRanges || node.position === undefined) {+ return node;+ }+ const {position: _canonicalPosition, ...released} = node;+ return released as Node;+}+function projectInlineNode(node: MarkdownAstPhrasingContent<RuntimeExtensionNode>,cache: LegacyProjectionCache,+ withRanges: boolean,): RuntimeInlineNode {const cached = cache.get(node);if (cached != null) {@@ -210,19 +233,25 @@ function projectInlineNode(case 'strong':projected = {type: 'bold',- children: node.children.map(child => projectInlineNode(child, cache)),+ children: node.children.map(child =>+ projectInlineNode(child, cache, withRanges),+ ),};break;case 'emphasis':projected = {type: 'italic',- children: node.children.map(child => projectInlineNode(child, cache)),+ children: node.children.map(child =>+ projectInlineNode(child, cache, withRanges),+ ),};break;case 'delete':projected = {type: 'strikethrough',- children: node.children.map(child => projectInlineNode(child, cache)),+ children: node.children.map(child =>+ projectInlineNode(child, cache, withRanges),+ ),};break;case 'inlineCode':@@ -235,7 +264,9 @@ function projectInlineNode(projected = {type: 'link',href: node.url,- children: node.children.map(child => projectInlineNode(child, cache)),+ children: node.children.map(child =>+ projectInlineNode(child, cache, withRanges),+ ),};break;case 'image':@@ -248,7 +279,7 @@ function projectInlineNode(projected = {type: 'break'};break;case 'extension':- projected = node;+ projected = projectExtensionNode(node, withRanges);break;}cache.set(node, projected);@@ -258,13 +289,16 @@ function projectInlineNode(function projectTableCell(node: MarkdownAstTableCell<RuntimeExtensionNode>,cache: LegacyProjectionCache,+ withRanges: boolean,): TableCellNodeWithMath<RuntimeExtensionNode> {const cached = cache.get(node);if (cached != null) {return cached as TableCellNodeWithMath<RuntimeExtensionNode>;}const projected = {- children: node.children.map(child => projectInlineNode(child, cache)),+ children: node.children.map(child =>+ projectInlineNode(child, cache, withRanges),+ ),};cache.set(node, projected);return projected;@@ -273,6 +307,7 @@ function projectTableCell(function projectListItem(node: MarkdownAstListItem<RuntimeExtensionNode>,cache: LegacyProjectionCache,+ withRanges: boolean,): ListItemNodeWithMath<RuntimeExtensionNode> {const cached = cache.get(node);if (cached != null) {@@ -280,7 +315,9 @@ function projectListItem(}const projected = {checked: node.checked,- children: node.children.map(child => projectBlockNode(child, cache)),+ children: node.children.map(child =>+ projectBlockNode(child, cache, withRanges),+ ),};cache.set(node, projected);return projected;@@ -289,26 +326,31 @@ function projectListItem(function projectBlockNode(node: MarkdownAstBlockContent<RuntimeExtensionNode>,cache: LegacyProjectionCache,+ withRanges: boolean,): RuntimeBlockNode {const cached = cache.get(node);if (cached != null) {return cached as RuntimeBlockNode;}- const metadata = projectRange(node.position);+ const metadata = withRanges ? projectRange(node.position) : {};let projected: RuntimeBlockNode;switch (node.type) {case 'heading':projected = {type: 'heading',level: node.depth,- children: node.children.map(child => projectInlineNode(child, cache)),+ children: node.children.map(child =>+ projectInlineNode(child, cache, withRanges),+ ),...metadata,};break;case 'paragraph':projected = {type: 'paragraph',- children: node.children.map(child => projectInlineNode(child, cache)),+ children: node.children.map(child =>+ projectInlineNode(child, cache, withRanges),+ ),...metadata,};break;@@ -326,7 +368,9 @@ function projectBlockNode(case 'blockquote':projected = {type: 'blockquote',- children: node.children.map(child => projectBlockNode(child, cache)),+ children: node.children.map(child =>+ projectBlockNode(child, cache, withRanges),+ ),...metadata,};break;@@ -337,7 +381,9 @@ function projectBlockNode(start: node.start,delimiter: node.delimiter,loose: node.spread,- items: node.children.map(item => projectListItem(item, cache)),+ items: node.children.map(item =>+ projectListItem(item, cache, withRanges),+ ),...metadata,};break;@@ -346,10 +392,12 @@ function projectBlockNode(node.children;projected = {type: 'table',- headers: header.children.map(cell => projectTableCell(cell, cache)),+ headers: header.children.map(cell =>+ projectTableCell(cell, cache, withRanges),+ ),alignments: [...node.align],rows: rows.map(row =>- row.children.map(cell => projectTableCell(cell, cache)),+ row.children.map(cell => projectTableCell(cell, cache, withRanges)),),...metadata,};@@ -362,7 +410,7 @@ function projectBlockNode(projected = {type: 'image', alt: node.alt, src: node.url, ...metadata};break;case 'extension':- projected = node;+ projected = projectExtensionNode(node, withRanges);break;}cache.set(node, projected);@@ -371,16 +419,29 @@ function projectBlockNode(function projectInlineNodes(nodes: ReadonlyArray<MarkdownAstPhrasingContent<RuntimeExtensionNode>>,+ withRanges: boolean,cache: LegacyProjectionCache = new WeakMap(),): RuntimeInlineNode[] {- return nodes.map(node => projectInlineNode(node, cache));+ return nodes.map(node => projectInlineNode(node, cache, withRanges));}function projectMarkdownRoot(root: MarkdownAstRoot<RuntimeExtensionNode>,+ withRanges: boolean,cache: LegacyProjectionCache = new WeakMap(),): RuntimeBlockNode[] {- return root.children.map(node => projectBlockNode(node, cache));+ return root.children.map(node => projectBlockNode(node, cache, withRanges));+}++/** Whether a caller asked for the released `range` field on its blocks. */+function wantsLegacyRanges(+ arg: ReadonlySet<string> | RuntimeParseOptions | undefined,+): boolean {+ return (+ arg != null &&+ typeof (arg as {has?: unknown}).has !== 'function' &&+ (arg as RuntimeParseOptions).sourceRanges === true+ );}// ---------------------------------------------------------------------------@@ -465,7 +526,15 @@ type ResolvedOptions = {readonly sourceIds: ReadonlySet<string> | undefined;readonly autolink: 'gfm' | undefined;readonly math?: boolean;+ /** Whether the caller asked for the released `range` projection. */readonly sourceRanges?: boolean;+ /**+ * Whether this parse authors canonical `position` offsets on its blocks.+ * Internal only, and independent of `sourceRanges`: the canonical tree+ * transforms observe always carries provenance, while `sourceRanges` alone+ * decides whether the released projection exposes it as `range`.+ */+ readonly astPositions?: boolean;readonly plugins?: PreparedMarkdownPlugins;readonly isFinal: boolean;readonly allowBlockSyntax?: boolean;@@ -484,18 +553,42 @@ type ResolvedOptions = {readonly linkDefs?: ReadonlyMap<string, string>;};-const EMPTY_OPTS: ResolvedOptions = {- sourceIds: undefined,- autolink: undefined,- isFinal: true,-};+/**+ * Every resolved options object is built here, so all of them share one key+ * set in one order — and therefore one hidden class. The parser reads these+ * fields in its hottest loops; letting an omitted-options parse and an+ * options-bag parse produce differently-shaped objects makes those reads+ * polymorphic in any process that does both, which is exactly what a caller+ * comparing an empty pipeline against a configured one does.+ */+function makeResolvedOptions(+ fields: Partial<ResolvedOptions> & {readonly isFinal: boolean},+): ResolvedOptions {+ return {+ sourceIds: fields.sourceIds,+ autolink: fields.autolink,+ math: fields.math,+ sourceRanges: fields.sourceRanges,+ astPositions: fields.astPositions ?? true,+ plugins: fields.plugins,+ isFinal: fields.isFinal,+ allowBlockSyntax: fields.allowBlockSyntax ?? true,+ baseOffset: fields.baseOffset,+ linkDefs: fields.linkDefs,+ };+}++const EMPTY_OPTS: ResolvedOptions = makeResolvedOptions({isFinal: true});+const EMPTY_INCREMENTAL_OPTS: ResolvedOptions = makeResolvedOptions({+ isFinal: false,+});function resolveOptions(arg: ReadonlySet<string> | RuntimeParseOptions | undefined,incremental = false,): ResolvedOptions {if (arg == null) {- return incremental ? {...EMPTY_OPTS, isFinal: false} : EMPTY_OPTS;+ return incremental ? EMPTY_INCREMENTAL_OPTS : EMPTY_OPTS;}// Duck-type the legacy `ReadonlySet<string>` form: any object whose// `.has` is callable is treated as the legacy sourceIds set. This is@@ -503,14 +596,13 @@ function resolveOptions(// or polyfilled `ReadonlySet` implementations as a `ParseOptions` bag// and silently lose citation resolution.if (typeof (arg as {has?: unknown}).has === 'function') {- return {+ return makeResolvedOptions({sourceIds: arg as ReadonlySet<string>,- autolink: undefined,isFinal: !incremental,- };+ });}const opts = arg as RuntimeParseOptions;- return {+ return makeResolvedOptions({sourceIds: opts.sourceIds,autolink: opts.autolink,math: opts.math === true ? true : undefined,@@ -520,8 +612,7 @@ function resolveOptions(? prepareMarkdownPlugins(opts.plugins): undefined,isFinal: incremental ? opts.isFinal === true : true,- allowBlockSyntax: true,- };+ });}// ---------------------------------------------------------------------------@@ -1265,7 +1356,7 @@ export function parseInline(text: string,arg?: ReadonlySet<string> | RuntimeParseOptions,): RuntimeInlineNode[] {- return projectInlineNodes(parseInlineAst(text, arg));+ return projectInlineNodes(parseInlineAst(text, arg), wantsLegacyRanges(arg));}/** @internal Canonical inline parse used by Markdown rendering. */@@ -1311,6 +1402,14 @@ function parseInlineImpl(opts: ResolvedOptions,): MarkdownAstPhrasingContent<RuntimeExtensionNode>[] {const nodes: MarkdownAstPhrasingContent<RuntimeExtensionNode>[] = [];+ // Only a plugin that actually contributes INLINE syntax may cost anything+ // per source position. A transform-only list contributes none, so it takes+ // the same path as an omitted or empty one: no candidate probe per+ // position, and no map lookup per character in the plain-text scan below.+ const inlineExtensionStarts =+ opts.plugins != null && opts.plugins.inlineByFirstCharacter.size > 0+ ? opts.plugins.inlineByFirstCharacter+ : undefined;let i = 0;while (i < text.length) {@@ -1529,7 +1628,7 @@ function parseInlineImpl(}// --- Extension syntax (built-ins and protected contexts win) ---- if (opts.plugins != null) {+ if (inlineExtensionStarts !== undefined) {const extension = matchExtensionSyntax(text, i, 'inline', opts);if (extension.status === 'match') {nodes.push(@@ -1549,7 +1648,8 @@ function parseInlineImpl(end < text.length &&!'*_~`[!\\\n\u3010'.includes(text[end]) &&!(opts.math && text[end] === '$') &&- opts.plugins?.inlineByFirstCharacter.has(text[end]) !== true+ (inlineExtensionStarts === undefined ||+ !inlineExtensionStarts.has(text[end]))) {end++;}@@ -1963,8 +2063,8 @@ function isTableSeparator(line: string): boolean {function nested(opts: ResolvedOptions): ResolvedOptions {const nestedOptions =opts.allowBlockSyntax === false ? opts : {...opts, allowBlockSyntax: false};- return nestedOptions.sourceRanges- ? {...nestedOptions, sourceRanges: false}+ return nestedOptions.sourceRanges || nestedOptions.astPositions+ ? {...nestedOptions, sourceRanges: false, astPositions: false}: nestedOptions;}@@ -2219,7 +2319,10 @@ export function parseMarkdown(input: string,arg?: ReadonlySet<string> | RuntimeParseOptions,): RuntimeBlockNode[] {- return projectMarkdownRoot(parseMarkdownAst(input, arg));+ return projectMarkdownRoot(+ parseMarkdownAst(input, arg),+ wantsLegacyRanges(arg),+ );}/** @internal Canonical block parse used by Markdown and Outline rendering. */@@ -2270,12 +2373,14 @@ function parseMarkdownImpl(const hasBlockExtensionSyntax =opts.allowBlockSyntax !== false &&(opts.plugins?.blockByFirstCharacter.size ?? 0) > 0;+ // Derived from the already-split lines rather than a second character+ // scan: every line contributes its own length plus the newline it ended on.const lineOffsets = [0];- if (opts.sourceRanges || hasBlockExtensionSyntax) {- for (let offset = 0; offset < cleaned.length; offset++) {- if (cleaned[offset] === '\n') {- lineOffsets.push(offset + 1);- }+ if (opts.astPositions === true || hasBlockExtensionSyntax) {+ let offset = 0;+ for (let index = 0; index < lines.length; index++) {+ offset += lines[index].length + 1;+ lineOffsets.push(offset);}}const blockExtensionMatches = new Map<number, ExtensionMatch>();@@ -2283,12 +2388,12 @@ function parseMarkdownImpl(// The line each block started on, parallel to `blocks`. Only collected when// ranges were asked for; a block's end is resolved after the loop, since the// branch that produced it has already moved `index` past whatever it read.- const blockStartLines: number[] | null = opts.sourceRanges ? [] : null;+ const blockStartLines: number[] | null =+ opts.astPositions === true ? [] : null;// Set only by a block that consumes blank lines as content, where the// positional end derivation would trim them away.- const blockEndLines: (number | undefined)[] | null = opts.sourceRanges- ? []- : null;+ const blockEndLines: (number | undefined)[] | null =+ opts.astPositions === true ? [] : null;let blockStartLine = 0;const pushBlock = (node: MarkdownAstBlockContent<RuntimeExtensionNode>,@@ -3166,7 +3271,7 @@ function trimUnsettledStructural(text: string): string {* rather than into the slice.*/function atOffset(opts: ResolvedOptions, offset: number): ResolvedOptions {- return opts.sourceRanges ? {...opts, baseOffset: offset} : opts;+ return opts.astPositions === true ? {...opts, baseOffset: offset} : opts;}/**@@ -3349,12 +3454,17 @@ export function parseMarkdownIncremental(state: IncrementalState<boolean>,arg?: ReadonlySet<string> | RuntimeParseOptions,): RuntimeBlockNode[] {+ const withRanges = wantsLegacyRanges(arg);const root = parseMarkdownAstIncremental(input, state, arg);const cache = incrementalCaches.get(state) ?? makeIncrementalCache(state);- const projected = projectMarkdownRoot(root, cache.projectionCache);+ const projected = projectMarkdownRoot(+ root,+ withRanges,+ cache.projectionCache,+ );if (cache.projectedRevision !== cache.settledRevision) {cache.projectedSettledBlocks = cache.settledAstBlocks.map(block =>- projectBlockNode(block, cache.projectionCache),+ projectBlockNode(block, cache.projectionCache, withRanges),);cache.projectedRevision = cache.settledRevision;}@@ -3516,6 +3626,7 @@ function parseMarkdownIncrementalAstBlocks(? trimOpenDisplayMath(unsettledRaw): trimUnsettledStructural(unsettledRaw);+ const legacyRanges = opts.sourceRanges === true;let parsedSettledBlocks = 0;if (reparseSettled) {cache.settledAstBlocks = state.settledText@@ -3524,7 +3635,7 @@ function parseMarkdownIncrementalAstBlocks(parsedSettledBlocks = cache.settledAstBlocks.length;cache.settledRevision++;cache.projectedSettledBlocks = cache.settledAstBlocks.map(block =>- projectBlockNode(block, cache.projectionCache),+ projectBlockNode(block, cache.projectionCache, legacyRanges),);cache.projectedRevision = cache.settledRevision;} else if (settledDelta !== '') {@@ -3537,13 +3648,14 @@ function parseMarkdownIncrementalAstBlocks(parsedSettledBlocks = deltaBlocks.length;cache.settledRevision++;const projectedDelta = deltaBlocks.map(block =>- projectBlockNode(block, cache.projectionCache),+ projectBlockNode(block, cache.projectionCache, legacyRanges),);if (mergedList) {const canonicalMerged = cache.settledAstBlocks[mergeIndex];const projectedMerged = projectBlockNode(canonicalMerged,cache.projectionCache,+ legacyRanges,);cache.projectedSettledBlocks[mergeIndex] = projectedMerged;cache.projectedSettledBlocks.push(...projectedDelta.slice(1));@@ -3555,11 +3667,11 @@ function parseMarkdownIncrementalAstBlocks(// The unsettled tail is trimmed before parsing, so its offset in the// document is where that trimmed text actually starts — not the boundary,- // which is a line index. If it somehow can't be located, parse it without- // ranges rather than report wrong ones. Only worth searching for when- // ranges were asked for: this runs on every streamed chunk.+ // which is a line index. If it somehow cannot be located, parse it without+ // positions rather than report wrong ones. The search starts at the+ // settled end, so it scans the tail and never the prefix.const unsettledStart =- unsettledText && opts.sourceRanges+ unsettledText && opts.astPositions === true? input.indexOf(unsettledText, cache.settledEnd): -1;const unsettledBlocks = unsettledTextdiff --git a/packages/core/src/Markdown/plugins/index.ts b/packages/core/src/Markdown/plugins/index.tsindex 3a57191455..3e3209ea6a 100644--- a/packages/core/src/Markdown/plugins/index.ts+++ b/packages/core/src/Markdown/plugins/index.ts@@ -18,6 +18,15 @@ export type {MarkdownFenceNode,MarkdownFenceTransformOptions,} from './semanticFence';+export {+ createMarkdownSourceDecoration,+ getMarkdownSourceDecorations,+} from './sourceDecoration';+export type {+ MarkdownSourceDecoration,+ MarkdownSourceDecorationOptions,+ MarkdownSourceDecorationRange,+} from './sourceDecoration';export type {MarkdownPluginData,MarkdownExtensionNode,diff --git a/packages/core/src/Markdown/plugins/protocol.ts b/packages/core/src/Markdown/plugins/protocol.tsindex 09e1d8ad11..faa5f5cb19 100644--- a/packages/core/src/Markdown/plugins/protocol.ts+++ b/packages/core/src/Markdown/plugins/protocol.ts@@ -102,9 +102,15 @@ export interface MarkdownTransformContext {}const markdownTransformPluginName = Symbol('MarkdownTransformPluginName');+const markdownTransformHasRenderer = Symbol('MarkdownTransformHasRenderer');+const markdownTransformSourceUnchanged = Symbol(+ 'MarkdownTransformSourceUnchanged',+);type InternalMarkdownTransformContext = MarkdownTransformContext & {readonly [markdownTransformPluginName]: string;+ readonly [markdownTransformHasRenderer]: (nodeName: string) => boolean;+ readonly [markdownTransformSourceUnchanged]: boolean;};/** @internal Reads the owning plugin name without expanding the public context. */@@ -116,6 +122,32 @@ export function getMarkdownTransformPluginName(];}+/**+ * @internal The ownership facts a Core helper needs to validate the nodes a+ * caller's callback handed it, without seeing the rest of the pipeline.+ */+export interface MarkdownHelperOwnership {+ readonly pluginName: string;+ readonly hasRenderer: (nodeName: string) => boolean;+ /** Whether source text still describes the current transform input. */+ readonly sourceUnchanged: boolean;+}++/** @internal Reads this transform's ownership, when Core is running it. */+export function getMarkdownHelperOwnership(+ context: MarkdownTransformContext,+): MarkdownHelperOwnership | undefined {+ const internal = context as Partial<InternalMarkdownTransformContext>;+ const pluginName = internal[markdownTransformPluginName];+ const hasRenderer = internal[markdownTransformHasRenderer];+ const sourceUnchanged = internal[markdownTransformSourceUnchanged];+ return pluginName === undefined ||+ hasRenderer === undefined ||+ sourceUnchanged === undefined+ ? undefined+ : {pluginName, hasRenderer, sourceUnchanged};+}+declare const markdownTransformNode: unique symbol;export interface MarkdownTransform<Node extends MarkdownExtensionNode = never> {@@ -127,6 +159,7 @@ export interface MarkdownTransform<Node extends MarkdownExtensionNode = never> {}const markdownTransformClaim = Symbol('MarkdownTransformClaim');+const markdownTransformTrusted = Symbol('MarkdownTransformTrusted');interface ClaimAwareMarkdownTransform<Node extends MarkdownExtensionNode = MarkdownExtensionNode,@@ -134,6 +167,34 @@ interface ClaimAwareMarkdownTransform<readonly [markdownTransformClaim]?: (source: string) => boolean;}+interface TrustAwareMarkdownTransform extends MarkdownTransform<never> {+ readonly [markdownTransformTrusted]?: true;+}++/**+ * @internal Marks a Core-authored transform whose OUTPUT STRUCTURE is entirely+ * Core-computed — it never inserts a caller-supplied node, never edits an+ * existing node's meaning, provenance, or ownership, and never mutates its+ * input. Core then skips the validation and immutability guards it applies to+ * plugin-authored output, whose purpose is to contain untrusted structure.+ *+ * Only helpers in this package may be marked, and only when every node they+ * emit is built here from validated inputs. A helper that inserts anything the+ * caller supplied — the text helper's replacement nodes, for example — is NOT+ * trusted: Core must still validate the tree it returns.+ */+export function markMarkdownTransformTrusted(+ transform: MarkdownTransform<never>,+): MarkdownTransform<never> {+ Object.defineProperty(transform, markdownTransformTrusted, {+ configurable: false,+ enumerable: false,+ value: true,+ writable: false,+ });+ return transform;+}+/** @internal Registers a cheap source-level claim check for a helper transform. */export function markMarkdownTransformClaim<Node extends MarkdownExtensionNode = never,@@ -369,6 +430,8 @@ interface PreparedTransform {readonly pluginName: string;readonly transform: MarkdownTransform<MarkdownExtensionNode>;readonly claims?: (source: string) => boolean;+ /** Core-authored helper whose output needs no plugin-output validation. */+ readonly trusted: boolean;}interface PreparedRenderer {@@ -388,6 +451,8 @@ export interface PreparedMarkdownPlugins {ReadonlyArray<PreparedSyntaxContribution>>;readonly transforms: ReadonlyArray<PreparedTransform>;+ /** Whether any transform is plugin-authored and needs Core's guards. */+ readonly hasUntrustedTransform: boolean;readonly syntaxIdentity: string;readonly renderers: ReadonlyMap<string, PreparedRenderer>;}@@ -465,6 +530,10 @@ export function prepareMarkdownPlugins(claims: (definition.transform as ClaimAwareMarkdownTransform)[markdownTransformClaim],+ trusted:+ (definition.transform as TrustAwareMarkdownTransform)[+ markdownTransformTrusted+ ] === true,});}if (definition.renderers != null) {@@ -513,6 +582,7 @@ export function prepareMarkdownPlugins(inlineByFirstCharacter,blockByFirstCharacter,transforms,+ hasUntrustedTransform: transforms.some(entry => !entry.trusted),syntaxIdentity: syntaxIdentity.join('\u0001'),renderers,};@@ -1098,27 +1168,294 @@ function validateAst();}-const deeplyFrozenAstValues = new WeakSet<object>();--function freezeAst<T>(value: T, seen: Set<object> = new Set()): T {- if (- value != null &&- typeof value === 'object' &&- !seen.has(value) &&- !deeplyFrozenAstValues.has(value)- ) {- seen.add(value);- for (const nested of Object.values(value)) {- freezeAst(nested, seen);+/**+ * Deeply freezes a Markdown tree.+ *+ * No memo set and no cycle set. Both were measured against this parser's own+ * fixture and together cost four times the freezing they were guarding: a+ * full freeze of a 12k-node tree runs at 19 percent of the parse that built+ * it, a WeakSet memo takes that to 47 percent, and a cycle set to 75. Their+ * absence is safe, not merely cheap: every tree reaching here is acyclic+ * (the parser builds no cycles, and `validateAst` rejects any plugin tree+ * that contains one), and `Object.freeze` on an already-frozen node is a+ * no-op, so re-walking a shared subtree is correct — just not free.+ *+ * Deliberately NOT skipping nodes that look frozen: a node a plugin froze+ * itself may still hold mutable children, and an undecorated sibling a+ * trusted helper carried over by identity must be frozen like any other.+ */+function freezeAstNode(value: unknown): void {+ if (value == null || typeof value !== 'object') {+ return;+ }+ if (Array.isArray(value)) {+ for (let index = 0; index < value.length; index++) {+ freezeAstNode(value[index]);+ }+ } else {+ // `for…in` over a plain AST node rather than Object.values: the entries+ // array is the dominant remaining cost of the walk.+ for (const key in value) {+ freezeAstNode((value as Record<string, unknown>)[key]);}- Object.freeze(value);- deeplyFrozenAstValues.add(value);}+ Object.freeze(value);+}++function freezeAst<T>(value: T): T {+ freezeAstNode(value);return value;}const reportedProductionPluginFailures = new Set<string>();+/**+ * Freezes only what `next` does not share with `previous`.+ *+ * Valid ONLY when no plugin-authored transform runs in this pipeline. The+ * nodes it skips are the ones a Core helper carried over untouched, and with+ * every transform Core-authored there is no actor that could mutate them:+ * they are left in exactly the state an omitted or empty plugin list leaves+ * them in, which is the state rendering already observes for every+ * zero-plugin document today. The moment one untrusted transform exists,+ * `applyMarkdownTransforms` uses the full `freezeAst` instead, before that+ * transform observes anything — a shared, undecorated sibling included.+ */+function freezeBuiltAst<T>(previous: unknown, next: T): T {+ if (next == null || typeof next !== 'object' || next === previous) {+ return next;+ }+ const before =+ previous != null && typeof previous === 'object'+ ? (previous as Record<string, unknown>)+ : undefined;+ if (Array.isArray(next)) {+ const beforeArray = Array.isArray(previous) ? previous : undefined;+ for (let index = 0; index < next.length; index++) {+ freezeBuiltAst(beforeArray?.[index], next[index]);+ }+ } else {+ for (const key in next) {+ freezeBuiltAst(before?.[key], (next as Record<string, unknown>)[key]);+ }+ }+ Object.freeze(next);+ return next;+}++const HELPER_PHRASING_TYPES = new Set([+ 'text',+ 'strong',+ 'emphasis',+ 'delete',+ 'inlineCode',+ 'inlineMath',+ 'break',+ 'link',+ 'image',+ 'citation',+ 'extension',+]);++/**+ * @internal Validates and freezes phrasing nodes a Core helper's caller just+ * authored, so the helper may put them straight into its output tree.+ * applying exactly the rules `validateAst` applies to newly-introduced nodes:+ * representable typed data only, no authored provenance, no foreign extension+ * ownership, no unrenderable extension, revalidated destinations, no nested+ * link, no block content, no cycles, and a bounded node count.+ *+ * A helper that validates and freezes its callback's output this way has+ * nothing left for Core to check, because every other node it returns is one+ * Core already validated. That is what lets such a helper run on the trusted+ * path while keeping FR10 and FR11 intact.+ *+ * Returns the frozen nodes, or throws — the caller reports the failure+ * through the ordinary transform failure path.+ */+export function adoptMarkdownHelperNode(+ node: unknown,+ ownership: MarkdownHelperOwnership,+ insideLink = false,+): void {+ adoptMarkdownHelperNodes(singleNodeScratch(node), ownership, insideLink);+}++/**+ * A one-element view for the single-node case, reused between calls so the+ * common replacement costs no array. Refilled on entry and never held past+ * the synchronous validation below, and a callback that re-enters the+ * helper gets a fresh array rather than disturbing an outer walk.+ */+let scratchInUse = false;+const reusableScratch: unknown[] = [undefined];+function singleNodeScratch(node: unknown): unknown[] {+ if (scratchInUse) {+ return [node];+ }+ reusableScratch[0] = node;+ return reusableScratch;+}++export function adoptMarkdownHelperNodes(+ nodes: ReadonlyArray<unknown>,+ ownership: MarkdownHelperOwnership,+ insideLink = false,+): void {+ // The overwhelmingly common replacement is one flat text node. Checking+ // that shape directly avoids the general walk's setup entirely, and this+ // runs once per match.+ if (nodes.length === 1) {+ const only = nodes[0];+ if (+ only != null &&+ typeof only === 'object' &&+ (only as {type?: unknown}).type === 'text' &&+ typeof (only as {value?: unknown}).value === 'string'+ ) {+ let extra = false;+ for (const key in only) {+ if (key !== 'type' && key !== 'value') {+ extra = true;+ break;+ }+ }+ if (!extra) {+ return;+ }+ }+ }+ // Cycle tracking is allocated only once a node with children appears: the+ // common replacement is a flat node, and this runs once per match.+ let seen: Set<object> | undefined;+ let count = 0;+ const visit = (value: unknown, withinLink: boolean): boolean => {+ if (value == null || typeof value !== 'object' || seen?.has(value)) {+ return false;+ }+ if (++count > 10_000) {+ return false;+ }+ if (Array.isArray((value as {children?: unknown}).children)) {+ seen ??= new Set<object>();+ seen.add(value);+ }+ const node = value as Record<string, unknown>;+ const type = node.type;+ if (typeof type !== 'string' || !HELPER_PHRASING_TYPES.has(type)) {+ return false;+ }+ // A helper's callback authors synthetic nodes only: provenance stays+ // Core's to assign, so neither field may be present.+ if (node.position !== undefined) {+ return false;+ }+ if (type !== 'extension' && node.source !== undefined) {+ return false;+ }+ if (node.data !== undefined && !isMarkdownPluginData(node.data)) {+ return false;+ }+ switch (type) {+ case 'text':+ case 'inlineCode':+ case 'inlineMath':+ if (typeof node.value !== 'string') {+ return false;+ }+ break;+ case 'link':+ if (+ withinLink ||+ typeof node.url !== 'string' ||+ !isSafeMarkdownParserUrl(node.url)+ ) {+ return false;+ }+ break;+ case 'image':+ if (+ typeof node.url !== 'string' ||+ !isSafeMarkdownParserUrl(node.url) ||+ typeof node.alt !== 'string'+ ) {+ return false;+ }+ break;+ case 'citation':+ if (typeof node.sourceId !== 'string') {+ return false;+ }+ break;+ case 'extension':+ if (+ node.plugin !== ownership.pluginName ||+ typeof node.name !== 'string' ||+ !ownership.hasRenderer(node.name) ||+ node.display !== 'inline' ||+ node.source !== undefined ||+ !isMarkdownPluginData(node.data)+ ) {+ return false;+ }+ break;+ case 'strong':+ case 'emphasis':+ case 'delete':+ case 'break':+ break;+ }+ if (type === 'strong' || type === 'emphasis' || type === 'delete') {+ if (!Array.isArray(node.children)) {+ return false;+ }+ for (const child of node.children) {+ if (!visit(child, withinLink)) {+ return false;+ }+ }+ } else if (type === 'link') {+ if (!Array.isArray(node.children)) {+ return false;+ }+ for (const child of node.children) {+ if (!visit(child, true)) {+ return false;+ }+ }+ } else if ('children' in node) {+ return false;+ }+ return true;+ };+ for (const node of nodes) {+ if (!visit(node, insideLink)) {+ throw new TypeError(+ 'Markdown helper callback returned an unrepresentable node',+ );+ }+ }+ // Frozen only once every node passed, so a rejected batch leaves the+ // caller's objects exactly as they were. Freezing is what makes adopting+ // a caller's object safe instead of copying it: the object is now part of+ // an immutable tree, and a later write by whoever still holds a reference+ // cannot reach into the document.+ const reentrant = nodes !== reusableScratch;+ if (!reentrant) {+ scratchInUse = true;+ }+ try {+ for (const node of nodes) {+ freezeAstNode(node);+ }+ } finally {+ if (!reentrant) {+ scratchInUse = false;+ reusableScratch[0] = undefined;+ }+ }+}+export function reportMarkdownPluginFailure(pluginName: string,phase: 'syntax' | 'transform' | 'render',@@ -1146,20 +1483,40 @@ export function applyMarkdownTransforms<Node extends MarkdownExtensionNode>(if (plugins == null || plugins.transforms.length === 0) {return root;}- const pluginNames = new Set(plugins.entries.map(entry => entry.name));- const rendererKeys = new Set(plugins.renderers.keys());+ let pluginNames: Set<string> | undefined;+ let rendererKeys: Set<string> | undefined;let document: MarkdownAstRoot<Node> = root;- let sourceHeadingMarkers: Set<SourceHeadingMarker> | undefined;- let started = false;+ // Heading identity is stamped on the mutable parser tree, before anything+ // freezes it, and only when a plugin-authored transform will actually be+ // validated against it. An all-trusted pipeline never pays for it.+ const sourceHeadingMarkers = plugins.hasUntrustedTransform+ ? markSourceHeadings(root)+ : undefined;+ let guarded = false;let priorTransformChangedTree = false;+ /** A trusted helper has produced a tree nothing has frozen yet. */+ let deferredFreeze = false;for (const prepared of plugins.transforms) {if (!priorTransformChangedTree && prepared.claims?.(source) === false) {continue;}- if (!started) {- sourceHeadingMarkers = markSourceHeadings(root);- document = freezeAst(root);- started = true;+ // A plugin-authored transform observes deeply frozen input and has its+ // output validated. Both guards exist to contain untrusted structure, so+ // a Core-authored trusted helper needs neither — and a pipeline of only+ // trusted helpers never pays for them at all. Every untrusted transform+ // still receives a frozen tree, including one that follows a trusted+ // helper, whose output is not frozen on the way out.+ if (!prepared.trusted) {+ if (!guarded) {+ pluginNames = new Set(plugins.entries.map(entry => entry.name));+ rendererKeys = new Set(plugins.renderers.keys());+ guarded = true;+ }+ // Deeply freezes the whole tree, including anything a trusted helper+ // left shared and mutable, before plugin-authored code observes it.+ // Subtrees already frozen by an earlier pass are skipped.+ document = freezeAst(document);+ deferredFreeze = false;}try {const context: InternalMarkdownTransformContext = {@@ -1174,6 +1531,9 @@ export function applyMarkdownTransforms<Node extends MarkdownExtensionNode>();},[markdownTransformPluginName]: prepared.pluginName,+ [markdownTransformHasRenderer]: (nodeName: string) =>+ plugins.renderers.has(`${prepared.pluginName}\u0000${nodeName}`),+ [markdownTransformSourceUnchanged]: !priorTransformChangedTree,};const next = prepared.transform(document, context) as unknown;if (@@ -1187,13 +1547,25 @@ export function applyMarkdownTransforms<Node extends MarkdownExtensionNode>(if (next === document) {continue;}+ if (prepared.trusted) {+ priorTransformChangedTree = true;+ // No freeze here. Nothing untrusted can observe this tree yet —+ // only another Core helper, which never mutates its input — so the+ // pipeline freezes once, below, before anything else sees it. That+ // freeze is a FULL walk, never one that stops at nodes merely+ // shared with an earlier unfrozen tree: an undecorated sibling a+ // helper carried over by identity must be frozen too.+ deferredFreeze = true;+ document = next as MarkdownAstRoot<Node>;+ continue;+ }const positions = collectSourceInvariants(document);const existingExtensions = collectExtensionSignatures(document);if (!validateAst(next,- pluginNames,- rendererKeys,+ pluginNames as ReadonlySet<string>,+ rendererKeys as ReadonlySet<string>,positions,sourceHeadingMarkers as ReadonlySet<SourceHeadingMarker>,prepared.pluginName,@@ -1211,7 +1583,16 @@ export function applyMarkdownTransforms<Node extends MarkdownExtensionNode>(reportMarkdownPluginFailure(prepared.pluginName, 'transform', error);}}- return document;+ if (!deferredFreeze) {+ return document;+ }+ // Freeze what the helpers built. With a plugin-authored transform in the+ // list the whole tree is frozen, because that plugin may observe any part+ // of it; with only Core helpers, the untouched remainder is left exactly+ // as a zero-plugin parse leaves it.+ return plugins.hasUntrustedTransform+ ? freezeAst(document)+ : freezeBuiltAst(root, document);}export function getMarkdownExtensionRenderer(diff --git a/packages/core/src/Markdown/plugins/semanticFence.ts b/packages/core/src/Markdown/plugins/semanticFence.tsindex 5aa1a33717..b39dc89ae7 100644--- a/packages/core/src/Markdown/plugins/semanticFence.ts+++ b/packages/core/src/Markdown/plugins/semanticFence.ts@@ -7,12 +7,15 @@* @position Optional helper layer compiled onto the core transform protocol*/-import type {MarkdownAstBlockContent, MarkdownAstCode} from '../ast';+import type {+ MarkdownAstBlockContent,+ MarkdownAstCode,+ MarkdownAstListItem,+} from '../ast';import {- freezeMarkdownPluginData,- getMarkdownTransformPluginName,- isMarkdownPluginData,+ getMarkdownHelperOwnership,markMarkdownTransformClaim,+ markMarkdownTransformTrusted,type MarkdownExtensionNode,type MarkdownPluginData,type MarkdownTransform,@@ -74,33 +77,74 @@ export function getMarkdownFenceProposal(return (node as FenceCode)[markdownFenceProposal];}-function clonePluginData<Data extends MarkdownPluginData>(value: Data): Data {+function cloneNestedPluginData(+ value: unknown,+ parent: object,+ ancestors: ReadonlySet<object> | undefined,+): MarkdownPluginData {if (value == null || typeof value !== 'object') {+ return clonePluginData(value, ancestors);+ }+ const nestedAncestors = new Set(ancestors);+ nestedAncestors.add(parent);+ return clonePluginData(value, nestedAncestors);+}++function clonePluginData(+ value: unknown,+ ancestors?: ReadonlySet<object>,+): MarkdownPluginData {+ if (+ value == null ||+ typeof value === 'boolean' ||+ typeof value === 'string'+ ) {+ return value as MarkdownPluginData;+ }+ if (typeof value === 'number') {+ if (!Number.isFinite(value)) {+ throw new TypeError('invalid plugin data');+ }return value;}+ if (+ typeof value !== 'object' ||+ ancestors?.has(value) === true ||+ (!Array.isArray(value) && Object.getPrototypeOf(value) !== Object.prototype)+ ) {+ throw new TypeError('invalid plugin data');+ }+ let clone: MarkdownPluginData;if (Array.isArray(value)) {- const clone: MarkdownPluginData[] = [];- for (const item of value) {- clone.push(clonePluginData(item));+ const items = new Array<MarkdownPluginData>(value.length);+ for (let index = 0; index < value.length; index++) {+ if (index in value) {+ items[index] = cloneNestedPluginData(value[index], value, ancestors);+ }+ }+ clone = Object.freeze(items);+ } else {+ const record: Record<string, MarkdownPluginData> = {};+ for (const key in value) {+ const nested = cloneNestedPluginData(+ (value as Record<string, unknown>)[key],+ value,+ ancestors,+ );+ if (key === '__proto__') {+ Object.defineProperty(record, key, {+ configurable: true,+ enumerable: true,+ value: nested,+ writable: true,+ });+ } else {+ record[key] = nested;+ }}- return clone as unknown as Data;+ clone = Object.freeze(record);}- return Object.fromEntries(- Object.entries(value).map(([key, nested]) => [- key,- clonePluginData(nested),- ]),- ) as Data;-}--function sameItems<T>(- left: ReadonlyArray<T>,- right: ReadonlyArray<T>,-): boolean {- return (- left.length === right.length &&- left.every((value, index) => value === right[index])- );+ return clone;}function createProposal(@@ -108,6 +152,7 @@ function createProposal(MarkdownExtensionNode<string, string, MarkdownPluginData, 'block'>>,pluginName: string,+ hasRenderer: (nodeName: string) => boolean,): MarkdownFenceProposal {if (node == null ||@@ -116,23 +161,34 @@ function createProposal(node.plugin !== pluginName ||typeof node.name !== 'string' ||node.name.trim() === '' ||+ // Every extension node must have a renderer and a text projection+ // (FR14). Checking it here is what lets Core run this helper on its+ // trusted path: the node is fully validated before it is inserted.+ !hasRenderer(node.name) ||node.display !== 'block' ||- !isMarkdownPluginData(node.data) ||'source' in node ||'position' in node) {throw new TypeError(- 'Markdown fence createNode must return an owned block extension node with finite data',+ 'Markdown fence createNode must return an owned block extension node with finite data and a registered renderer',);}+ let data: MarkdownPluginData;+ try {+ data = clonePluginData(node.data);+ } catch {+ throw new TypeError(+ 'Markdown fence createNode must return an owned block extension node with finite data and a registered renderer',+ );+ }return Object.freeze({node: Object.freeze({type: 'extension' as const,plugin: node.plugin,name: node.name,display: 'block' as const,- data: freezeMarkdownPluginData(clonePluginData(node.data)),+ data,}),});}@@ -160,6 +216,7 @@ function annotateCode(node: MarkdownAstCode,languages: ReadonlySet<string>,pluginName: string,+ hasRenderer: (nodeName: string) => boolean,createNode: (context: MarkdownFenceContext<string>,) =>@@ -197,7 +254,7 @@ function annotateCode(Object.defineProperty(annotated, markdownFenceProposal, {configurable: false,enumerable: true,- value: createProposal(proposalNode, pluginName),+ value: createProposal(proposalNode, pluginName, hasRenderer),writable: false,});return annotated;@@ -207,6 +264,7 @@ function transformBlocks(blocks: ReadonlyArray<MarkdownAstBlockContent<MarkdownExtensionNode>>,languages: ReadonlySet<string>,pluginName: string,+ hasRenderer: (nodeName: string) => boolean,createNode: (context: MarkdownFenceContext<string>,) =>@@ -217,37 +275,59 @@ function transformBlocks(| undefined,report: (message: string) => void,): ReadonlyArray<MarkdownAstBlockContent<MarkdownExtensionNode>> {- const next = blocks.map(block => {+ let next: MarkdownAstBlockContent<MarkdownExtensionNode>[] | undefined;+ for (let index = 0; index < blocks.length; index++) {+ const block = blocks[index];+ let replacement: MarkdownAstBlockContent<MarkdownExtensionNode>;switch (block.type) {case 'code':- return annotateCode(block, languages, pluginName, createNode, report);+ replacement = annotateCode(+ block,+ languages,+ pluginName,+ hasRenderer,+ createNode,+ report,+ );+ break;case 'blockquote': {const children = transformBlocks(block.children,languages,pluginName,+ hasRenderer,createNode,report,);- return children === block.children ? block : {...block, children};+ replacement =+ children === block.children ? block : {...block, children};+ break;}case 'list': {- let changed = false;- const children = block.children.map(item => {+ let items: MarkdownAstListItem<MarkdownExtensionNode>[] | undefined;+ for (+ let itemIndex = 0;+ itemIndex < block.children.length;+ itemIndex+++ ) {+ const item = block.children[itemIndex];const itemChildren = transformBlocks(item.children,languages,pluginName,+ hasRenderer,createNode,report,);if (itemChildren === item.children) {- return item;+ items?.push(item);+ continue;}- changed = true;- return {...item, children: itemChildren};- });- return changed ? {...block, children} : block;+ items ??= block.children.slice(0, itemIndex);+ items.push({...item, children: itemChildren});+ }+ replacement = items == null ? block : {...block, children: items};+ break;}case 'heading':case 'paragraph':@@ -256,10 +336,15 @@ function transformBlocks(case 'thematicBreak':case 'image':case 'extension':- return block;+ replacement = block;+ break;}- });- return sameItems(blocks, next) ? blocks : next;+ if (next === undefined && replacement !== block) {+ next = blocks.slice(0, index);+ }+ next?.push(replacement);+ }+ return next ?? blocks;}export function createMarkdownFenceTransform<@@ -291,6 +376,12 @@ export function createMarkdownFenceTransform<const declaredLanguages = Object.freeze([...options.languages]);const languages = new Set<string>(declaredLanguages);+ const sourceNeedles = Object.freeze(+ declaredLanguages.flatMap(language => [+ `\`\`\`${language}`,+ `~~~${language}`,+ ]),+ );const createNode = options.createNode as (context: MarkdownFenceContext<string>,) =>@@ -303,8 +394,8 @@ export function createMarkdownFenceTransform<root,context: MarkdownTransformContext,) => {- const pluginName = getMarkdownTransformPluginName(context);- if (pluginName == null) {+ const ownership = getMarkdownHelperOwnership(context);+ if (ownership == null) {throw new TypeError('Markdown fence transforms must run through createMarkdownPlugin',);@@ -312,18 +403,22 @@ export function createMarkdownFenceTransform<const children = transformBlocks(root.children,languages,- pluginName,+ ownership.pluginName,+ ownership.hasRenderer,createNode,context.report,);return children === root.children ? root : {...root, children};};- return markMarkdownTransformClaim(transform, source =>- declaredLanguages.some(- language =>- source.includes(`\`\`\`${language}`) ||- source.includes(`~~~${language}`),- ),+ // Every node this helper inserts is validated above against the same+ // rules Core applies to plugin-authored output — owned plugin name,+ // registered renderer, representable frozen data, no authored provenance+ // — and nothing else in the tree is touched. Core may therefore skip+ // revalidating the whole document after it runs.+ return markMarkdownTransformTrusted(+ markMarkdownTransformClaim(transform, source =>+ sourceNeedles.some(needle => source.includes(needle)),+ ) as MarkdownTransform<never>,);}diff --git a/packages/core/src/Markdown/plugins/sourceDecoration.test.tsx b/packages/core/src/Markdown/plugins/sourceDecoration.test.tsxnew file mode 100644index 0000000000..92aeb223d8--- /dev/null+++ b/packages/core/src/Markdown/plugins/sourceDecoration.test.tsx@@ -0,0 +1,799 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file sourceDecoration.test.tsx+ * @input Validated source ranges, overlapping claims, and decorated documents+ * @output Behavioral, determinism, invariance, streaming, and ownership coverage+ * @position Acceptance tests for the optional Markdown source-decoration helper+ */++import {renderToString} from 'react-dom/server';+import {describe, expect, expectTypeOf, it, vi} from 'vitest';+import {Markdown} from '../Markdown';+import {+ createIncrementalState,+ parseMarkdown,+ parseMarkdownAst,+ parseMarkdownAstIncremental,+ parseMarkdownIncremental,+ trimStreamingArtifacts,+} from '../parser';+import {createMarkdownPlugin} from './protocol';+import type {+ MarkdownExtensionNode,+ MarkdownSyntaxPluginDefinition,+ MarkdownTransform,+} from './protocol';+import {parseOutlineFromMarkdown} from '../../Outline/parseOutlineFromMarkdown';+import {+ createMarkdownSourceDecoration,+ getMarkdownSourceDecorations,+} from './sourceDecoration';+import type {MarkdownSourceDecoration} from './sourceDecoration';++const SOURCE = [+ '# Release notes',+ '',+ 'Ada shipped the parser.',+ '',+ '> Quoted detail',+].join('\n');++type Seen = Map<string, ReadonlyArray<MarkdownSourceDecoration>>;++function spanOf(text: string, source: string = SOURCE) {+ const start = source.indexOf(text);+ expect(start).toBeGreaterThanOrEqual(0);+ return {start, end: start + text.length};+}++/**+ * Reads decorations back through the public protocol: a transform that runs+ * after the decorating one observes exactly what later plugins and rendering+ * observe.+ */+function observerPlugin(name: string, seen: Seen) {+ return createMarkdownPlugin({+ name,+ apiVersion: 1,+ transform(root) {+ seen.clear();+ root.children.forEach((block, index) => {+ seen.set(`${index}:${block.type}`, getMarkdownSourceDecorations(block));+ });+ return root;+ },+ });+}++function decorationPlugin(+ name: string,+ transform: MarkdownTransform<never>,+): ReturnType<typeof createMarkdownPlugin> {+ return createMarkdownPlugin({name, apiVersion: 1, transform});+}++function decorationsFor(+ source: string,+ ranges: ReadonlyArray<{start: number; end: number}>,+ name = 'hit',+): Seen {+ const seen: Seen = new Map();+ parseMarkdown(source, {+ plugins: [+ decorationPlugin(+ `${name}-decoration`,+ createMarkdownSourceDecoration({name, ranges}),+ ),+ observerPlugin(`${name}-observer`, seen),+ ],+ });+ return seen;+}++describe('createMarkdownSourceDecoration', () => {+ it('decorates positioned blocks without the caller opting into sourceRanges', () => {+ const paragraph = spanOf('Ada shipped the parser.');+ const seen: Seen = new Map();+ parseMarkdown(SOURCE, {+ plugins: [+ decorationPlugin(+ 'covers-paragraph',+ createMarkdownSourceDecoration({+ name: 'review-hit',+ ranges: [{...paragraph, data: {reviewer: 'ada'}}],+ }),+ ),+ observerPlugin('covers-paragraph-observer', seen),+ ],+ });++ expect(seen.get('1:paragraph')).toEqual([+ {name: 'review-hit', data: {reviewer: 'ada'}},+ ]);+ expect(seen.get('0:heading')).toEqual([]);+ expect(seen.get('2:blockquote')).toEqual([]);+ });++ it('decorates every block a range touches, including partial overlap', () => {+ const heading = spanOf('# Release notes');+ const paragraph = spanOf('Ada shipped the parser.');++ // One range covering two whole blocks.+ const spanning = decorationsFor(SOURCE, [+ {start: heading.start, end: paragraph.end},+ ]);+ expect(spanning.get('0:heading')).toEqual([{name: 'hit'}]);+ expect(spanning.get('1:paragraph')).toEqual([{name: 'hit'}]);+ expect(spanning.get('2:blockquote')).toEqual([]);++ // A range strictly inside one block still decorates that block: a search+ // hit is a sub-span of the prose it was found in.+ const inside = decorationsFor(SOURCE, [spanOf('shipped')]);+ expect(inside.get('1:paragraph')).toEqual([{name: 'hit'}]);+ expect(inside.get('0:heading')).toEqual([]);++ // A range straddling a boundary touches both blocks.+ const straddling = decorationsFor(SOURCE, [+ {start: paragraph.end - 4, end: spanOf('> Quoted detail').start + 3},+ ]);+ expect(straddling.get('1:paragraph')).toEqual([{name: 'hit'}]);+ expect(straddling.get('2:blockquote')).toEqual([{name: 'hit'}]);+ });++ it('is observable through the rendered component without changing its output', () => {+ const paragraph = spanOf('Ada shipped the parser.');+ const seen: Seen = new Map();+ const plugins = [+ decorationPlugin(+ 'component-decoration',+ createMarkdownSourceDecoration({+ name: 'search-hit',+ ranges: [paragraph],+ }),+ ),+ observerPlugin('component-observer', seen),+ ];++ const decorated = renderToString(+ <Markdown plugins={plugins}>{SOURCE}</Markdown>,+ );++ expect(seen.get('1:paragraph')).toEqual([{name: 'search-hit'}]);+ expect(decorated).toBe(renderToString(<Markdown>{SOURCE}</Markdown>));+ });++ it('is observable through Outline without changing heading identity', () => {+ const seen: Seen = new Map();+ const plugins = [+ decorationPlugin(+ 'outline-decoration',+ createMarkdownSourceDecoration({+ name: 'search-hit',+ ranges: [spanOf('# Release notes')],+ }),+ ),+ observerPlugin('outline-observer', seen),+ ];++ expect(parseOutlineFromMarkdown(SOURCE, {plugins})).toEqual(+ parseOutlineFromMarkdown(SOURCE),+ );+ expect(seen.get('0:heading')).toEqual([{name: 'search-hit'}]);+ });++ it('exposes decorations only on the settled snapshot, never present then absent', () => {+ // An ordinary table: while it streams, the covering block is re-parsed+ // from paragraph to table and its extent changes under the range.+ const source = [+ '# Title',+ '',+ 'Intro prose here.',+ '',+ '| Item | Detail |',+ '| --- | --- |',+ '| 1 | two |',+ '',+ 'Tail prose.',+ ].join('\n');++ for (const target of [+ 'Intro prose here.',+ '| Item | Detail |',+ '| 1 | two |',+ 'Tail prose.',+ ]) {+ const start = source.indexOf(target);+ const range = {start, end: start + target.length};+ const seen: Seen = new Map();+ const plugins = [+ decorationPlugin(+ 'streaming-decoration',+ createMarkdownSourceDecoration({name: 'hit', ranges: [range]}),+ ),+ observerPlugin('streaming-observer', seen),+ ];++ const state = createIncrementalState();+ const presence: boolean[] = [];+ for (let length = 1; length <= source.length; length++) {+ const chunk = source.slice(0, length);+ const isFinal = length === source.length;+ parseMarkdownIncremental(+ isFinal ? chunk : trimStreamingArtifacts(chunk, {}),+ state,+ {plugins, isFinal},+ );+ presence.push([...seen.values()].some(entries => entries.length > 0));+ }++ // Monotone by construction: absent while the source can still change,+ // present once it cannot. No present -> absent transition exists.+ expect(presence.slice(0, -1).every(shown => !shown)).toBe(true);+ expect(presence[presence.length - 1]).toBe(true);+ }+ });++ it('decorates blocks an earlier transform reordered', () => {+ const heading = spanOf('# Release notes');+ const quote = spanOf('> Quoted detail');+ // A transform that reverses the document puts spans out of source order,+ // so a sweep that assumed document order would miss or mis-assign them.+ const reverse = createMarkdownPlugin({+ name: 'reverse-blocks',+ apiVersion: 1,+ transform: root => ({...root, children: [...root.children].reverse()}),+ });+ const seen: Seen = new Map();+ parseMarkdown(SOURCE, {+ plugins: [+ reverse,+ decorationPlugin(+ 'reordered',+ createMarkdownSourceDecoration({+ name: 'hit',+ ranges: [heading, {...quote, data: {tail: true}}],+ }),+ ),+ observerPlugin('reordered-observer', seen),+ ],+ });++ // Reversed order: blockquote, paragraph, heading.+ expect(seen.get('0:blockquote')).toEqual([+ {name: 'hit', data: {tail: true}},+ ]);+ expect(seen.get('1:paragraph')).toEqual([]);+ expect(seen.get('2:heading')).toEqual([{name: 'hit'}]);+ });++ it('resolves overlapping, duplicate, and unordered ranges deterministically', () => {+ const paragraph = spanOf('Ada shipped the parser.');+ const wide = {start: 0, end: paragraph.end};+ const run = () => {+ const seen: Seen = new Map();+ parseMarkdown(SOURCE, {+ plugins: [+ decorationPlugin(+ 'overlaps',+ createMarkdownSourceDecoration({+ name: 'hit',+ ranges: [+ {...paragraph, data: {rank: 2}},+ wide,+ {...paragraph, data: {rank: 1}},+ // Exact duplicate of the first range.+ {...paragraph, data: {rank: 2}},+ paragraph,+ ],+ }),+ ),+ observerPlugin('overlaps-observer', seen),+ ],+ });+ return seen.get('1:paragraph');+ };++ // Sorted by start, then end, then payload; the repeated rank-2 range+ // collapses into one entry.+ expect(run()).toEqual([+ // The wide range starts first.+ {name: 'hit'},+ // Then the three paragraph-aligned ranges, payload-free first.+ {name: 'hit'},+ {name: 'hit', data: {rank: 1}},+ {name: 'hit', data: {rank: 2}},+ ]);+ expect(run()).toEqual(run());+ });++ it('records each range once per block however many blocks it sweeps', () => {+ const sections = 200;+ const source = Array.from(+ {length: sections},+ (_, index) => `Section ${index} prose.`,+ ).join('\n\n');+ // One range per block plus one range spanning the whole document.+ const ranges = [+ {start: 0, end: source.length},+ ...Array.from({length: sections}, (_, index) =>+ spanOf(`Section ${index} prose.`, source),+ ),+ ];+ const seen = decorationsFor(source, ranges);++ expect(seen.size).toBe(sections);+ for (let index = 0; index < sections; index++) {+ // The document-wide range and exactly this block's own range.+ expect(seen.get(`${index}:paragraph`)).toEqual([+ {name: 'hit'},+ {name: 'hit'},+ ]);+ }+ });++ it('appends later plugins after earlier ones', () => {+ const paragraph = spanOf('Ada shipped the parser.');+ const seen: Seen = new Map();+ parseMarkdown(SOURCE, {+ plugins: [+ decorationPlugin(+ 'first-pass',+ createMarkdownSourceDecoration({name: 'first', ranges: [paragraph]}),+ ),+ decorationPlugin(+ 'second-pass',+ createMarkdownSourceDecoration({name: 'second', ranges: [paragraph]}),+ ),+ observerPlugin('order-observer', seen),+ ],+ });++ expect(seen.get('1:paragraph')).toEqual([+ {name: 'first'},+ {name: 'second'},+ ]);+ });++ it('leaves parsed results, rendered output, and heading identity unchanged', () => {+ const decorated = decorationPlugin(+ 'invariance',+ createMarkdownSourceDecoration({+ name: 'hit',+ ranges: [spanOf('# Release notes'), spanOf('Ada shipped the parser.')],+ }),+ );++ expect(parseMarkdown(SOURCE, {plugins: [decorated]})).toEqual(+ parseMarkdown(SOURCE),+ );+ expect(+ parseMarkdown(SOURCE, {sourceRanges: true, plugins: [decorated]}),+ ).toEqual(parseMarkdown(SOURCE, {sourceRanges: true}));++ const plain = renderToString(<Markdown>{SOURCE}</Markdown>);+ const withDecoration = renderToString(+ <Markdown plugins={[decorated]}>{SOURCE}</Markdown>,+ );+ expect(withDecoration).toBe(plain);+ });++ it('never mutates the document a transform received', () => {+ const paragraph = spanOf('Ada shipped the parser.');+ const before: ReadonlyArray<MarkdownSourceDecoration>[] = [];+ const after: ReadonlyArray<MarkdownSourceDecoration>[] = [];+ const capture = (+ name: string,+ sink: ReadonlyArray<MarkdownSourceDecoration>[],+ ) =>+ createMarkdownPlugin({+ name,+ apiVersion: 1,+ transform(root) {+ const block = root.children[1];+ sink.push(getMarkdownSourceDecorations(block));+ expect(Object.isFrozen(block)).toBe(true);+ return root;+ },+ });++ parseMarkdown(SOURCE, {+ plugins: [+ capture('mutation-before', before),+ decorationPlugin(+ 'mutation-decorate',+ createMarkdownSourceDecoration({name: 'hit', ranges: [paragraph]}),+ ),+ capture('mutation-after', after),+ ],+ });++ expect(before).toEqual([[]]);+ expect(after).toEqual([[{name: 'hit'}]]);+ });++ it('freezes every sibling before an untrusted plugin observes the tree', () => {+ // A trusted helper carries undecorated siblings over by identity. Those+ // siblings must still be deeply frozen before plugin-authored code runs,+ // or a later plugin can mutate a node the helper never touched.+ const source = '# Title\n\nAlpha prose.\n\nBeta prose.\n\nGamma prose.';+ const target = source.indexOf('Beta prose.');+ const observed: {type: string; frozen: boolean; textFrozen: boolean}[] = [];+ let mutationRejected = false;++ const attacker = createMarkdownPlugin({+ name: 'sibling-attacker',+ apiVersion: 1,+ transform(root) {+ root.children.forEach(block => {+ const child =+ 'children' in block+ ? (block.children[0] as object | undefined)+ : undefined;+ observed.push({+ type: block.type,+ frozen: Object.isFrozen(block),+ textFrozen: child == null || Object.isFrozen(child),+ });+ });+ // Attempt an in-place edit of an UNDECORATED sibling's text node.+ const victim = root.children[1];+ const text =+ victim.type === 'paragraph'+ ? (victim.children[0] as {value: string})+ : undefined;+ try {+ if (text != null) {+ text.value = 'HACKED';+ }+ mutationRejected = text?.value !== 'HACKED';+ } catch {+ mutationRejected = true;+ }+ return root;+ },+ });++ const blocks = parseMarkdown(source, {+ plugins: [+ decorationPlugin(+ 'sibling-decoration',+ createMarkdownSourceDecoration({+ name: 'hit',+ ranges: [{start: target, end: target + 'Beta prose.'.length}],+ }),+ ),+ attacker,+ ],+ });++ expect(observed).toHaveLength(4);+ expect(observed.every(entry => entry.frozen && entry.textFrozen)).toBe(+ true,+ );+ expect(mutationRejected).toBe(true);+ // The undecorated sibling reached rendering unchanged.+ expect(blocks[1]).toEqual({+ type: 'paragraph',+ children: [{type: 'text', content: 'Alpha prose.'}],+ });+ });++ it('leaves a prior streamed snapshot untouched when a later chunk decorates', () => {+ const source = 'Alpha prose.\n\nBeta prose.';+ const target = source.indexOf('Beta prose.');+ const plugins = [+ decorationPlugin(+ 'snapshot-decoration',+ createMarkdownSourceDecoration({+ name: 'hit',+ ranges: [{start: target, end: target + 'Beta prose.'.length}],+ }),+ ),+ ];++ const state = createIncrementalState();+ const partial = parseMarkdownAstIncremental(+ source.slice(0, target),+ state,+ {plugins, isFinal: false},+ ).children;+ const partialSnapshot = JSON.parse(JSON.stringify(partial));+ const partialBlocks = [...partial];++ parseMarkdownAstIncremental(source, state, {plugins, isFinal: true});++ // The earlier snapshot is a settled value a caller may still hold: the+ // later chunk's decoration must not have reached back into it.+ expect(JSON.parse(JSON.stringify(partial))).toEqual(partialSnapshot);+ expect(partialBlocks.map(getMarkdownSourceDecorations)).toEqual(+ partialBlocks.map(() => []),+ );+ });++ it('copies caller ranges so later mutation cannot change decorations', () => {+ const paragraph = spanOf('Ada shipped the parser.');+ const payload = {reviewer: 'ada'};+ const ranges = [{...paragraph, data: payload}];+ const transform = createMarkdownSourceDecoration({name: 'hit', ranges});+ payload.reviewer = 'linus';+ ranges.push({start: 0, end: 4, data: payload});++ const seen: Seen = new Map();+ parseMarkdown(SOURCE, {+ plugins: [+ decorationPlugin('immutable-options', transform),+ observerPlugin('immutable-options-observer', seen),+ ],+ });++ expect(seen.get('1:paragraph')).toEqual([+ {name: 'hit', data: {reviewer: 'ada'}},+ ]);+ expect(seen.get('0:heading')).toEqual([]);+ });++ it('leaves plugin extension nodes to their owner', () => {+ type NoteNode = MarkdownExtensionNode<+ 'decoration-notes',+ 'note',+ {readonly body: string},+ 'block'+ >;+ const source = ':::note\nOwned by the plugin\n:::\n\nAda shipped it.';+ const notes = createMarkdownPlugin<'decoration-notes', NoteNode>({+ name: 'decoration-notes',+ apiVersion: 1,+ parseKey: 'v1',+ syntax: {+ block: [+ {+ startsWith: [':::note'],+ maxSpan: 200,+ tokenize({source: input, offset, end, isFinal}) {+ const close = input.indexOf('\n:::', offset + 7);+ if (close < 0 || close + 4 > end) {+ return isFinal ? {status: 'no-match'} : {status: 'defer'};+ }+ return {+ status: 'match',+ end: close + 4,+ node: {+ type: 'extension',+ plugin: 'decoration-notes',+ name: 'note',+ display: 'block',+ data: {body: input.slice(offset + 7, close).trim()},+ },+ };+ },+ },+ ],+ },+ renderers: {+ note: {+ render: ({node}) => node.data.body,+ toText: node => node.data.body,+ },+ },+ } satisfies MarkdownSyntaxPluginDefinition<'decoration-notes', NoteNode>);++ const seen: Seen = new Map();+ const blocks = parseMarkdown(source, {+ plugins: [+ notes,+ decorationPlugin(+ 'extension-ownership',+ createMarkdownSourceDecoration({+ name: 'hit',+ ranges: [{start: 0, end: source.length}],+ }),+ ),+ observerPlugin('extension-ownership-observer', seen),+ ],+ });++ expect(seen.get('0:extension')).toEqual([]);+ expect(seen.get('1:paragraph')).toEqual([{name: 'hit'}]);+ expect(blocks[0]).toMatchObject({+ type: 'extension',+ data: {body: 'Owned by the plugin'},+ });+ });++ it('stores decorations in a versioned Core-owned envelope', () => {+ const root = parseMarkdownAst(SOURCE, {+ plugins: [+ decorationPlugin(+ 'envelope-shape',+ createMarkdownSourceDecoration({+ name: 'hit',+ ranges: [spanOf('Ada shipped the parser.')],+ }),+ ),+ ],+ });++ expect(root.children[1]?.data).toEqual({+ 'astryx:sourceDecorations': {+ kind: 'astryx.markdown.sourceDecorations',+ version: 1,+ entries: [{name: 'hit'}],+ },+ });+ });++ it('never merges into foreign data and reports the skip once', () => {+ const warning = vi.spyOn(console, 'warn').mockImplementation(() => {});+ const seen: Seen = new Map();+ // Two foreign shapes: an opaque scalar, and an unbranded array parked at+ // the envelope key by another plugin.+ const foreign = createMarkdownPlugin({+ name: 'foreign-data',+ apiVersion: 1,+ transform(root) {+ return {+ ...root,+ children: root.children.map(block => {+ if (block.type === 'heading') {+ return {...block, data: 'opaque'};+ }+ if (block.type === 'blockquote') {+ return {+ ...block,+ data: {'astryx:sourceDecorations': [{name: 'forged'}]},+ };+ }+ return block;+ }),+ };+ },+ });++ parseMarkdown(SOURCE, {+ plugins: [+ foreign,+ decorationPlugin(+ 'foreign-data-decoration',+ createMarkdownSourceDecoration({+ name: 'hit',+ ranges: [{start: 0, end: SOURCE.length}],+ }),+ ),+ observerPlugin('foreign-data-observer', seen),+ ],+ });++ expect(seen.get('0:heading')).toEqual([]);+ expect(seen.get('2:blockquote')).toEqual([]);+ expect(seen.get('1:paragraph')).toEqual([{name: 'hit'}]);+ expect(warning).toHaveBeenCalledTimes(1);+ expect(warning.mock.calls[0]?.[0]).toContain('foreign-data-decoration');+ warning.mockRestore();+ });++ it('keeps unrelated data on a node it decorates', () => {+ const paragraph = spanOf('Ada shipped the parser.');+ const annotate = createMarkdownPlugin({+ name: 'annotate-first',+ apiVersion: 1,+ transform(root) {+ return {+ ...root,+ children: root.children.map(block =>+ block.type === 'paragraph'+ ? {...block, data: {owner: 'ada'}}+ : block,+ ),+ };+ },+ });++ const root = parseMarkdownAst(SOURCE, {+ plugins: [+ annotate,+ decorationPlugin(+ 'annotate-decoration',+ createMarkdownSourceDecoration({name: 'hit', ranges: [paragraph]}),+ ),+ ],+ });++ expect(root.children[1]?.data).toEqual({+ owner: 'ada',+ 'astryx:sourceDecorations': {+ kind: 'astryx.markdown.sourceDecorations',+ version: 1,+ entries: [{name: 'hit'}],+ },+ });+ });++ it('rejects invalid options at creation', () => {+ expect(() =>+ createMarkdownSourceDecoration({name: ' ', ranges: []}),+ ).toThrow(TypeError);+ expect(() =>+ createMarkdownSourceDecoration({+ name: 'hit',+ ranges: [{start: 4, end: 4}],+ }),+ ).toThrow(TypeError);+ expect(() =>+ createMarkdownSourceDecoration({+ name: 'hit',+ ranges: [{start: -1, end: 4}],+ }),+ ).toThrow(TypeError);+ expect(() =>+ createMarkdownSourceDecoration({+ name: 'hit',+ ranges: [{start: 0, end: 1.5}],+ }),+ ).toThrow(TypeError);+ expect(() =>+ createMarkdownSourceDecoration({+ name: 'hit',+ ranges: [+ {start: 0, end: 4, data: {render: (() => null) as unknown as string}},+ ],+ }),+ ).toThrow(TypeError);+ });++ it('treats an empty range list as a no-op transform', () => {+ const seen = decorationsFor(SOURCE, []);+ expect([...seen.values()].flat()).toEqual([]);+ });++ it('types as an extension-free transform and reads only its own envelope', () => {+ const transform = createMarkdownSourceDecoration({+ name: 'hit',+ ranges: [{start: 0, end: 4}],+ });+ expectTypeOf(transform).toExtend<MarkdownTransform<never>>();+ // A decoration plugin introduces no extension nodes, so it needs no+ // renderers.+ const plugin = createMarkdownPlugin({+ name: 'typed-decoration',+ apiVersion: 1,+ transform,+ });+ expectTypeOf(plugin.name).toEqualTypeOf<string>();+ expectTypeOf(getMarkdownSourceDecorations({})).toEqualTypeOf<+ ReadonlyArray<MarkdownSourceDecoration>+ >();++ expect(getMarkdownSourceDecorations({data: 'opaque'})).toEqual([]);+ expect(+ getMarkdownSourceDecorations({data: {decorations: [{name: 'x'}]}}),+ ).toEqual([]);+ expect(+ getMarkdownSourceDecorations({+ data: {'astryx:sourceDecorations': [{name: 'x'}]},+ }),+ ).toEqual([]);+ expect(+ getMarkdownSourceDecorations({+ data: {+ 'astryx:sourceDecorations': {+ kind: 'astryx.markdown.sourceDecorations',+ version: 2,+ entries: [{name: 'x'}],+ },+ },+ }),+ ).toEqual([]);+ expect(+ getMarkdownSourceDecorations({+ data: {+ 'astryx:sourceDecorations': {+ kind: 'astryx.markdown.sourceDecorations',+ version: 1,+ entries: [{name: 'x'}],+ },+ },+ }),+ ).toEqual([{name: 'x'}]);+ });+});diff --git a/packages/core/src/Markdown/plugins/sourceDecoration.ts b/packages/core/src/Markdown/plugins/sourceDecoration.tsnew file mode 100644index 0000000000..f0ff9dbe12--- /dev/null+++ b/packages/core/src/Markdown/plugins/sourceDecoration.ts@@ -0,0 +1,539 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file sourceDecoration.ts+ * @input Caller-supplied UTF-16 source ranges and non-semantic decoration payloads+ * @output Immutable Markdown transforms that annotate Core-authored source-backed nodes+ * @position Optional helper layer compiled onto the core transform protocol+ */++import type {+ MarkdownAstDataValue,+ MarkdownAstNodeBase,+ MarkdownAstPosition,+ MarkdownAstRoot,+} from '../ast';+import {+ isMarkdownPluginData,+ markMarkdownTransformClaim,+ markMarkdownTransformTrusted,+ type MarkdownExtensionNode,+ type MarkdownPluginData,+ type MarkdownTransform,+} from './protocol';++/**+ * Reserved node-data key and brand of the Core-owned decoration envelope.+ * Decorations are only ever read from, and merged into, a value carrying all+ * three: an arbitrary payload that happens to sit at this key is foreign data+ * and is left untouched.+ */+const ENVELOPE_KEY = 'astryx:sourceDecorations';+const ENVELOPE_BRAND = 'astryx.markdown.sourceDecorations';+const ENVELOPE_VERSION = 1;++/** One decoration recorded on an annotated node. */+export interface MarkdownSourceDecoration {+ /** Stable decoration name supplied by the helper's caller. */+ readonly name: string;+ /** Optional finite JSON-like payload copied from the matching range. */+ readonly data?: MarkdownPluginData;+}++export interface MarkdownSourceDecorationRange {+ /** Inclusive UTF-16 offset into the parsed source. */+ readonly start: number;+ /** Exclusive UTF-16 offset into the parsed source. */+ readonly end: number;+ /** Optional finite JSON-like payload recorded with this range. */+ readonly data?: MarkdownPluginData;+}++export interface MarkdownSourceDecorationOptions {+ /** Stable decoration name identifying every entry this helper records. */+ readonly name: string;+ /**+ * Source ranges to decorate, as UTF-16 offsets into the same string the+ * parser received. Order does not matter: entries are recorded in a fixed+ * start, end, payload order so overlapping and duplicate ranges resolve the+ * same way on every run. An empty list is a valid no-op.+ */+ readonly ranges: ReadonlyArray<MarkdownSourceDecorationRange>;+}++/** A decoration entry as stored: plain, frozen, JSON-like node data. */+type DecorationEntry = {readonly [key: string]: MarkdownAstDataValue};++interface NormalizedRange {+ readonly start: number;+ readonly end: number;+ readonly entry: DecorationEntry;+}++/**+ * Every AST node shares an optional `position` and `data`; decoration is node+ * kind agnostic, so the walk below reads only those two fields plus children.+ */+type DecoratableNode = MarkdownAstNodeBase & {+ readonly type: string;+ readonly children?: ReadonlyArray<DecoratableNode>;+};++interface RunState {+ readonly report: (message: string) => void;+ hasReported: boolean;+}++const NO_DECORATIONS: ReadonlyArray<MarkdownSourceDecoration> = Object.freeze(+ [],+);++function isPlainObject(+ value: MarkdownAstDataValue | undefined,+): value is {readonly [key: string]: MarkdownAstDataValue} {+ return value != null && typeof value === 'object' && !Array.isArray(value);+}++function cloneDecorationData<Data extends MarkdownPluginData>(+ value: Data,+): Data {+ if (value == null || typeof value !== 'object') {+ return value;+ }+ if (Array.isArray(value)) {+ return Object.freeze(value.map(cloneDecorationData)) as Data;+ }+ const clone: Record<string, MarkdownAstDataValue> = {};+ for (const [key, nested] of Object.entries(value)) {+ clone[key] = cloneDecorationData(nested);+ }+ return Object.freeze(clone) as Data;+}++function fail(message: string): never {+ throw new TypeError(`Markdown source decoration: ${message}`);+}++function normalizeRanges(+ name: string,+ ranges: ReadonlyArray<MarkdownSourceDecorationRange>,+): ReadonlyArray<NormalizedRange> {+ const normalized: (NormalizedRange & {readonly key: string})[] = [];+ const seen = new Set<string>();+ for (const range of ranges) {+ if (+ !Number.isInteger(range.start) ||+ !Number.isInteger(range.end) ||+ range.start < 0 ||+ range.end <= range.start+ ) {+ fail('ranges must be ordered, non-empty, non-negative UTF-16 offsets');+ }+ if (range.data !== undefined && !isMarkdownPluginData(range.data)) {+ fail('range data must be finite JSON-like plugin data');+ }+ const data =+ range.data === undefined ? undefined : cloneDecorationData(range.data);+ const key = `${range.start}\u0000${range.end}\u0000${+ data === undefined ? '' : JSON.stringify(data)+ }`;+ if (seen.has(key)) {+ continue;+ }+ seen.add(key);+ normalized.push({+ start: range.start,+ end: range.end,+ entry: Object.freeze(+ data === undefined ? {name} : {name, data},+ ) as DecorationEntry,+ key,+ });+ }+ // Sorted by start, then end, then payload: the sweep below needs start+ // order, and the tail of the comparison makes overlapping and duplicate+ // ranges resolve identically on every run.+ normalized.sort(+ (left, right) =>+ left.start - right.start ||+ left.end - right.end ||+ (left.key < right.key ? -1 : left.key > right.key ? 1 : 0),+ );+ return Object.freeze(+ normalized.map(({start, end, entry}) => Object.freeze({start, end, entry})),+ );+}++/**+ * A node's Core-authored span, or `undefined` when it has no usable one.+ * Read into two numbers rather than an object: this runs once per node of+ * every decorated document.+ */+function spanStart(position: MarkdownAstPosition | undefined): number {+ if (position === undefined) {+ return -1;+ }+ const start = position.start.offset;+ const end = position.end.offset;+ return typeof start === 'number' &&+ typeof end === 'number' &&+ start >= 0 &&+ end >= start+ ? start+ : -1;+}++/**+ * Reads the decoration entries already on a node. `null` means the node+ * carries data this helper does not own, so it must be left alone.+ */+function existingEntries(+ data: MarkdownAstDataValue | undefined,+): ReadonlyArray<MarkdownAstDataValue> | undefined | null {+ if (data === undefined) {+ return undefined;+ }+ if (!isPlainObject(data)) {+ return null;+ }+ const envelope = data[ENVELOPE_KEY];+ if (envelope === undefined) {+ return undefined;+ }+ if (+ !isPlainObject(envelope) ||+ envelope.kind !== ENVELOPE_BRAND ||+ envelope.version !== ENVELOPE_VERSION ||+ !Array.isArray(envelope.entries) ||+ !envelope.entries.every(isDecorationEntry)+ ) {+ return null;+ }+ return envelope.entries;+}++function isDecorationEntry(value: MarkdownAstDataValue): boolean {+ return isPlainObject(value) && typeof value.name === 'string';+}++function envelope(+ entries: ReadonlyArray<MarkdownAstDataValue>,+): MarkdownAstDataValue {+ return {+ kind: ENVELOPE_BRAND,+ version: ENVELOPE_VERSION,+ entries,+ };+}++function reportSkip(state: RunState): void {+ if (state.hasReported) {+ return;+ }+ state.hasReported = true;+ state.report(+ 'skipped a node whose data is not a Core-owned source-decoration envelope',+ );+}++/**+ * Records `entries` in the node's decoration envelope, or returns `undefined`+ * when the node carries data this helper does not own. Skipping keeps a+ * foreign payload intact instead of overwriting or extending it.+ */+function mergeDecorations(+ node: DecoratableNode,+ entries: ReadonlyArray<DecorationEntry>,+ state: RunState,+): MarkdownAstDataValue | undefined {+ const current = existingEntries(node.data);+ if (current === null) {+ reportSkip(state);+ return undefined;+ }+ const merged = envelope(+ current === undefined ? entries : [...current, ...entries],+ );+ return node.data === undefined+ ? {[ENVELOPE_KEY]: merged}+ : {...(node.data as object), [ENVELOPE_KEY]: merged};+}++/**+ * Whether the positioned children already sit in ascending span order, and+ * the span-ordered index list when they do not. An earlier transform may+ * reorder, drop, or interleave blocks, so document order is never assumed —+ * but the ordinary case is detected in one pass and needs no index list at+ * all.+ */+function spanOrder(+ children: ReadonlyArray<DecoratableNode>,+): ReadonlyArray<number> | undefined {+ let previous = -1;+ for (let index = 0; index < children.length; index++) {+ const start = spanStart(children[index].position);+ if (start < 0) {+ continue;+ }+ if (start < previous) {+ const indices: number[] = [];+ for (let scan = 0; scan < children.length; scan++) {+ if (spanStart(children[scan].position) >= 0) {+ indices.push(scan);+ }+ }+ indices.sort(+ (left, right) =>+ spanStart(children[left].position) -+ spanStart(children[right].position) || left - right,+ );+ return indices;+ }+ previous = start;+ }+ return undefined;+}++/**+ * The first slot whose node can overlap `offset` — that is, the first one+ * ending after it. Spans ascend, so the blocks a range touches are one+ * contiguous run and a search finds its start without walking the document.+ * Decorating a known range therefore costs the blocks it actually covers,+ * not the length of the document it sits in.+ */+function firstSlotEndingAfter(+ children: ReadonlyArray<DecoratableNode>,+ order: ReadonlyArray<number> | undefined,+ length: number,+ offset: number,+): number {+ let low = 0;+ let high = length;+ while (low < high) {+ const middle = (low + high) >>> 1;+ const node = children[order?.[middle] ?? middle];+ const end = node.position?.end.offset;+ if (typeof end === 'number' && end <= offset) {+ low = middle + 1;+ } else {+ high = middle;+ }+ }+ return low;+}++/**+ * Sweeps the start-sorted ranges across the span-ordered children, keeping+ * the ranges that can still overlap in an active window. Each range is+ * admitted and retired at most once, so the pass costs the blocks the ranges+ * reach plus the matches — never the whole document per range.+ */+function decorateChildren(+ children: ReadonlyArray<DecoratableNode>,+ ranges: ReadonlyArray<NormalizedRange>,+ state: RunState,+): ReadonlyArray<DecoratableNode> {+ const order = spanOrder(children);+ const length = order?.length ?? children.length;+ let next: DecoratableNode[] | undefined;+ let cursor = 0;+ const active: NormalizedRange[] = [];++ // Skip straight to the first block the earliest range can reach. Unordered+ // trees fall back to a full sweep, since a sorted index list makes the+ // search valid only over positioned nodes.+ let slot =+ order === undefined+ ? firstSlotEndingAfter(children, order, length, ranges[0].start)+ : 0;++ for (; slot < length; slot++) {+ // Once every range has been admitted and retired, no later span can+ // match: the rest of the document is left untouched.+ if (cursor >= ranges.length && active.length === 0) {+ break;+ }+ const index = order?.[slot] ?? slot;+ const child = children[index];+ const start = spanStart(child.position);+ if (start < 0) {+ continue;+ }+ const end = child.position?.end.offset as number;+ while (cursor < ranges.length && ranges[cursor].start < end) {+ active.push(ranges[cursor++]);+ }+ // Retire in place: spans are visited in ascending start order, so a range+ // that ended before this one starts cannot match any later span either.+ let kept = 0;+ for (let active_slot = 0; active_slot < active.length; active_slot++) {+ if (active[active_slot].end > start) {+ active[kept++] = active[active_slot];+ }+ }+ active.length = kept;+ if (kept === 0) {+ // No range reaches this block. When the next range starts beyond this+ // block, jump to the first block it can reach rather than walking the+ // gap between; when it starts inside this one the next block is+ // already the answer and searching for it would cost more than+ // stepping to it.+ if (+ order === undefined &&+ cursor < ranges.length &&+ ranges[cursor].start > end+ ) {+ const jump = firstSlotEndingAfter(+ children,+ order,+ length,+ ranges[cursor].start,+ );+ if (jump > slot) {+ slot = jump - 1;+ }+ }+ continue;+ }+ const decorated = decorateNode(child, active, state);+ if (decorated === child) {+ continue;+ }+ next ??= children.slice();+ next[index] = decorated;+ }+ return next ?? children;+}++/**+ * Decorates one node every range in `ranges` already overlaps. Decoration+ * stops at the outermost positioned node, so a range never records itself+ * twice down one branch.+ */+function decorateNode(+ node: DecoratableNode,+ ranges: ReadonlyArray<NormalizedRange>,+ state: RunState,+): DecoratableNode {+ // Extension nodes belong to the plugin that produced them; their data is+ // part of their identity, so decoration never touches them.+ if (node.type === 'extension') {+ return node;+ }+ const data = mergeDecorations(+ node,+ ranges.map(range => range.entry),+ state,+ );+ return data === undefined ? node : {...node, data};+}++/**+ * Compiles validated source ranges into an immutable transform that records+ * non-semantic presentation metadata on the Core-authored nodes those ranges+ * overlap.+ *+ * A node is decorated only when the parser authored its source position and+ * the range overlaps its span, so the helper never changes AST meaning,+ * rendered output, copyable text, accessible names, focus order, navigation,+ * or source provenance. Core authors positions on top-level blocks, so a+ * range decorates every block it touches, whatever order those blocks sit in+ * after earlier transforms.+ *+ * Decorations describe settled source, so they appear on the final snapshot+ * of a streaming document rather than on its partial chunks: a partial+ * block's extent is still changing, and exposing decorations from it would+ * let one appear and then vanish. Exposure is therefore monotone — never+ * present, then absent.+ *+ * Decorations are stored in one Core-owned versioned envelope and read back+ * with `getMarkdownSourceDecorations`; data the helper does not own is never+ * merged into or overwritten.+ *+ * @example+ * ```+ * const reviewHits = createMarkdownPlugin({+ * name: 'review-hits',+ * apiVersion: 1,+ * transform: createMarkdownSourceDecoration({+ * name: 'review-hit',+ * ranges: [{start: 0, end: 12, data: {reviewer: 'ada'}}],+ * }),+ * });+ * ```+ */+export function createMarkdownSourceDecoration(+ options: MarkdownSourceDecorationOptions,+): MarkdownTransform<never> {+ if (typeof options.name !== 'string' || options.name.trim() === '') {+ fail('name must be a non-empty string');+ }+ if (!Array.isArray(options.ranges)) {+ fail('ranges must be an array');+ }+ const ranges = normalizeRanges(options.name, options.ranges);+ const smallestStart = ranges.length === 0 ? -1 : ranges[0].start;++ const transform: MarkdownTransform<never> = (document, context) => {+ // Decorations describe settled source. While a document streams, the+ // block a range falls in can still be re-parsed, re-split, merged into a+ // table or list, or withheld entirely, so a range's covering block is not+ // stable and a decoration computed from it could appear, vanish, and+ // reappear across chunks. Waiting for the settled snapshot makes exposure+ // monotone by construction: absent while the source can still change,+ // then present once it cannot.+ if (!context.isFinal || ranges.length === 0) {+ return document;+ }+ // A range starting at or past the end of the source overlaps nothing.+ const usable =+ ranges[ranges.length - 1].start < context.source.length+ ? ranges+ : ranges.filter(range => range.start < context.source.length);+ if (usable.length === 0) {+ return document;+ }+ const root = document as unknown as DecoratableNode;+ if (root.children == null) {+ return document;+ }+ const state: RunState = {+ report: message => context.report(message),+ hasReported: false,+ };+ const children = decorateChildren(root.children, usable, state);+ return children === root.children+ ? document+ : ({+ ...document,+ children,+ } as unknown as MarkdownAstRoot<MarkdownExtensionNode>);+ };++ // Every node this helper emits is built here, from ranges validated above:+ // it inserts no caller-supplied node and changes no existing node's meaning,+ // provenance, or ownership. Core may therefore skip the validation and+ // freezing it applies to plugin-authored output.+ return markMarkdownTransformTrusted(+ markMarkdownTransformClaim(+ transform,+ source => smallestStart >= 0 && source.length > smallestStart,+ ),+ );+}++/**+ * Reads the decorations a source-decoration helper recorded on a node.+ * Returns an empty list for undecorated nodes and for any value that is not+ * a current Core-owned decoration envelope.+ */+export function getMarkdownSourceDecorations(+ node: MarkdownAstNodeBase,+): ReadonlyArray<MarkdownSourceDecoration> {+ const entries = existingEntries(node.data);+ // `existingEntries` returns entries only from a branded envelope whose+ // every member passed `isDecorationEntry`, so each one has a string name.+ return entries == null || entries.length === 0+ ? NO_DECORATIONS+ : (entries as unknown as ReadonlyArray<MarkdownSourceDecoration>);+}diff --git a/packages/core/src/Markdown/plugins/textTransform.test.ts b/packages/core/src/Markdown/plugins/textTransform.test.tsindex c7b2eff66e..59d97bf3d6 100644--- a/packages/core/src/Markdown/plugins/textTransform.test.ts+++ b/packages/core/src/Markdown/plugins/textTransform.test.ts@@ -7,8 +7,8 @@* @position Acceptance tests for the optional Markdown text helper*/-import {describe, expect, expectTypeOf, it} from 'vitest';-import {parseInline, parseMarkdown} from '../parser';+import {describe, expect, expectTypeOf, it, vi} from 'vitest';+import {parseInline, parseMarkdown, parseMarkdownAst} from '../parser';import {createMarkdownPlugin} from './protocol';import type {MarkdownExtensionNode} from './protocol';import {createMarkdownTextTransform} from './textTransform';@@ -352,4 +352,116 @@ describe('createMarkdownTextTransform', () => {expectTypeOf(compileOnlyRendererRequirement).toBeFunction();expectTypeOf(createMarkdownTextTransform).toBeFunction();});++ it('adopts a fresh replacement node and freezes it against later writes', () => {+ // A callback that builds a node per match hands over a fresh object, so+ // the helper puts it straight into the tree rather than copying it. That+ // is only safe because the node is frozen on the way in: whoever still+ // holds the reference cannot reach into the document afterwards.+ const handedOver: {type: 'text'; value: string}[] = [];+ const plugin = createMarkdownPlugin({+ name: 'fresh-nodes',+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern: /TODO/g,+ requiredSubstrings: ['TODO'],+ replace: () => {+ const node = {type: 'text' as const, value: 'done'};+ handedOver.push(node);+ return node;+ },+ }),+ });++ const blocks = parseMarkdown('TODO and TODO again', {plugins: [plugin]});+ expect(handedOver).toHaveLength(2);+ expect(handedOver.every(node => Object.isFrozen(node))).toBe(true);++ // A late write by the caller is rejected and the document is unaffected.+ expect(() => {+ 'use strict';+ handedOver[0].value = 'HACKED';+ }).toThrow(TypeError);+ expect(blocks).toEqual([+ {+ type: 'paragraph',+ children: [+ {type: 'text', content: 'done'},+ {type: 'text', content: ' and '},+ {type: 'text', content: 'done'},+ {type: 'text', content: ' again'},+ ],+ },+ ]);+ });++ it('copies a replacement node the callback hands over more than once', () => {+ // One object cannot occupy two positions in the tree, and the first+ // position's node is already frozen. A callback that returns the same+ // object for every match therefore gets a copy from the second match on.+ const shared = {type: 'text' as const, value: 'done'};+ const plugin = createMarkdownPlugin({+ name: 'shared-node',+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern: /TODO/g,+ requiredSubstrings: ['TODO'],+ replace: () => shared,+ }),+ });++ const root = parseMarkdownAst('TODO and TODO again', {plugins: [plugin]});+ const paragraph = root.children[0];+ if (paragraph.type !== 'paragraph') {+ throw new Error('Expected a paragraph');+ }+ const [first, , third] = paragraph.children;+ expect(first).not.toBe(third);+ expect(first).toEqual({type: 'text', value: 'done'});+ expect(third).toEqual({type: 'text', value: 'done'});++ // Both documents agree with a run whose callback allocates per match.+ expect(parseMarkdown('TODO and TODO again', {plugins: [plugin]})).toEqual(+ parseMarkdown('TODO and TODO again', {+ plugins: [+ createMarkdownPlugin({+ name: 'per-match-node',+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern: /TODO/g,+ requiredSubstrings: ['TODO'],+ replace: () => ({type: 'text', value: 'done'}),+ }),+ }),+ ],+ }),+ );+ });++ it('still rejects an unrepresentable node without freezing the batch', () => {+ const rejected = {+ type: 'text' as const,+ value: 'ok',+ position: {start: {offset: 0}, end: {offset: 2}},+ };+ const warning = vi.spyOn(console, 'warn').mockImplementation(() => {});+ const plugin = createMarkdownPlugin({+ name: 'forged-provenance',+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern: /TODO/g,+ requiredSubstrings: ['TODO'],+ replace: () => rejected as never,+ }),+ });++ // Authored provenance is refused, the document keeps its source text,+ // and the caller's object is left exactly as it was.+ expect(parseMarkdown('TODO', {plugins: [plugin]})).toEqual(+ parseMarkdown('TODO'),+ );+ expect(Object.isFrozen(rejected)).toBe(false);+ expect(warning).toHaveBeenCalled();+ warning.mockRestore();+ });});diff --git a/packages/core/src/Markdown/plugins/textTransform.ts b/packages/core/src/Markdown/plugins/textTransform.tsindex ff8229df61..5e123c80a2 100644--- a/packages/core/src/Markdown/plugins/textTransform.ts+++ b/packages/core/src/Markdown/plugins/textTransform.ts@@ -7,13 +7,35 @@* @position Optional helper layer compiled onto the core transform protocol*/-import type {MarkdownAstBlockContent, MarkdownAstPhrasingContent} from '../ast';+import type {+ MarkdownAstBlockContent,+ MarkdownAstListItem,+ MarkdownAstPhrasingContent,+ MarkdownAstTableCell,+ MarkdownAstTableRow,+} from '../ast';import {+ getMarkdownHelperOwnership,+ adoptMarkdownHelperNode,+ adoptMarkdownHelperNodes,markMarkdownTransformClaim,+ markMarkdownTransformTrusted,type MarkdownExtensionNode,+ type MarkdownHelperOwnership,type MarkdownTransform,} from './protocol';+/**+ * Ownership for a transform invoked outside Core's pipeline. It owns no+ * plugin name and no renderer, so a callback's extension node is rejected+ * while ordinary phrasing still works.+ */+const UNOWNED: MarkdownHelperOwnership = Object.freeze({+ pluginName: '\u0000unowned',+ hasRenderer: () => false,+ sourceUnchanged: false,+});+export interface MarkdownTextTransformContext {readonly parentType:'heading' | 'paragraph' | 'strong' | 'emphasis' | 'delete' | 'tableCell';@@ -46,30 +68,45 @@ export interface MarkdownTextTransformOptions<| ReadonlyArray<MarkdownAstPhrasingContent<Node>>;}+/**+ * Replacement nodes already placed in a document by this helper.+ *+ * A callback that builds a node per match hands over a fresh object, and+ * that object can go straight into the tree — copying it would allocate a+ * second one for nothing, and it is validated and frozen on the way in+ * either way. A callback that returns the SAME object for several matches+ * is the case that needs a copy: one object cannot occupy two positions in+ * a tree, and a node already frozen into an earlier position must not be+ * reused for a later one. Weak, so remembering a node never keeps it alive.+ */+const adoptedReplacements = new WeakSet<object>();++/** One reusable expression per transform invocation. */+interface Scanner {+ readonly pattern: RegExp;+ busy: boolean;+}+type AnyExtension = MarkdownExtensionNode;type Phrasing = MarkdownAstPhrasingContent<AnyExtension>;type Block = MarkdownAstBlockContent<AnyExtension>;-function sameItems<T>(- left: ReadonlyArray<T>,- right: ReadonlyArray<T>,-): boolean {- return (- left.length === right.length &&- left.every((value, index) => value === right[index])- );-}--function cloneReplacement<T>(value: T, seen = new Map<object, unknown>()): T {+/**+ * Copies a caller's node away from them. The cycle-tracking map is allocated+ * only once a nested object actually appears: the overwhelmingly common+ * replacement is a flat node of primitives, and this runs once per match.+ */+function cloneReplacement<T>(value: T, seen?: Map<object, unknown>): T {if (value == null || typeof value !== 'object') {return value;}- const cached = seen.get(value);+ const cached = seen?.get(value);if (cached != null) {return cached as T;}if (Array.isArray(value)) {const clone: unknown[] = [];+ seen ??= new Map<object, unknown>();seen.set(value, clone);for (const item of value) {clone.push(cloneReplacement(item, seen));@@ -80,9 +117,18 @@ function cloneReplacement<T>(value: T, seen = new Map<object, unknown>()): T {return value;}const clone: Record<string, unknown> = {};- seen.set(value, clone);- for (const [key, nested] of Object.entries(value)) {- clone[key] = cloneReplacement(nested, seen);+ let tracked = seen;+ for (const key in value) {+ const nested = (value as Record<string, unknown>)[key];+ if (nested != null && typeof nested === 'object') {+ if (tracked === undefined) {+ tracked = new Map<object, unknown>();+ tracked.set(value, clone);+ }+ clone[key] = cloneReplacement(nested, tracked);+ } else {+ clone[key] = nested;+ }}return clone as T;}@@ -107,8 +153,31 @@ function replaceTextNode<Node extends MarkdownExtensionNode>(value: string,options: MarkdownTextTransformOptions<Node>,parentType: MarkdownTextTransformContext['parentType'],+ ownership: MarkdownHelperOwnership,+ scanner: Scanner,): ReadonlyArray<Phrasing> | null {- const pattern = new RegExp(options.pattern.source, options.pattern.flags);+ // The conservative substring hint decides per text node, not just per+ // document: source without one cannot contain an eligible match, so most+ // prose is skipped before any scanning happens.+ // Disabled for case-insensitive patterns, exactly as the document-level+ // claim is, so JavaScript case folding stays the only authority.+ const required = options.pattern.ignoreCase+ ? undefined+ : options.requiredSubstrings;+ if (required != null && !required.some(hint => value.includes(hint))) {+ return null;+ }+ // One expression per transform invocation rather than one per text node.+ // A `replace` callback that re-enters this helper gets its own, so a+ // nested scan can never disturb the outer one's `lastIndex`.+ const pattern = scanner.busy+ ? new RegExp(options.pattern.source, options.pattern.flags)+ : scanner.pattern;+ const reentrant = pattern !== scanner.pattern;+ if (!reentrant) {+ scanner.busy = true;+ }+ pattern.lastIndex = 0;const output: Phrasing[] = [];let cursor = 0;let changed = false;@@ -145,11 +214,39 @@ function replaceTextNode<Node extends MarkdownExtensionNode>(output.push({type: 'text', value: value.slice(cursor, start)});}const replacement = options.replace(match, {parentType});- const replacements = Array.isArray(replacement)- ? replacement- : [replacement];- for (const replacementNode of replacements) {- output.push(cloneReplacement(replacementNode));+ // The callback's nodes are the only untrusted structure this helper can+ // introduce. Each one is checked here under the same rules Core applies+ // to newly-introduced nodes — representable data, no authored+ // provenance, no foreign extension ownership, no cycles — and frozen,+ // so the tree this transform returns needs no further checking. A node+ // the callback has handed over before is copied first; a fresh one is+ // adopted as it stands.+ if (Array.isArray(replacement)) {+ const adopted: Phrasing[] = [];+ for (const replacementNode of replacement) {+ adopted.push(+ adoptedReplacements.has(replacementNode)+ ? cloneReplacement(replacementNode)+ : replacementNode,+ );+ }+ adoptMarkdownHelperNodes(adopted, ownership);+ for (const node of adopted) {+ adoptedReplacements.add(node);+ }+ output.push(...adopted);+ } else {+ // One node is the overwhelmingly common reply, and it is worth its+ // own path: wrapping it in an array to share the loop above would+ // allocate twice per match, which on a large document is the+ // helper's dominant garbage.+ const single = replacement as Phrasing;+ const adopted = adoptedReplacements.has(single)+ ? cloneReplacement(single)+ : single;+ adoptMarkdownHelperNode(adopted, ownership);+ adoptedReplacements.add(adopted);+ output.push(adopted);}cursor = resolvedEnd;pattern.lastIndex =@@ -163,6 +260,9 @@ function replaceTextNode<Node extends MarkdownExtensionNode>(changed = true;}+ if (!reentrant) {+ scanner.busy = false;+ }if (!changed) {return null;}@@ -176,13 +276,30 @@ function transformPhrasing<Node extends MarkdownExtensionNode>(children: ReadonlyArray<Phrasing>,options: MarkdownTextTransformOptions<Node>,parentType: MarkdownTextTransformContext['parentType'],+ ownership: MarkdownHelperOwnership,+ scanner: Scanner,): ReadonlyArray<Phrasing> {- const next: Phrasing[] = [];- for (const node of children) {+ // Lazy rebuild: unchanged phrasing is returned as-is, so prose with no+ // match costs one walk and no allocation. Only the first change copies.+ let next: Phrasing[] | undefined;+ for (let index = 0; index < children.length; index++) {+ const node = children[index];if (node.type === 'text') {- next.push(- ...(replaceTextNode(node.value, options, parentType) ?? [node]),+ const replaced = replaceTextNode(+ node.value,+ options,+ parentType,+ ownership,+ scanner,);+ if (replaced == null) {+ next?.push(node);+ continue;+ }+ next ??= children.slice(0, index);+ for (const replacement of replaced) {+ next.push(replacement);+ }continue;}if (@@ -190,79 +307,172 @@ function transformPhrasing<Node extends MarkdownExtensionNode>(node.type === 'emphasis' ||node.type === 'delete') {- const nested = transformPhrasing(node.children, options, node.type);- next.push(- sameItems(node.children, nested) ? node : {...node, children: nested},+ const nested = transformPhrasing(+ node.children,+ options,+ node.type,+ ownership,+ scanner,);+ if (nested === node.children) {+ next?.push(node);+ continue;+ }+ next ??= children.slice(0, index);+ next.push({...node, children: nested});continue;}// Links, images, code, math, citations, breaks, and accepted extension// syntax are protected contexts owned by their existing semantics.- next.push(node);+ next?.push(node);}- return sameItems(children, next) ? children : next;+ return next ?? children;+}++function blockMayContainRequiredSource<Node extends MarkdownExtensionNode>(+ block: Block,+ source: string | undefined,+ options: MarkdownTextTransformOptions<Node>,+): boolean {+ const required = options.requiredSubstrings;+ if (source === undefined || required === undefined) {+ return true;+ }+ const start = block.position?.start.offset;+ const end = block.position?.end.offset;+ if (typeof start !== 'number' || typeof end !== 'number') {+ return true;+ }+ for (const hint of required) {+ const match = source.indexOf(hint, start);+ if (match >= 0 && match < end) {+ return true;+ }+ }+ return false;}function transformBlocks<Node extends MarkdownExtensionNode>(blocks: ReadonlyArray<Block>,options: MarkdownTextTransformOptions<Node>,+ ownership: MarkdownHelperOwnership,+ scanner: Scanner,+ source: string | undefined,): ReadonlyArray<Block> {- const next = blocks.map(block => {+ // Lazy rebuild, matching transformPhrasing: a block list with no match is+ // returned unchanged and allocates nothing.+ let next: Block[] | undefined;+ for (let index = 0; index < blocks.length; index++) {+ const block = blocks[index];+ if (!blockMayContainRequiredSource(block, source, options)) {+ next?.push(block);+ continue;+ }+ let replacement: Block;switch (block.type) {case 'heading':case 'paragraph': {- const children = transformPhrasing(block.children, options, block.type);- return children === block.children ? block : {...block, children};+ const children = transformPhrasing(+ block.children,+ options,+ block.type,+ ownership,+ scanner,+ );+ replacement =+ children === block.children ? block : {...block, children};+ break;}case 'blockquote': {- const children = transformBlocks(block.children, options);- return children === block.children ? block : {...block, children};+ const children = transformBlocks(+ block.children,+ options,+ ownership,+ scanner,+ source,+ );+ replacement =+ children === block.children ? block : {...block, children};+ break;}case 'list': {- let changed = false;- const children = block.children.map(item => {- const itemChildren = transformBlocks(item.children, options);+ let items: MarkdownAstListItem<AnyExtension>[] | undefined;+ for (+ let item_index = 0;+ item_index < block.children.length;+ item_index+++ ) {+ const item = block.children[item_index];+ const itemChildren = transformBlocks(+ item.children,+ options,+ ownership,+ scanner,+ source,+ );if (itemChildren === item.children) {- return item;+ items?.push(item);+ continue;}- changed = true;- return {...item, children: itemChildren};- });- return changed ? {...block, children} : block;+ items ??= block.children.slice(0, item_index);+ items.push({...item, children: itemChildren});+ }+ replacement = items == null ? block : {...block, children: items};+ break;}case 'table': {- let changed = false;- const children = block.children.map(row => {- let rowChanged = false;- const cells = row.children.map(cell => {+ let rows: MarkdownAstTableRow<AnyExtension>[] | undefined;+ for (+ let row_index = 0;+ row_index < block.children.length;+ row_index+++ ) {+ const row = block.children[row_index];+ let cells: MarkdownAstTableCell<AnyExtension>[] | undefined;+ for (+ let cell_index = 0;+ cell_index < row.children.length;+ cell_index+++ ) {+ const cell = row.children[cell_index];const cellChildren = transformPhrasing(cell.children,options,'tableCell',+ ownership,+ scanner,);if (cellChildren === cell.children) {- return cell;+ cells?.push(cell);+ continue;}- rowChanged = true;- return {...cell, children: cellChildren};- });- if (!rowChanged) {- return row;+ cells ??= row.children.slice(0, cell_index);+ cells.push({...cell, children: cellChildren});+ }+ if (cells == null) {+ rows?.push(row);+ continue;}- changed = true;- return {...row, children: cells};- });- return changed ? {...block, children} : block;+ rows ??= block.children.slice(0, row_index);+ rows.push({...row, children: cells});+ }+ replacement = rows == null ? block : {...block, children: rows};+ break;}case 'code':case 'math':case 'image':case 'thematicBreak':case 'extension':- return block;+ replacement = block;+ break;+ }+ if (next === undefined && replacement !== block) {+ next = blocks.slice(0, index);}- });- return sameItems(blocks, next) ? blocks : next;+ next?.push(replacement);+ }+ return next ?? blocks;}export function createMarkdownTextTransform(@@ -305,14 +515,40 @@ export function createMarkdownTextTransform(getEndIndex: options.getEndIndex,replace: options.replace,});- const transform: MarkdownTransform<MarkdownExtensionNode> = root => {- const children = transformBlocks(root.children, normalizedOptions);+ const transform: MarkdownTransform<MarkdownExtensionNode> = (+ root,+ context,+ ) => {+ const ownership = getMarkdownHelperOwnership(context) ?? UNOWNED;+ const children = transformBlocks(+ root.children,+ normalizedOptions,+ ownership,+ {+ pattern: new RegExp(+ normalizedOptions.pattern.source,+ normalizedOptions.pattern.flags,+ ),+ busy: false,+ },+ ownership.sourceUnchanged &&+ requiredSubstrings != null &&+ !normalizedOptions.pattern.ignoreCase+ ? context.source+ : undefined,+ );return children === root.children ? root : {...root, children};};+ // Every node this helper returns is either one Core already validated or+ // one `replaceTextNode` just validated against the same rules, so Core may+ // run it on the trusted path.+ const trusted = markMarkdownTransformTrusted(+ transform as MarkdownTransform<never>,+ ) as MarkdownTransform<MarkdownExtensionNode>;if (requiredSubstrings == null || options.pattern.ignoreCase) {- return transform;+ return trusted;}- return markMarkdownTransformClaim(transform, source =>+ return markMarkdownTransformClaim(trusted, source =>requiredSubstrings.some(required => source.includes(required)),);}