An open source design system that's fully customizable and agent ready
TypeScript
feat(markdown): add text transform helper (#6342)
Cindy Zhangeec3a1989274
diff --git a/.changeset/markdown-text-transform.md b/.changeset/markdown-text-transform.mdnew file mode 100644index 0000000000..e189f14f57--- /dev/null+++ b/.changeset/markdown-text-transform.md@@ -0,0 +1,9 @@+---+'@astryxdesign/core': patch+---++[feat] Markdown: add an immutable text-transform helper++Use `createMarkdownTextTransform()` to replace matching prose with typed Markdown nodes while preserving links, images, code, math, citations, and existing extension syntax as protected contexts.++@cixzhangdiff --git a/apps/storybook/stories/Markdown.demoPlugins.tsx b/apps/storybook/stories/Markdown.demoPlugins.tsxindex 0aa59767a0..96efef98b1 100644--- a/apps/storybook/stories/Markdown.demoPlugins.tsx+++ b/apps/storybook/stories/Markdown.demoPlugins.tsx@@ -9,6 +9,7 @@import {createMarkdownPlugin,+ createMarkdownTextTransform,type MarkdownExtensionNode,type MarkdownSyntaxPluginDefinition,} from '@astryxdesign/core/Markdown/plugins';@@ -20,6 +21,13 @@ type MentionNode = MarkdownExtensionNode<'inline'>;+type TodoNode = MarkdownExtensionNode<+ 'demo-todos',+ 'todo',+ {readonly label: string},+ 'inline'+>;+type CalloutNode = MarkdownExtensionNode<'demo-callouts','callout',@@ -101,9 +109,32 @@ const calloutDefinition = {},} satisfies MarkdownSyntaxPluginDefinition<'demo-callouts', CalloutNode>;+const todoPlugin = createMarkdownPlugin<'demo-todos', TodoNode>({+ name: 'demo-todos',+ apiVersion: 1,+ transform: createMarkdownTextTransform<TodoNode>({+ pattern: /\bTODO\b/g,+ requiredSubstrings: ['TODO'],+ replace: () => ({+ type: 'extension',+ plugin: 'demo-todos',+ name: 'todo',+ display: 'inline',+ data: {label: 'TODO'},+ }),+ }),+ renderers: {+ todo: {+ render: ({node}) => <mark>{node.data.label}</mark>,+ toText: node => node.data.label,+ },+ },+});+export const markdownDemoPlugins = [createMarkdownPlugin<'demo-mentions', MentionNode>(mentionDefinition),createMarkdownPlugin<'demo-callouts', CalloutNode>(calloutDefinition),+ todoPlugin,] as const;interface DelayedValue<Value> {diff --git a/apps/storybook/stories/Markdown.stories.tsx b/apps/storybook/stories/Markdown.stories.tsxindex 64c4ddc6b2..bd77222950 100644--- a/apps/storybook/stories/Markdown.stories.tsx+++ b/apps/storybook/stories/Markdown.stories.tsx@@ -494,7 +494,7 @@ export const SyntaxPlugins: Story = {<div style={{maxWidth: 680}}><Markdown plugins={markdownDemoPlugins}>{- '# Plugin composition\n\nHello @{Ada}. Ordinary **Markdown** keeps its behavior.\n\n:::note\nThis callout and mention are typed extension nodes.\n:::\n\nProtected contexts stay literal: `@{Linus}` and [@{Grace}](/people).'+ '# Plugin composition\n\nHello @{Ada}. Ordinary **Markdown** keeps its behavior, while TODO becomes a transform-owned node.\n\n:::note\nThis callout and mention are typed extension nodes.\n:::\n\nProtected contexts stay literal: `TODO @{Linus}` and [TODO @{Grace}](/people).'}</Markdown></div>diff --git a/packages/core/src/Markdown/Markdown.doc.mjs b/packages/core/src/Markdown/Markdown.doc.mjsindex 39c3d3a39e..a194547003 100644--- a/packages/core/src/Markdown/Markdown.doc.mjs+++ b/packages/core/src/Markdown/Markdown.doc.mjs@@ -252,6 +252,11 @@ export const docs = {description:'Use plugins created by createMarkdownPlugin for reusable syntax, immutable AST transforms, and typed extension rendering. Keep the ordered list stable while its syntax configuration is unchanged.',},+ {+ guidance: true,+ description:+ 'Use createMarkdownTextTransform for prose matching; it preserves code, links, images, citations, math, and accepted extension syntax as protected contexts. Provide requiredSubstrings only when they conservatively cover every possible match.',+ },{guidance: true,description:@@ -302,24 +307,22 @@ import {Text} from '@astryxdesign/core/Text';`,},{- label: 'Immutable transform plugin',+ label: 'Text transform helper',code: `import {Markdown} from '@astryxdesign/core/Markdown';-import {createMarkdownPlugin} from '@astryxdesign/core/Markdown/plugins';+import {+ createMarkdownPlugin,+ createMarkdownTextTransform,+} from '@astryxdesign/core/Markdown/plugins';const finalLabels = createMarkdownPlugin({name: 'final-labels',apiVersion: 1,- transform(root) {- return {- ...root,- children: root.children.map(node =>- node.type === 'heading'- ? {...node, children: [{type: 'text', value: 'Final'}]}- : node,- ),- };- },+ transform: createMarkdownTextTransform({+ pattern: /\\bDraft\\b/g,+ requiredSubstrings: ['Draft'],+ replace: () => ({type: 'text', value: 'Final'}),+ }),});<Markdown plugins={[finalLabels]}># Draft</Markdown>;@@ -553,6 +556,11 @@ export const docsZh = {description:'Use plugins created by createMarkdownPlugin for reusable syntax, immutable AST transforms, and typed extension rendering. Keep the ordered list stable while its syntax configuration is unchanged.',},+ {+ guidance: true,+ description:+ 'Use createMarkdownTextTransform for prose matching; it preserves code, links, images, citations, math, and accepted extension syntax as protected contexts. Provide requiredSubstrings only when they conservatively cover every possible match.',+ },{guidance: true,description:@@ -605,6 +613,11 @@ export const docsDense = {description:'Use plugins created by createMarkdownPlugin for reusable syntax, immutable AST transforms, and typed extension rendering. Keep the ordered list stable while its syntax configuration is unchanged.',},+ {+ guidance: true,+ description:+ 'Use createMarkdownTextTransform for prose matching; it preserves code, links, images, citations, math, and accepted extension syntax as protected contexts. Provide requiredSubstrings only when they conservatively cover every possible match.',+ },{guidance: true,description:diff --git a/packages/core/src/Markdown/Markdown.helpers.perf.test.ts b/packages/core/src/Markdown/Markdown.helpers.perf.test.tsnew file mode 100644index 0000000000..5c4dcd7ea8--- /dev/null+++ b/packages/core/src/Markdown/Markdown.helpers.perf.test.ts@@ -0,0 +1,89 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file Markdown.helpers.perf.test.ts+ * @input Deterministic 200/500-section Markdown and zero-work text helpers+ * @output Zero helper-transform invocations for hinted no-claim plugins+ * @position Helper dispatch regression; full FR23 evidence lands after all helpers+ */++import {describe, expect, it} from 'vitest';+import {parseMarkdown} from './parser';+import {createMarkdownPlugin} from './plugins';+import type {MarkdownTransform} from './plugins/protocol';+import {createMarkdownTextTransform} from './plugins/textTransform';++let transformCalls = 0;++function benchmarkDocument(sections: number): string {+ return Array.from({length: sections}, (_, index) =>+ [+ `## Section ${index}`,+ '',+ `AST-${index} belongs to @{owner-${index}} with TODO follow-up.`,+ '',+ `- First item ${index}`,+ '- Second item',+ '',+ `> Quoted detail ${index}`,+ '',+ '| Item | Detail |',+ '| --- | --- |',+ `| ${index} | ordinary prose |`,+ '',+ '```text',+ `opaque ${index}`,+ '```',+ ].join('\n'),+ ).join('\n\n');+}++function observeTransformCalls(+ transform: MarkdownTransform,+): MarkdownTransform {+ const observed: MarkdownTransform = (root, context) => {+ transformCalls++;+ return transform(root, context);+ };+ // Preserve the helper's private source-claim metadata on the observer.+ Object.defineProperties(+ observed,+ Object.getOwnPropertyDescriptors(transform),+ );+ return observed;+}++const zeroWorkPlugins = Array.from({length: 5}, (_, index) =>+ createMarkdownPlugin({+ name: `zero-work-${index}`,+ apiVersion: 1,+ transform: observeTransformCalls(+ createMarkdownTextTransform({+ pattern: new RegExp(`NEVER_MATCH_${index}`, 'g'),+ requiredSubstrings: [`NEVER_MATCH_${index}`],+ replace: () => {+ throw new Error('An unclaimed helper callback ran');+ },+ }),+ ),+ }),+);++describe('Markdown helper performance', () => {+ it.each([200, 500])(+ 'skips five hinted no-claim helpers at %i sections',+ sections => {+ const source = benchmarkDocument(sections);+ const baseline = parseMarkdown(source);+ transformCalls = 0;++ expect(parseMarkdown(source, {plugins: zeroWorkPlugins})).toEqual(+ baseline,+ );+ expect(parseMarkdown(source, {plugins: [...zeroWorkPlugins]})).toEqual(+ baseline,+ );+ expect(transformCalls).toBe(0);+ },+ );+});diff --git a/packages/core/src/Markdown/Markdown.public.test.ts b/packages/core/src/Markdown/Markdown.public.test.tsindex 12f5e62314..dd6cddf9a0 100644--- a/packages/core/src/Markdown/Markdown.public.test.ts+++ b/packages/core/src/Markdown/Markdown.public.test.ts@@ -15,7 +15,11 @@ import {parseMarkdownIncremental,visitMarkdownNodes,} from './index';-import {createMarkdownPlugin, isMarkdownExtensionNode} from './plugins';+import {+ createMarkdownPlugin,+ createMarkdownTextTransform,+ isMarkdownExtensionNode,+} from './plugins';import type {MarkdownExtensionNode,MarkdownSyntaxPluginDefinition,@@ -193,6 +197,7 @@ describe('Markdown public parser types', () => {expectTypeOf(nodes).toEqualTypeOf<InlineNode<PublicNode>[]>();expectTypeOf(visitMarkdownNodes).toBeFunction();+ expectTypeOf(createMarkdownTextTransform).toBeFunction();expectTypeOf(isMarkdownExtensionNode).toBeFunction();function compileOnlyPluginGuards() {diff --git a/packages/core/src/Markdown/plugins/index.ts b/packages/core/src/Markdown/plugins/index.tsindex 27319b888e..486c9de9ff 100644--- a/packages/core/src/Markdown/plugins/index.ts+++ b/packages/core/src/Markdown/plugins/index.ts@@ -7,6 +7,11 @@*/export {createMarkdownPlugin, isMarkdownExtensionNode} from './protocol';+export {createMarkdownTextTransform} from './textTransform';+export type {+ MarkdownTextTransformContext,+ MarkdownTextTransformOptions,+} from './textTransform';export type {MarkdownPluginData,MarkdownExtensionNode,diff --git a/packages/core/src/Markdown/plugins/protocol.ts b/packages/core/src/Markdown/plugins/protocol.tsindex a9058793aa..35d9a11cba 100644--- a/packages/core/src/Markdown/plugins/protocol.ts+++ b/packages/core/src/Markdown/plugins/protocol.ts@@ -101,10 +101,39 @@ export interface MarkdownTransformContext {report(message: string): void;}-export type MarkdownTransform = (- document: MarkdownAstRoot<MarkdownExtensionNode>,- context: MarkdownTransformContext,-) => MarkdownAstRoot<MarkdownExtensionNode>;+declare const markdownTransformNode: unique symbol;++export interface MarkdownTransform<Node extends MarkdownExtensionNode = never> {+ (+ document: MarkdownAstRoot<MarkdownExtensionNode>,+ context: MarkdownTransformContext,+ ): MarkdownAstRoot<MarkdownExtensionNode>;+ readonly [markdownTransformNode]?: Node;+}++const markdownTransformClaim = Symbol('MarkdownTransformClaim');++interface ClaimAwareMarkdownTransform<+ Node extends MarkdownExtensionNode = MarkdownExtensionNode,+> extends MarkdownTransform<Node> {+ readonly [markdownTransformClaim]?: (source: string) => boolean;+}++/** @internal Registers a cheap source-level claim check for a helper transform. */+export function markMarkdownTransformClaim<+ Node extends MarkdownExtensionNode = never,+>(+ transform: MarkdownTransform<Node>,+ claims: (source: string) => boolean,+): MarkdownTransform<Node> {+ Object.defineProperty(transform, markdownTransformClaim, {+ configurable: false,+ enumerable: false,+ value: claims,+ writable: false,+ });+ return transform;+}export interface MarkdownExtensionRenderer<Node extends MarkdownExtensionNode> {/** Pure render callback. Hooks belong in components returned by this callback. */@@ -130,7 +159,7 @@ export interface MarkdownSyntaxPluginDefinition<> extends MarkdownPluginDefinitionBase<Name> {readonly parseKey: string;readonly syntax: MarkdownSyntaxCapability<Node>;- readonly transform?: MarkdownTransform;+ readonly transform?: MarkdownTransform<Node>;readonly renderers: MarkdownExtensionRenderers<Node>;}@@ -138,7 +167,7 @@ export type MarkdownTransformPluginDefinition<Name extends string,Node extends MarkdownExtensionNode<Name> = never,> = MarkdownPluginDefinitionBase<Name> & {- readonly transform: MarkdownTransform;+ readonly transform: MarkdownTransform<Node>;readonly parseKey?: never;readonly syntax?: never;} & ([Node] extends [never]@@ -323,7 +352,8 @@ export interface PreparedSyntaxContribution {interface PreparedTransform {readonly pluginName: string;- readonly transform: MarkdownTransform;+ readonly transform: MarkdownTransform<MarkdownExtensionNode>;+ readonly claims?: (source: string) => boolean;}interface PreparedRenderer {@@ -417,6 +447,9 @@ export function prepareMarkdownPlugins(transforms.push({pluginName: definition.name,transform: definition.transform,+ claims: (definition.transform as ClaimAwareMarkdownTransform)[+ markdownTransformClaim+ ],});}if (definition.renderers != null) {@@ -990,9 +1023,19 @@ export function applyMarkdownTransforms<Node extends MarkdownExtensionNode>(}const pluginNames = new Set(plugins.entries.map(entry => entry.name));const rendererKeys = new Set(plugins.renderers.keys());- const sourceHeadingMarkers = markSourceHeadings(root);- let document = freezeAst(root);+ let document: MarkdownAstRoot<Node> = root;+ let sourceHeadingMarkers: Set<SourceHeadingMarker> | undefined;+ let started = false;+ let priorTransformChangedTree = false;for (const prepared of plugins.transforms) {+ if (!priorTransformChangedTree && prepared.claims?.(source) === false) {+ continue;+ }+ if (!started) {+ sourceHeadingMarkers = markSourceHeadings(root);+ document = freezeAst(root);+ started = true;+ }try {const next = prepared.transform(document, {source,@@ -1025,7 +1068,7 @@ export function applyMarkdownTransforms<Node extends MarkdownExtensionNode>(pluginNames,rendererKeys,positions,- sourceHeadingMarkers,+ sourceHeadingMarkers as ReadonlySet<SourceHeadingMarker>,prepared.pluginName,existingExtensions,display,@@ -1033,6 +1076,7 @@ export function applyMarkdownTransforms<Node extends MarkdownExtensionNode>() {throw new TypeError('Transform returned an invalid Markdown document');}+ priorTransformChangedTree = true;document = freezeAst(next) as MarkdownAstRoot<Node>;} catch (error) {reportMarkdownPluginFailure(prepared.pluginName, 'transform', error);diff --git a/packages/core/src/Markdown/plugins/textTransform.test.ts b/packages/core/src/Markdown/plugins/textTransform.test.tsnew file mode 100644index 0000000000..c7b2eff66e--- /dev/null+++ b/packages/core/src/Markdown/plugins/textTransform.test.ts@@ -0,0 +1,355 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file textTransform.test.ts+ * @input Text-transform helper selectors, callbacks, and protected contexts+ * @output Behavioral, ordering, typing, and work-count regression coverage+ * @position Acceptance tests for the optional Markdown text helper+ */++import {describe, expect, expectTypeOf, it} from 'vitest';+import {parseInline, parseMarkdown} from '../parser';+import {createMarkdownPlugin} from './protocol';+import type {MarkdownExtensionNode} from './protocol';+import {createMarkdownTextTransform} from './textTransform';+import type {MarkdownTextTransformContext} from './textTransform';++type TodoNode = MarkdownExtensionNode<+ 'todo-labels',+ 'todo',+ {readonly label: string},+ 'inline'+>;++function todoPlugin(+ onReplace?: (context: MarkdownTextTransformContext) => void,+) {+ return createMarkdownPlugin<'todo-labels', TodoNode>({+ name: 'todo-labels',+ apiVersion: 1,+ transform: createMarkdownTextTransform<TodoNode>({+ pattern: /TODO/g,+ requiredSubstrings: ['TODO'],+ replace(_match, context) {+ onReplace?.(context);+ return {+ type: 'extension',+ plugin: 'todo-labels',+ name: 'todo',+ display: 'inline',+ data: {label: 'TODO'},+ };+ },+ }),+ renderers: {+ todo: {+ render: ({node}) => node.data.label,+ toText: node => node.data.label,+ },+ },+ });+}++function extensionCount(value: unknown): number {+ if (Array.isArray(value)) {+ return value.reduce<number>(+ (count, child) => count + extensionCount(child),+ 0,+ );+ }+ if (value == null || typeof value !== 'object') {+ return 0;+ }+ const record = value as Record<string, unknown>;+ const own = record.type === 'extension' ? 1 : 0;+ return (+ own ++ Object.values(record).reduce<number>(+ (count, child) => count + extensionCount(child),+ 0,+ )+ );+}++describe('createMarkdownTextTransform', () => {+ it('replaces eligible prose and preserves protected contexts', () => {+ const parents: string[] = [];+ const plugin = todoPlugin(context => parents.push(context.parentType));+ const blocks = parseMarkdown(+ [+ 'TODO **TODO** [TODO](/tasks) `TODO` [source]',+ '',+ '> TODO',+ '',+ '| Item |',+ '| --- |',+ '| TODO |',+ '',+ '```text',+ 'TODO',+ '```',+ ].join('\n'),+ {plugins: [plugin], sourceIds: new Set(['source'])},+ );++ expect(extensionCount(blocks)).toBe(4);+ expect(parents).toEqual(['paragraph', 'strong', 'paragraph', 'tableCell']);+ expect(JSON.stringify(blocks)).toContain('"type":"link","href":"/tasks"');+ expect(JSON.stringify(blocks)).toContain('"type":"code","content":"TODO"');+ });++ it('runs helpers in plugin order and supports removal', () => {+ const replace = (name: string, pattern: RegExp, value: string) =>+ createMarkdownPlugin({+ name,+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern,+ requiredSubstrings: [pattern.source],+ replace: () => (value === '' ? [] : {type: 'text' as const, value}),+ }),+ });++ expect(+ parseMarkdown('A', {+ plugins: [replace('a-to-b', /A/g, 'B'), replace('b-to-c', /B/g, 'C')],+ }),+ ).toMatchObject([+ {+ type: 'paragraph',+ children: [{type: 'text', content: 'C'}],+ },+ ]);+ expect(+ parseMarkdown('Remove me', {+ plugins: [replace('remove', /Remove /g, '')],+ }),+ ).toMatchObject([+ {type: 'paragraph', children: [{type: 'text', content: 'me'}]},+ ]);+ });++ it('clones a hoisted replacement for each claim', () => {+ const replacement = {type: 'text' as const, value: 'done'};+ const plugin = createMarkdownPlugin({+ name: 'hoisted-replacement',+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern: /X/g,+ requiredSubstrings: ['X'],+ replace: () => replacement,+ }),+ });++ expect(parseMarkdown('X X', {plugins: [plugin]})).toMatchObject([+ {+ type: 'paragraph',+ children: [+ {type: 'text', content: 'done'},+ {type: 'text', content: ' '},+ {type: 'text', content: 'done'},+ ],+ },+ ]);+ });++ it('does not invoke callbacks for absent or protected-only matches', () => {+ let replacements = 0;+ const plugin = todoPlugin(() => replacements++);++ parseMarkdown('No work here.', {plugins: [plugin]});+ expect(replacements).toBe(0);+ parseMarkdown('`TODO`\n\n```text\nTODO\n```', {plugins: [plugin]});+ expect(replacements).toBe(0);+ });++ it('uses case-insensitive source claims with case-insensitive patterns', () => {+ const plugin = createMarkdownPlugin({+ name: 'case-insensitive',+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern: /todo/gi,+ requiredSubstrings: ['todo'],+ replace: () => ({type: 'text', value: 'done'}),+ }),+ });++ expect(parseMarkdown('TODO', {plugins: [plugin]})).toMatchObject([+ {type: 'paragraph', children: [{type: 'text', content: 'done'}]},+ ]);+ const unicodeCaseFold = createMarkdownPlugin({+ name: 'unicode-case-fold',+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern: /Σ/gi,+ requiredSubstrings: ['Σ'],+ replace: () => ({type: 'text', value: 'sigma'}),+ }),+ });+ expect(parseMarkdown('ς', {plugins: [unicodeCaseFold]})).toMatchObject([+ {type: 'paragraph', children: [{type: 'text', content: 'sigma'}]},+ ]);+ const digits = createMarkdownPlugin({+ name: 'digits-without-prefilter',+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern: /\d+/g,+ replace: match => ({type: 'text', value: `[${match[0]}]`}),+ }),+ });+ expect(parseMarkdown('Item 42', {plugins: [digits]})).toMatchObject([+ {+ type: 'paragraph',+ children: [+ {type: 'text', content: 'Item '},+ {type: 'text', content: '[42]'},+ ],+ },+ ]);+ });++ it('supports bounded claim extension and decline', () => {+ let replacements = 0;+ const plugin = createMarkdownPlugin({+ name: 'hash-tags',+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern: /#/g,+ requiredSubstrings: ['#'],+ getEndIndex(text, match) {+ const end = text.indexOf(' ', match.index);+ return end < 0 ? text.length : end;+ },+ replace(match) {+ replacements++;+ return {type: 'text', value: match.index === 0 ? 'tag' : 'nested'};+ },+ }),+ });++ expect(parseInline('#one #two', {plugins: [plugin]})).toMatchObject([+ {type: 'text', content: 'tag'},+ {type: 'text', content: ' '},+ {type: 'text', content: 'nested'},+ ]);+ expect(replacements).toBe(2);+ });++ it('supports accepted zero-width insertions at text boundaries', () => {+ const start = createMarkdownPlugin({+ name: 'insert-start',+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern: /^/g,+ replace: () => ({type: 'text', value: '['}),+ }),+ });+ const end = createMarkdownPlugin({+ name: 'insert-end',+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern: /$/g,+ replace: () => ({type: 'text', value: ']'}),+ }),+ });++ expect(parseMarkdown('value', {plugins: [start]})).toMatchObject([+ {+ type: 'paragraph',+ children: [+ {type: 'text', content: '['},+ {type: 'text', content: 'value'},+ ],+ },+ ]);+ expect(parseMarkdown('value', {plugins: [end]})).toMatchObject([+ {+ type: 'paragraph',+ children: [+ {type: 'text', content: 'value'},+ {type: 'text', content: ']'},+ ],+ },+ ]);+ });++ it('advances declined zero-width matches by Unicode code point', () => {+ let replacements = 0;+ const astral = createMarkdownPlugin({+ name: 'astral-zero-width',+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern: /(?=😀)/gu,+ requiredSubstrings: ['😀'],+ getEndIndex: () => false,+ replace: () => {+ replacements++;+ return {type: 'text', value: 'unexpected'};+ },+ }),+ });+ const end = createMarkdownPlugin({+ name: 'end-zero-width',+ apiVersion: 1,+ transform: createMarkdownTextTransform({+ pattern: /$/gu,+ requiredSubstrings: ['😀'],+ getEndIndex: () => false,+ replace: () => {+ replacements++;+ return {type: 'text', value: 'unexpected'};+ },+ }),+ });++ expect(parseMarkdown('😀', {plugins: [astral, end]})).toMatchObject([+ {type: 'paragraph', children: [{type: 'text', content: '😀'}]},+ ]);+ expect(replacements).toBe(0);+ });++ it('requires a global pattern and keeps the helper typed as a transform', () => {+ expect(() =>+ createMarkdownTextTransform({+ pattern: /TODO/,+ requiredSubstrings: ['TODO'],+ replace: () => ({type: 'text', value: 'done'}),+ }),+ ).toThrow(/global flag/);+ expect(() =>+ createMarkdownTextTransform({+ pattern: /TODO/g,+ requiredSubstrings: [] as never,+ replace: () => ({type: 'text', value: 'done'}),+ }),+ ).toThrow(/requiredSubstring/);+ expect(() =>+ createMarkdownTextTransform({+ pattern: /TODO/g,+ requiredSubstrings: [42] as never,+ replace: () => ({type: 'text', value: 'done'}),+ }),+ ).toThrow(/requiredSubstrings/);+ const ownedTransform = createMarkdownTextTransform<TodoNode>({+ pattern: /TODO/g,+ requiredSubstrings: ['TODO'],+ replace: () => ({+ type: 'extension',+ plugin: 'todo-labels',+ name: 'todo',+ display: 'inline',+ data: {label: 'TODO'},+ }),+ });+ function compileOnlyRendererRequirement() {+ // @ts-expect-error a helper that owns extension nodes requires renderers+ createMarkdownPlugin({+ name: 'todo-labels',+ apiVersion: 1,+ transform: ownedTransform,+ });+ }+ expectTypeOf(compileOnlyRendererRequirement).toBeFunction();+ expectTypeOf(createMarkdownTextTransform).toBeFunction();+ });+});diff --git a/packages/core/src/Markdown/plugins/textTransform.ts b/packages/core/src/Markdown/plugins/textTransform.tsnew file mode 100644index 0000000000..ff8229df61--- /dev/null+++ b/packages/core/src/Markdown/plugins/textTransform.ts@@ -0,0 +1,318 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file textTransform.ts+ * @input Regex selectors and replacement callbacks for eligible Markdown prose+ * @output Immutable Markdown transforms with protected-context traversal+ * @position Optional helper layer compiled onto the core transform protocol+ */++import type {MarkdownAstBlockContent, MarkdownAstPhrasingContent} from '../ast';+import {+ markMarkdownTransformClaim,+ type MarkdownExtensionNode,+ type MarkdownTransform,+} from './protocol';++export interface MarkdownTextTransformContext {+ readonly parentType:+ 'heading' | 'paragraph' | 'strong' | 'emphasis' | 'delete' | 'tableCell';+}++export interface MarkdownTextTransformOptions<+ Node extends MarkdownExtensionNode = never,+> {+ /** Global expression matched against eligible built-in prose only. */+ readonly pattern: RegExp;+ /**+ * Literal substrings such that source containing an eligible match contains+ * at least one. Core uses them to skip the transform without walking the AST+ * when none are present. Omit this optimization when no conservative literal+ * exists; the helper will inspect eligible prose on every run. This fast path+ * is also disabled for case-insensitive patterns so JavaScript case folding+ * remains exact.+ */+ readonly requiredSubstrings?: readonly [string, ...string[]];+ /** Optionally decline or extend a regex claim. */+ readonly getEndIndex?: (+ text: string,+ match: RegExpExecArray,+ ) => number | false;+ replace(+ match: RegExpExecArray,+ context: MarkdownTextTransformContext,+ ):+ | MarkdownAstPhrasingContent<Node>+ | ReadonlyArray<MarkdownAstPhrasingContent<Node>>;+}++type AnyExtension = MarkdownExtensionNode;+type Phrasing = MarkdownAstPhrasingContent<AnyExtension>;+type Block = MarkdownAstBlockContent<AnyExtension>;++function sameItems<T>(+ left: ReadonlyArray<T>,+ right: ReadonlyArray<T>,+): boolean {+ return (+ left.length === right.length &&+ left.every((value, index) => value === right[index])+ );+}++function cloneReplacement<T>(value: T, seen = new Map<object, unknown>()): T {+ if (value == null || typeof value !== 'object') {+ return value;+ }+ const cached = seen.get(value);+ if (cached != null) {+ return cached as T;+ }+ if (Array.isArray(value)) {+ const clone: unknown[] = [];+ seen.set(value, clone);+ for (const item of value) {+ clone.push(cloneReplacement(item, seen));+ }+ return clone as T;+ }+ if (Object.getPrototypeOf(value) !== Object.prototype) {+ return value;+ }+ const clone: Record<string, unknown> = {};+ seen.set(value, clone);+ for (const [key, nested] of Object.entries(value)) {+ clone[key] = cloneReplacement(nested, seen);+ }+ return clone as T;+}++function advanceStringIndex(+ value: string,+ index: number,+ unicode: boolean,+): number {+ if (!unicode || index >= value.length) {+ return index + 1;+ }+ const first = value.charCodeAt(index);+ if (first < 0xd800 || first > 0xdbff || index + 1 >= value.length) {+ return index + 1;+ }+ const second = value.charCodeAt(index + 1);+ return second >= 0xdc00 && second <= 0xdfff ? index + 2 : index + 1;+}++function replaceTextNode<Node extends MarkdownExtensionNode>(+ value: string,+ options: MarkdownTextTransformOptions<Node>,+ parentType: MarkdownTextTransformContext['parentType'],+): ReadonlyArray<Phrasing> | null {+ const pattern = new RegExp(options.pattern.source, options.pattern.flags);+ const output: Phrasing[] = [];+ let cursor = 0;+ let changed = false;+ let match: RegExpExecArray | null;++ while ((match = pattern.exec(value)) != null) {+ const start = match.index;+ const defaultEnd = start + match[0].length;+ const resolvedEnd = options.getEndIndex?.(value, match) ?? defaultEnd;+ if (resolvedEnd === false) {+ if (match[0].length === 0) {+ pattern.lastIndex = advanceStringIndex(+ value,+ pattern.lastIndex,+ pattern.unicode || pattern.flags.includes('v'),+ );+ }+ continue;+ }+ if (+ !Number.isInteger(resolvedEnd) ||+ resolvedEnd < defaultEnd ||+ resolvedEnd < start ||+ resolvedEnd > value.length+ ) {+ throw new TypeError(+ 'Markdown text transform returned an invalid end index',+ );+ }+ if (start < cursor) {+ continue;+ }+ if (start > cursor) {+ output.push({type: 'text', value: value.slice(cursor, start)});+ }+ const replacement = options.replace(match, {parentType});+ const replacements = Array.isArray(replacement)+ ? replacement+ : [replacement];+ for (const replacementNode of replacements) {+ output.push(cloneReplacement(replacementNode));+ }+ cursor = resolvedEnd;+ pattern.lastIndex =+ resolvedEnd === start+ ? advanceStringIndex(+ value,+ pattern.lastIndex,+ pattern.unicode || pattern.flags.includes('v'),+ )+ : Math.max(pattern.lastIndex, resolvedEnd);+ changed = true;+ }++ if (!changed) {+ return null;+ }+ if (cursor < value.length) {+ output.push({type: 'text', value: value.slice(cursor)});+ }+ return output;+}++function transformPhrasing<Node extends MarkdownExtensionNode>(+ children: ReadonlyArray<Phrasing>,+ options: MarkdownTextTransformOptions<Node>,+ parentType: MarkdownTextTransformContext['parentType'],+): ReadonlyArray<Phrasing> {+ const next: Phrasing[] = [];+ for (const node of children) {+ if (node.type === 'text') {+ next.push(+ ...(replaceTextNode(node.value, options, parentType) ?? [node]),+ );+ continue;+ }+ if (+ node.type === 'strong' ||+ node.type === 'emphasis' ||+ node.type === 'delete'+ ) {+ const nested = transformPhrasing(node.children, options, node.type);+ next.push(+ sameItems(node.children, nested) ? node : {...node, children: nested},+ );+ continue;+ }+ // Links, images, code, math, citations, breaks, and accepted extension+ // syntax are protected contexts owned by their existing semantics.+ next.push(node);+ }+ return sameItems(children, next) ? children : next;+}++function transformBlocks<Node extends MarkdownExtensionNode>(+ blocks: ReadonlyArray<Block>,+ options: MarkdownTextTransformOptions<Node>,+): ReadonlyArray<Block> {+ const next = blocks.map(block => {+ switch (block.type) {+ case 'heading':+ case 'paragraph': {+ const children = transformPhrasing(block.children, options, block.type);+ return children === block.children ? block : {...block, children};+ }+ case 'blockquote': {+ const children = transformBlocks(block.children, options);+ return children === block.children ? block : {...block, children};+ }+ case 'list': {+ let changed = false;+ const children = block.children.map(item => {+ const itemChildren = transformBlocks(item.children, options);+ if (itemChildren === item.children) {+ return item;+ }+ changed = true;+ return {...item, children: itemChildren};+ });+ return changed ? {...block, children} : block;+ }+ case 'table': {+ let changed = false;+ const children = block.children.map(row => {+ let rowChanged = false;+ const cells = row.children.map(cell => {+ const cellChildren = transformPhrasing(+ cell.children,+ options,+ 'tableCell',+ );+ if (cellChildren === cell.children) {+ return cell;+ }+ rowChanged = true;+ return {...cell, children: cellChildren};+ });+ if (!rowChanged) {+ return row;+ }+ changed = true;+ return {...row, children: cells};+ });+ return changed ? {...block, children} : block;+ }+ case 'code':+ case 'math':+ case 'image':+ case 'thematicBreak':+ case 'extension':+ return block;+ }+ });+ return sameItems(blocks, next) ? blocks : next;+}++export function createMarkdownTextTransform(+ options: MarkdownTextTransformOptions<never>,+): MarkdownTransform<never>;+export function createMarkdownTextTransform<Node extends MarkdownExtensionNode>(+ options: MarkdownTextTransformOptions<Node>,+): MarkdownTransform<Node>;+export function createMarkdownTextTransform(+ options: MarkdownTextTransformOptions<MarkdownExtensionNode>,+): MarkdownTransform<MarkdownExtensionNode> {+ if (!options.pattern.global) {+ throw new TypeError(+ 'Markdown text transform patterns must use the global flag',+ );+ }+ if (+ options.requiredSubstrings != null &&+ (!Array.isArray(options.requiredSubstrings) ||+ options.requiredSubstrings.length === 0 ||+ options.requiredSubstrings.some(+ value => typeof value !== 'string' || value === '',+ ))+ ) {+ throw new TypeError(+ 'Markdown text transform requiredSubstrings must be non-empty strings',+ );+ }+ const requiredSubstrings =+ options.requiredSubstrings == null+ ? undefined+ : (Object.freeze([...options.requiredSubstrings]) as readonly [+ string,+ ...string[],+ ]);+ const normalizedOptions: MarkdownTextTransformOptions<MarkdownExtensionNode> =+ Object.freeze({+ pattern: new RegExp(options.pattern.source, options.pattern.flags),+ requiredSubstrings,+ getEndIndex: options.getEndIndex,+ replace: options.replace,+ });+ const transform: MarkdownTransform<MarkdownExtensionNode> = root => {+ const children = transformBlocks(root.children, normalizedOptions);+ return children === root.children ? root : {...root, children};+ };+ if (requiredSubstrings == null || options.pattern.ignoreCase) {+ return transform;+ }+ return markMarkdownTransformClaim(transform, source =>+ requiredSubstrings.some(required => source.includes(required)),+ );+}