Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Keep nested component bytes when editing a sibling in WYSIWYG (#6231)

Dimitri POSTOLOVb1afebb3f017
diff --git a/.changeset/nested-component-bytes.md b/.changeset/nested-component-bytes.md
new file mode 100644
index 000000000..22b6a1761
--- /dev/null
+++ b/.changeset/nested-component-bytes.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Editing text beside a multi-line component inside a list item, a blockquote that is not a GitHub alert, or another component no longer re-indents that component or turns it into a parse error.
diff --git a/packages/app/package.json b/packages/app/package.json
index b1bd4eab7..3a1a20f4a 100644
--- a/packages/app/package.json
+++ b/packages/app/package.json
@@ -41,7 +41,7 @@
"measure:stress": "bash scripts/measure-stress.sh",
"measure:sweep": "bash scripts/measure-sweep.sh",
"perf:compare": "bash scripts/perf-compare.sh",
- "test:e2e": "playwright test tests/stress/empty-bullet-before-link.e2e.ts tests/stress/list-item-drag.e2e.ts tests/stress/ux-interactions.e2e.ts tests/stress/file-tree-create.e2e.ts tests/stress/file-tree-drag-to-root.e2e.ts tests/stress/file-tree-deselect-to-root.e2e.ts tests/stress/file-tree-collapse-spaced-folder.e2e.ts tests/stress/sidebar-file-list-fills.e2e.ts tests/stress/file-tree-sticky-collapse.e2e.ts tests/stress/file-tree-collapse-persistence.e2e.ts tests/stress/file-tree-fractional-zoom.e2e.ts tests/stress/create-then-rename-editable.e2e.ts tests/stress/find-replace.e2e.ts tests/stress/external-link-cue.e2e.ts tests/stress/editor-tabs.e2e.ts tests/stress/crdt-stress.e2e.ts tests/stress/slash-command.e2e.ts tests/stress/paste-fidelity.e2e.ts tests/stress/fr-7a-disconnect-source-mode.e2e.ts tests/stress/docs-open.e2e.ts tests/stress/frozen-table-headers.e2e.ts tests/stress/asset-embed.e2e.ts tests/stress/asset-embed-advanced.e2e.ts tests/stress/asset-click-dispatch.e2e.ts tests/stress/handoff.e2e.ts tests/stress/multi-agent-presence.e2e.ts tests/stress/editor-mode-persistence.e2e.ts tests/stress/lint-config-toggle.e2e.ts tests/stress/plugin-enable-notice.e2e.ts tests/stress/schema-edit-fields-view.e2e.ts tests/stress/sidebar-search-pill.e2e.ts tests/stress/navigation-history.e2e.ts tests/stress/command-palette-semantic.e2e.ts tests/stress/agent-activity-panel.e2e.ts tests/stress/drop-pipeline-auto-open.e2e.ts tests/stress/command-palette-flicker.e2e.ts tests/stress/cm6-list-hanging-indent.e2e.ts tests/stress/ng7-rapid-nav-coherence.e2e.ts tests/stress/reveal-on-activate.e2e.ts tests/stress/selection-indicator.e2e.ts tests/stress/new-file-cross-doc-bleed.e2e.ts tests/stress/new-item-shortcut-fast-path.e2e.ts tests/stress/editor-mode-flip-cross-doc-bleed.e2e.ts tests/stress/editor-area-viewport-resize.e2e.ts tests/stress/qa-sidebar-responsive.e2e.ts tests/stress/prd-6955-reassertion-repro.e2e.ts tests/stress/prd-6955-reassertion-wedge.e2e.ts tests/stress/prd-6914-repro.e2e.ts tests/stress/showall-lazy-tree.e2e.ts tests/stress/tabs-component-strip.e2e.ts tests/stress/source-find-scroll.e2e.ts tests/stress/harness-app-warmth.e2e.ts tests/stress/qa-canary-authoring-both-modes.e2e.ts tests/stress/qa-canary-live-typing.e2e.ts tests/stress/keystroke-cadence-danger-space.e2e.ts tests/stress/jsx-unregistered-backspace-delete.e2e.ts tests/stress/jsx-unregistered-ime-concurrent.e2e.ts tests/stress/jsx-backspace-delete.e2e.ts tests/stress/link-authoring-bytes.e2e.ts tests/stress/link-authoring-apex.e2e.ts tests/stress/qa019-suffix-precedence.e2e.ts tests/stress/command-palette-parity.e2e.ts tests/stress/settings-search.e2e.ts tests/stress/settings-close-button.e2e.ts tests/stress/okf-generated-index-settings.e2e.ts tests/stress/okf-recommended-skill-install.e2e.ts tests/stress/chip-popover-positioning.e2e.ts tests/stress/docs-open-scroll-restore.e2e.ts tests/stress/file-tree-decoration-chip-strip-repro.e2e.ts tests/stress/grip-click-nodeselect.e2e.ts tests/stress/jsx-halo-semantics.e2e.ts tests/stress/jsx-image-body-click.e2e.ts tests/stress/jsx-prop-panel-placeholder.e2e.ts tests/stress/jsx-range-encompass-halo.e2e.ts tests/stress/jsx-wildcard-convert.e2e.ts tests/stress/markdown-lint.e2e.ts tests/stress/frontmatter-schema-badge.e2e.ts tests/stress/frontmatter-glob-picker.e2e.ts tests/stress/unified-problems.e2e.ts tests/stress/mid-type-recovery.e2e.ts tests/stress/observer-a-multi-client.e2e.ts tests/stress/outline-toolbar-occlusion.e2e.ts tests/stress/selection-surface-pane-clip.e2e.ts tests/stress/quiet-tree-hidden-doc.e2e.ts tests/stress/raw-mdx-fallback-onblur-upgrade.e2e.ts tests/stress/rename-noext-probe.e2e.ts tests/stress/show-ok-folders.e2e.ts tests/stress/source-polish.e2e.ts tests/stress/warm-skeleton-scroll-restore.e2e.ts tests/stress/props-toggle-scroll-preserve.e2e.ts tests/stress/yjs-no-dual-import.e2e.ts tests/stress/graph-panel-surfaces.e2e.ts tests/stress/asset-embed-real-fidelity.e2e.ts tests/stress/image-invalid-placeholder.e2e.ts tests/stress/local-target-audit.e2e.ts tests/stress/rename-content-preservation.e2e.ts tests/stress/rename-consolidation.e2e.ts tests/stress/outline-active-heading.e2e.ts tests/stress/outline-navigation.e2e.ts tests/stress/prd-7627-default-swatch.e2e.ts tests/stress/clipboard-dollar-math-source-fallback.e2e.ts tests/stress/source-mode-undo-keymap.e2e.ts tests/stress/source-undo-mode-flip.e2e.ts tests/stress/timeline-recovered-restore.e2e.ts tests/stress/blank-line-preservation.e2e.ts tests/stress/landing-helpers.e2e.ts tests/stress/agent-follow-scroll.e2e.ts tests/stress/mode-switch-landing.e2e.ts tests/stress/mode-switch-ordinal-divergence.e2e.ts tests/stress/comment-annotation-visibility.e2e.ts tests/stress/comment-opens-in-panel.e2e.ts tests/stress/qa-miles-edge-blanks.e2e.ts tests/stress/prop-upload.e2e.ts tests/stress/slash-command-auto-open.e2e.ts tests/stress/clipboard-relative-url-source-fallback.e2e.ts tests/stress/editor-split-view.e2e.ts tests/stress/language-load-failure.e2e.ts tests/stress/language-first-paint.e2e.ts tests/stress/user-text-direction.e2e.ts tests/stress/pseudolocale.e2e.ts tests/stress/language-picker.e2e.ts tests/stress/zh-hans-coverage-sweep.e2e.ts tests/stress/preview-tab-promotion.e2e.ts tests/stress/cv-paint-lock-click.e2e.ts tests/stress/cv-auto-paint-lock-click.e2e.ts tests/stress/harness-port-ownership.e2e.ts tests/stress/server-reuse-admission.e2e.ts tests/stress/server-reuse-reset.e2e.ts tests/stress/prebuilt-dev-serving.e2e.ts tests/stress/history-traversal-preview-tab.e2e.ts tests/stress/tab-close-scroll-restore.e2e.ts tests/stress/skills-sidebar-open.e2e.ts tests/stress/trailing-affordance.e2e.ts tests/stress/text-doc-composer-inset.e2e.ts tests/stress/composer-control-alignment.e2e.ts tests/stress/agents-settings-accessible-names.e2e.ts tests/stress/composer-growth-eof-reveal.e2e.ts tests/stress/theme-fade.e2e.ts tests/stress/theme-color-conversion.e2e.ts tests/stress/composer-open-caret-reveal.e2e.ts tests/stress/spellcheck-browser-absence.e2e.ts tests/stress/agents-panel-reload-visibility.e2e.ts tests/stress/rail-flex-distribution-law.e2e.ts tests/stress/audit-409-console-record.e2e.ts tests/stress/pagelist-refresh-lifetime.e2e.ts tests/stress/list-keymap.e2e.ts tests/stress/okignore-settings.e2e.ts tests/stress/assertion-timeout.e2e.ts tests/stress/loop-markers.e2e.ts tests/stress/template-admission-detail.e2e.ts",
+ "test:e2e": "playwright test tests/stress/empty-bullet-before-link.e2e.ts tests/stress/list-item-drag.e2e.ts tests/stress/ux-interactions.e2e.ts tests/stress/file-tree-create.e2e.ts tests/stress/file-tree-drag-to-root.e2e.ts tests/stress/file-tree-deselect-to-root.e2e.ts tests/stress/file-tree-collapse-spaced-folder.e2e.ts tests/stress/sidebar-file-list-fills.e2e.ts tests/stress/file-tree-sticky-collapse.e2e.ts tests/stress/file-tree-collapse-persistence.e2e.ts tests/stress/file-tree-fractional-zoom.e2e.ts tests/stress/create-then-rename-editable.e2e.ts tests/stress/find-replace.e2e.ts tests/stress/external-link-cue.e2e.ts tests/stress/editor-tabs.e2e.ts tests/stress/crdt-stress.e2e.ts tests/stress/slash-command.e2e.ts tests/stress/paste-fidelity.e2e.ts tests/stress/fr-7a-disconnect-source-mode.e2e.ts tests/stress/docs-open.e2e.ts tests/stress/frozen-table-headers.e2e.ts tests/stress/asset-embed.e2e.ts tests/stress/asset-embed-advanced.e2e.ts tests/stress/asset-click-dispatch.e2e.ts tests/stress/handoff.e2e.ts tests/stress/multi-agent-presence.e2e.ts tests/stress/editor-mode-persistence.e2e.ts tests/stress/lint-config-toggle.e2e.ts tests/stress/plugin-enable-notice.e2e.ts tests/stress/schema-edit-fields-view.e2e.ts tests/stress/sidebar-search-pill.e2e.ts tests/stress/navigation-history.e2e.ts tests/stress/command-palette-semantic.e2e.ts tests/stress/agent-activity-panel.e2e.ts tests/stress/drop-pipeline-auto-open.e2e.ts tests/stress/command-palette-flicker.e2e.ts tests/stress/cm6-list-hanging-indent.e2e.ts tests/stress/ng7-rapid-nav-coherence.e2e.ts tests/stress/reveal-on-activate.e2e.ts tests/stress/selection-indicator.e2e.ts tests/stress/new-file-cross-doc-bleed.e2e.ts tests/stress/new-item-shortcut-fast-path.e2e.ts tests/stress/editor-mode-flip-cross-doc-bleed.e2e.ts tests/stress/editor-area-viewport-resize.e2e.ts tests/stress/qa-sidebar-responsive.e2e.ts tests/stress/prd-6955-reassertion-repro.e2e.ts tests/stress/prd-6955-reassertion-wedge.e2e.ts tests/stress/prd-6914-repro.e2e.ts tests/stress/showall-lazy-tree.e2e.ts tests/stress/tabs-component-strip.e2e.ts tests/stress/source-find-scroll.e2e.ts tests/stress/harness-app-warmth.e2e.ts tests/stress/qa-canary-authoring-both-modes.e2e.ts tests/stress/qa-canary-live-typing.e2e.ts tests/stress/keystroke-cadence-danger-space.e2e.ts tests/stress/jsx-unregistered-backspace-delete.e2e.ts tests/stress/jsx-unregistered-ime-concurrent.e2e.ts tests/stress/jsx-backspace-delete.e2e.ts tests/stress/link-authoring-bytes.e2e.ts tests/stress/link-authoring-apex.e2e.ts tests/stress/qa019-suffix-precedence.e2e.ts tests/stress/command-palette-parity.e2e.ts tests/stress/settings-search.e2e.ts tests/stress/settings-close-button.e2e.ts tests/stress/okf-generated-index-settings.e2e.ts tests/stress/okf-recommended-skill-install.e2e.ts tests/stress/chip-popover-positioning.e2e.ts tests/stress/docs-open-scroll-restore.e2e.ts tests/stress/file-tree-decoration-chip-strip-repro.e2e.ts tests/stress/grip-click-nodeselect.e2e.ts tests/stress/jsx-halo-semantics.e2e.ts tests/stress/jsx-image-body-click.e2e.ts tests/stress/jsx-prop-panel-placeholder.e2e.ts tests/stress/jsx-range-encompass-halo.e2e.ts tests/stress/jsx-wildcard-convert.e2e.ts tests/stress/markdown-lint.e2e.ts tests/stress/frontmatter-schema-badge.e2e.ts tests/stress/frontmatter-glob-picker.e2e.ts tests/stress/unified-problems.e2e.ts tests/stress/mid-type-recovery.e2e.ts tests/stress/observer-a-multi-client.e2e.ts tests/stress/outline-toolbar-occlusion.e2e.ts tests/stress/selection-surface-pane-clip.e2e.ts tests/stress/quiet-tree-hidden-doc.e2e.ts tests/stress/raw-mdx-fallback-onblur-upgrade.e2e.ts tests/stress/rename-noext-probe.e2e.ts tests/stress/show-ok-folders.e2e.ts tests/stress/source-polish.e2e.ts tests/stress/warm-skeleton-scroll-restore.e2e.ts tests/stress/props-toggle-scroll-preserve.e2e.ts tests/stress/yjs-no-dual-import.e2e.ts tests/stress/graph-panel-surfaces.e2e.ts tests/stress/asset-embed-real-fidelity.e2e.ts tests/stress/image-invalid-placeholder.e2e.ts tests/stress/local-target-audit.e2e.ts tests/stress/rename-content-preservation.e2e.ts tests/stress/rename-consolidation.e2e.ts tests/stress/outline-active-heading.e2e.ts tests/stress/outline-navigation.e2e.ts tests/stress/prd-7627-default-swatch.e2e.ts tests/stress/clipboard-dollar-math-source-fallback.e2e.ts tests/stress/source-mode-undo-keymap.e2e.ts tests/stress/source-undo-mode-flip.e2e.ts tests/stress/timeline-recovered-restore.e2e.ts tests/stress/blank-line-preservation.e2e.ts tests/stress/nested-component-sibling-edit-bytes.e2e.ts tests/stress/landing-helpers.e2e.ts tests/stress/agent-follow-scroll.e2e.ts tests/stress/mode-switch-landing.e2e.ts tests/stress/mode-switch-ordinal-divergence.e2e.ts tests/stress/comment-annotation-visibility.e2e.ts tests/stress/comment-opens-in-panel.e2e.ts tests/stress/qa-miles-edge-blanks.e2e.ts tests/stress/prop-upload.e2e.ts tests/stress/slash-command-auto-open.e2e.ts tests/stress/clipboard-relative-url-source-fallback.e2e.ts tests/stress/editor-split-view.e2e.ts tests/stress/language-load-failure.e2e.ts tests/stress/language-first-paint.e2e.ts tests/stress/user-text-direction.e2e.ts tests/stress/pseudolocale.e2e.ts tests/stress/language-picker.e2e.ts tests/stress/zh-hans-coverage-sweep.e2e.ts tests/stress/preview-tab-promotion.e2e.ts tests/stress/cv-paint-lock-click.e2e.ts tests/stress/cv-auto-paint-lock-click.e2e.ts tests/stress/harness-port-ownership.e2e.ts tests/stress/server-reuse-admission.e2e.ts tests/stress/server-reuse-reset.e2e.ts tests/stress/prebuilt-dev-serving.e2e.ts tests/stress/history-traversal-preview-tab.e2e.ts tests/stress/tab-close-scroll-restore.e2e.ts tests/stress/skills-sidebar-open.e2e.ts tests/stress/trailing-affordance.e2e.ts tests/stress/text-doc-composer-inset.e2e.ts tests/stress/composer-control-alignment.e2e.ts tests/stress/agents-settings-accessible-names.e2e.ts tests/stress/composer-growth-eof-reveal.e2e.ts tests/stress/theme-fade.e2e.ts tests/stress/theme-color-conversion.e2e.ts tests/stress/composer-open-caret-reveal.e2e.ts tests/stress/spellcheck-browser-absence.e2e.ts tests/stress/agents-panel-reload-visibility.e2e.ts tests/stress/rail-flex-distribution-law.e2e.ts tests/stress/audit-409-console-record.e2e.ts tests/stress/pagelist-refresh-lifetime.e2e.ts tests/stress/list-keymap.e2e.ts tests/stress/okignore-settings.e2e.ts tests/stress/assertion-timeout.e2e.ts tests/stress/loop-markers.e2e.ts tests/stress/template-admission-detail.e2e.ts",
"test:e2e:install-browsers": "playwright install chromium webkit firefox",
"test:visual": "playwright test --config playwright.visual.config.ts",
"test:visual:update": "playwright test --config playwright.visual.config.ts --update-snapshots",
diff --git a/packages/app/tests/stress/nested-component-sibling-edit-bytes.e2e.ts b/packages/app/tests/stress/nested-component-sibling-edit-bytes.e2e.ts
new file mode 100644
index 000000000..83e21b99d
--- /dev/null
+++ b/packages/app/tests/stress/nested-component-sibling-edit-bytes.e2e.ts
@@ -0,0 +1,149 @@
+import { randomUUID } from 'node:crypto';
+import type { Page } from '@playwright/test';
+import {
+ expect,
+ placeCaretAtEndOfText,
+ test,
+ waitForActiveProviderSynced as waitForProvider,
+} from './_helpers';
+
+interface NestedComponentCase {
+ title: string;
+ seed: string;
+ editedText: string;
+}
+
+const TYPED = 'Z';
+
+const CASES: readonly NestedComponentCase[] = [
+ {
+ title: 'a Callout inside a list item',
+ seed: '# Nested\n\n- item\n\n <Callout type="info">\n body line\n </Callout>\n\n- next\n\nAfter.\n',
+ editedText: 'item',
+ },
+ {
+ title: 'a Callout inside a blockquote',
+ seed: '# Nested\n\n> quote\n>\n> <Callout type="info">\n> body line\n> </Callout>\n\nAfter.\n',
+ editedText: 'quote',
+ },
+ {
+ title: 'an unknown component inside a list item',
+ seed: '# Nested\n\n- item\n\n <Unknown a="1">\n body line\n </Unknown>\n\n- next\n\nAfter.\n',
+ editedText: 'item',
+ },
+ {
+ title: 'an unknown component inside a Callout',
+ seed: '# Nested\n\n<Callout type="info">\n para\n\n <Unknown a="1">\n body line\n </Unknown>\n</Callout>\n\nAfter.\n',
+ editedText: 'para',
+ },
+ {
+ title: 'a Callout inside a Callout',
+ seed: '# Nested\n\n<Callout type="info">\n para\n\n <Callout type="warning">\n body line\n </Callout>\n</Callout>\n\nAfter.\n',
+ editedText: 'para',
+ },
+ {
+ title: 'a Callout inside an ordered list item',
+ seed: '# Nested\n\n10) item\n\n <Callout type="info">\n body line\n </Callout>\n\n11) next\n\nAfter.\n',
+ editedText: 'item',
+ },
+ {
+ title: 'a Callout inside a blockquote inside a list item',
+ seed: '# Nested\n\n- item\n\n > <Callout type="info">\n > body line\n > </Callout>\n\n- next\n\nAfter.\n',
+ editedText: 'item',
+ },
+ {
+ title: 'a Callout inside a list item inside a blockquote',
+ seed: '# Nested\n\n> - item\n>\n> <Callout type="info">\n> body line\n> </Callout>\n\nAfter.\n',
+ editedText: 'item',
+ },
+ {
+ title: 'a Callout on the first line of a list item',
+ seed: '# Nested\n\n- <Callout type="info">\n body line\n </Callout>\n- item\n\nAfter.\n',
+ editedText: 'item',
+ },
+ {
+ title: 'a multi-line expression inside a list item',
+ seed: "# Nested\n\n- item\n\n {\n 'body line'\n }\n\n- next\n\nAfter.\n",
+ editedText: 'item',
+ },
+];
+
+function readSource(page: Page): Promise<string> {
+ return page.evaluate(() => window.__activeProvider?.document.getText('source').toString() ?? '');
+}
+
+async function openSeededDoc(page: Page, docName: string): Promise<void> {
+ await page.goto(`/#/${docName}`);
+ await waitForProvider(page);
+ await page.waitForSelector('.ProseMirror:not(.composer-prosemirror)');
+ await page.waitForFunction(
+ () => {
+ const editor = window.__activeEditor;
+ if (!editor?.state.doc.textContent.includes('body line')) return false;
+ let unconvertedWildcard = false;
+ editor.state.doc.descendants((node) => {
+ if (
+ node.type.name === 'jsxComponent' &&
+ (node.attrs.componentName === 'Unknown' || node.attrs.kind === 'expression')
+ ) {
+ unconvertedWildcard = true;
+ }
+ });
+ return !unconvertedWildcard;
+ },
+ null,
+ { timeout: 10_000 },
+ );
+}
+
+function readParseErrorFallbacks(page: Page): Promise<string[]> {
+ return page.evaluate(() => {
+ const editor = window.__activeEditor;
+ if (!editor) throw new Error('readParseErrorFallbacks: window.__activeEditor not set');
+ const reasons: string[] = [];
+ editor.state.doc.descendants((node) => {
+ const reason = typeof node.attrs.reason === 'string' ? node.attrs.reason : '';
+ if (node.type.name === 'rawMdxFallback' && !reason.startsWith('Unregistered component:')) {
+ reasons.push(reason);
+ }
+ });
+ return reasons;
+ });
+}
+
+test.describe('a WYSIWYG edit beside a nested multi-line component', () => {
+ for (const c of CASES) {
+ test(`keeps the bytes of ${c.title}`, async ({ page, api }) => {
+ const docName = `test-nested-component-${randomUUID().slice(0, 8)}`;
+ const expected = c.seed.replace(c.editedText, `${c.editedText}${TYPED}`);
+ await api.createPage(`${docName}.md`);
+ await api.testReset(docName);
+ await api.replaceDoc(docName, c.seed);
+ await openSeededDoc(page, docName);
+
+ expect(
+ await readSource(page),
+ 'opening the document changed its bytes before any edit, so the sibling edit below is not what rewrote the nested component',
+ ).toBe(c.seed);
+
+ await placeCaretAtEndOfText(page, c.editedText);
+ await page.keyboard.type(TYPED);
+
+ await expect
+ .poll(() => readSource(page), { timeout: 10_000 })
+ .toContain(`${c.editedText}${TYPED}`);
+ expect(
+ await readSource(page),
+ 'the sibling edit re-serialized the container, and the untouched nested component did not keep its bytes',
+ ).toBe(expected);
+
+ await page.reload();
+ await openSeededDoc(page, docName);
+ expect(await readSource(page)).toBe(expected);
+ expect(
+ await readParseErrorFallbacks(page),
+ 'after a reload the nested component renders as a parse-error box',
+ ).toEqual([]);
+ });
+ }
+});
diff --git a/packages/core/src/markdown/merged-walker.ts b/packages/core/src/markdown/merged-walker.ts
index bfd7e3109..861003fb4 100644
--- a/packages/core/src/markdown/merged-walker.ts
+++ b/packages/core/src/markdown/merged-walker.ts
@@ -2,7 +2,11 @@ import type { Nodes, Parent, Root } from 'mdast';
import { SKIP, visit } from 'unist-util-visit';
import type { VFile } from 'vfile';
import { promoteInParent } from './autolink-promotion.ts';
-import { applyPositionSliceToNode } from './position-slice.ts';
+import {
+ applyPositionSliceToNode,
+ needsNestedRawRelativization,
+ relativizeNestedMdxSourceRaw,
+} from './position-slice.ts';
import { promoteTagsInParent } from './tag-promotion.ts';
import { KNOWN_MDAST_TYPES, toRawMdxFallbackMdast } from './unknown-mdast-guard.ts';
@@ -12,6 +16,7 @@ export function mergedPostParseWalkerPlugin() {
const debug = typeof process !== 'undefined' && process.env?.OK_DEBUG_POSITION_SLICE === '1';
+ let sawNestedRawSourceMdx = false;
visit(tree, (node, index, parent) => {
if (
parent !== undefined &&
@@ -38,6 +43,9 @@ export function mergedPostParseWalkerPlugin() {
debug,
parent as { type?: string } | undefined,
);
+ if (needsNestedRawRelativization(node as Nodes, parent as { type?: string } | undefined)) {
+ sawNestedRawSourceMdx = true;
+ }
if (node.type === 'heading' && parent !== undefined && typeof index === 'number') {
const headingEnd = node.position?.end?.line;
@@ -55,5 +63,6 @@ export function mergedPostParseWalkerPlugin() {
}
}
});
+ if (sawNestedRawSourceMdx) relativizeNestedMdxSourceRaw(tree, source);
};
}
diff --git a/packages/core/src/markdown/position-slice.ts b/packages/core/src/markdown/position-slice.ts
index 480c28077..9c20b6e68 100644
--- a/packages/core/src/markdown/position-slice.ts
+++ b/packages/core/src/markdown/position-slice.ts
@@ -1,4 +1,6 @@
-import type { Nodes, Root } from 'mdast';
+import type { Blockquote, FootnoteDefinition, ListItem, Nodes, Root } from 'mdast';
+import type { MdxFlowExpression, MdxTextExpression } from 'mdast-util-mdx-expression';
+import type { MdxJsxFlowElement, MdxJsxTextElement } from 'mdast-util-mdx-jsx';
import { visit } from 'unist-util-visit';
import type { VFile } from 'vfile';
import { type EscapeProvenanceEntry, hasEscapeProvenance } from './mdast-augmentation.ts';
@@ -596,6 +598,273 @@ function definitionLabelCloseIndex(slice: string): number {
return -1;
}
+type RawSourceMdxNode =
+ | MdxJsxFlowElement
+ | MdxJsxTextElement
+ | MdxFlowExpression
+ | MdxTextExpression;
+
+type LinePrefixContainer = Blockquote | ListItem | FootnoteDefinition;
+
+interface PrefixConsumption {
+ offset: number;
+ complete: boolean;
+}
+
+interface LineContent extends PrefixConsumption {
+ contentStart: number;
+}
+
+const RAW_SOURCE_MDX_TYPES: ReadonlySet<string> = new Set([
+ 'mdxJsxFlowElement',
+ 'mdxJsxTextElement',
+ 'mdxFlowExpression',
+ 'mdxTextExpression',
+]);
+
+const LIST_MARKER_RE = /^(?:[-+*]|\d{1,9}[.)])/;
+
+const FOOTNOTE_CONTINUATION_WIDTH = 4;
+
+interface PrefixScan {
+ readonly source: string;
+ readonly lineStarts: readonly number[];
+ readonly widths: Map<LinePrefixContainer, number | null>;
+ readonly contentOffsets: Map<LinePrefixContainer, number | null>;
+}
+
+function isRawSourceMdxNode(node: Nodes): node is RawSourceMdxNode {
+ return RAW_SOURCE_MDX_TYPES.has(node.type);
+}
+
+function isLinePrefixContainer(node: Nodes): node is LinePrefixContainer {
+ return (
+ node.type === 'blockquote' || node.type === 'listItem' || node.type === 'footnoteDefinition'
+ );
+}
+
+export function needsNestedRawRelativization(
+ node: Nodes,
+ parent: { type?: string } | undefined,
+): boolean {
+ if (parent === undefined || parent.type === 'root' || !isRawSourceMdxNode(node)) return false;
+ if (node.type === 'mdxJsxFlowElement' && parent.type === 'mdxJsxFlowElement') return true;
+ const raw = node.data?.sourceRaw;
+ return typeof raw === 'string' && raw.includes('\n');
+}
+
+function createPrefixScan(source: string): PrefixScan {
+ const lineStarts = [0];
+ for (
+ let newline = source.indexOf('\n');
+ newline !== -1;
+ newline = source.indexOf('\n', newline + 1)
+ ) {
+ lineStarts.push(newline + 1);
+ }
+ return { source, lineStarts, widths: new Map(), contentOffsets: new Map() };
+}
+
+function lineStartOf(scan: PrefixScan, offset: number): number {
+ const { lineStarts } = scan;
+ let low = 0;
+ let high = lineStarts.length - 1;
+ while (low < high) {
+ const middle = (low + high + 1) >> 1;
+ if (lineStarts[middle] <= offset) low = middle;
+ else high = middle - 1;
+ }
+ return lineStarts[low];
+}
+
+function isBlankFrom(source: string, offset: number): boolean {
+ let cursor = offset;
+ while (source[cursor] === ' ' || source[cursor] === '\t' || source[cursor] === '\r') cursor++;
+ return cursor >= source.length || source[cursor] === '\n';
+}
+
+function isSpaceRun(source: string, from: number, to: number): boolean {
+ for (let cursor = from; cursor < to; cursor++) {
+ if (source[cursor] !== ' ') return false;
+ }
+ return true;
+}
+
+function findFirstLineContentOffset(
+ container: ListItem | FootnoteDefinition,
+ source: string,
+ start: number,
+): number | null {
+ if (container.type === 'footnoteDefinition') {
+ const label = /\[\^[^\]\n]*\]:[ \t]*/y;
+ label.lastIndex = start;
+ const match = label.exec(source);
+ return match ? start + match[0].length : null;
+ }
+ const marker = LIST_MARKER_RE.exec(source.slice(start, start + 11));
+ if (!marker) return null;
+ let cursor = start + marker[0].length;
+ while (source[cursor] === ' ') cursor++;
+ return cursor;
+}
+
+function firstLineContentOffset(
+ scan: PrefixScan,
+ container: ListItem | FootnoteDefinition,
+ start: number,
+): number | null {
+ const memo = scan.contentOffsets.get(container);
+ if (memo !== undefined) return memo;
+ const offset = findFirstLineContentOffset(container, scan.source, start);
+ scan.contentOffsets.set(container, offset);
+ return offset;
+}
+
+function containerWidth(
+ scan: PrefixScan,
+ containers: readonly LinePrefixContainer[],
+ index: number,
+): number | null {
+ const container = containers[index];
+ const memo = scan.widths.get(container);
+ if (memo !== undefined) return memo;
+ let width: number | null = null;
+ if (container.type === 'footnoteDefinition') {
+ width = FOOTNOTE_CONTINUATION_WIDTH;
+ } else if (container.type === 'listItem') {
+ width = listItemWidth(scan, container, containers.slice(0, index));
+ }
+ scan.widths.set(container, width);
+ return width;
+}
+
+function listItemWidth(
+ scan: PrefixScan,
+ item: ListItem,
+ outer: readonly LinePrefixContainer[],
+): number | null {
+ const { source } = scan;
+ const start = item.position?.start?.offset;
+ if (typeof start !== 'number') return null;
+ const before = consumeLinePrefixes(scan, lineStartOf(scan, start), outer);
+ if (!before.complete || before.offset > start) return null;
+ if (!isSpaceRun(source, before.offset, start)) return null;
+ const marker = LIST_MARKER_RE.exec(source.slice(start, start + 11));
+ if (!marker) return null;
+ const afterMarker = start + marker[0].length;
+ let spaces = 0;
+ while (source[afterMarker + spaces] === ' ') spaces++;
+ if (source[afterMarker + spaces] === '\t') return null;
+ const contentGap = isBlankFrom(source, afterMarker) ? 1 : spaces;
+ return start - before.offset + marker[0].length + contentGap;
+}
+
+function consumeLinePrefix(
+ scan: PrefixScan,
+ offset: number,
+ lineStart: number,
+ containers: readonly LinePrefixContainer[],
+ index: number,
+): number | null {
+ const { source } = scan;
+ const container = containers[index];
+ if (container.type === 'blockquote') {
+ let cursor = offset;
+ while (source[cursor] === ' ') cursor++;
+ if (source[cursor] !== '>') return null;
+ cursor++;
+ if (source[cursor] === ' ' || source[cursor] === '\t') cursor++;
+ return cursor;
+ }
+ const start = container.position?.start?.offset;
+ if (typeof start !== 'number') return null;
+ if (start >= lineStart) return firstLineContentOffset(scan, container, start);
+ const width = containerWidth(scan, containers, index);
+ if (width === null) return null;
+ let cursor = offset;
+ while (cursor < offset + width && source[cursor] === ' ') cursor++;
+ if (cursor === offset + width) return cursor;
+ return isBlankFrom(source, cursor) ? cursor : null;
+}
+
+function consumeLinePrefixes(
+ scan: PrefixScan,
+ lineStart: number,
+ containers: readonly LinePrefixContainer[],
+): PrefixConsumption {
+ let offset = lineStart;
+ for (let index = 0; index < containers.length; index++) {
+ const next = consumeLinePrefix(scan, offset, lineStart, containers, index);
+ if (next === null) return { offset, complete: false };
+ offset = next;
+ }
+ return { offset, complete: true };
+}
+
+function relativizeNestedRaw(
+ node: RawSourceMdxNode,
+ parent: Nodes,
+ scan: PrefixScan,
+ consumeLine: (lineStart: number) => LineContent,
+): void {
+ const { source } = scan;
+ const raw = node.data?.sourceRaw;
+ const start = node.position?.start?.offset;
+ if (typeof raw !== 'string' || typeof start !== 'number') return;
+ if (source.slice(start, start + raw.length) !== raw) return;
+ const lines = raw.split('\n');
+ let lineStart = start + lines[0].length + 1;
+ for (let i = 1; i < lines.length; i++) {
+ const line = lines[i];
+ const consumed = consumeLine(lineStart);
+ lines[i] = line.slice(Math.min(consumed.offset - lineStart, line.length));
+ lineStart += line.length + 1;
+ }
+ let leadingIndent = '';
+ if (node.type === 'mdxJsxFlowElement' && parent.type === 'mdxJsxFlowElement') {
+ const content = consumeLine(lineStartOf(scan, start));
+ if (content.complete && content.contentStart === start) {
+ leadingIndent = source.slice(content.offset, start);
+ }
+ }
+ const relativized = leadingIndent + lines.join('\n');
+ if (relativized !== raw) node.data = { ...node.data, sourceRaw: relativized };
+}
+
+export function relativizeNestedMdxSourceRaw(tree: Root, source: string): void {
+ const scan = createPrefixScan(source);
+ const containers: LinePrefixContainer[] = [];
+ let lineContents = new Map<number, LineContent>();
+ const consumeLine = (lineStart: number): LineContent => {
+ let content = lineContents.get(lineStart);
+ if (content === undefined) {
+ const consumed = consumeLinePrefixes(scan, lineStart, containers);
+ let contentStart = consumed.offset;
+ while (source[contentStart] === ' ') contentStart++;
+ content = { ...consumed, contentStart };
+ lineContents.set(lineStart, content);
+ }
+ return content;
+ };
+ const walk = (node: Nodes, parent: Nodes): void => {
+ if (parent.type !== 'root' && isRawSourceMdxNode(node)) {
+ relativizeNestedRaw(node, parent, scan, consumeLine);
+ }
+ if (!('children' in node)) return;
+ const isContainer = isLinePrefixContainer(node);
+ if (isContainer) {
+ containers.push(node);
+ lineContents = new Map();
+ }
+ for (const child of node.children) walk(child as Nodes, node);
+ if (isContainer) {
+ containers.pop();
+ lineContents = new Map();
+ }
+ };
+ for (const child of tree.children) walk(child, tree);
+}
+
export function positionSlicePlugin() {
return (tree: Root, file: VFile) => {
const source = typeof file.value === 'string' ? file.value : '';
@@ -603,8 +872,11 @@ export function positionSlicePlugin() {
const debug = typeof process !== 'undefined' && process.env?.OK_DEBUG_POSITION_SLICE === '1';
+ let sawNestedRawSourceMdx = false;
visit(tree, (node: Nodes, _index, parent) => {
applyPositionSliceToNode(node, source, debug, parent ?? undefined);
+ if (needsNestedRawRelativization(node, parent ?? undefined)) sawNestedRawSourceMdx = true;
});
+ if (sawNestedRawSourceMdx) relativizeNestedMdxSourceRaw(tree, source);
};
}