Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Fix literal Markdown handling in generated indexes (PRD-8392) (#4510)

serafin-garciad53feceb70c6
diff --git a/.changeset/literal-generated-index-labels.md b/.changeset/literal-generated-index-labels.md
new file mode 100644
index 000000000..7c1aaf74c
--- /dev/null
+++ b/.changeset/literal-generated-index-labels.md
@@ -0,0 +1,6 @@
+---
+'@inkeep/open-knowledge': patch
+'@inkeep/open-knowledge-core': minor
+---
+
+Preserve generated-index metadata and folder labels as literal text. Labels containing Markdown punctuation may gain backslash escapes in machine-owned index files. On upgrade, changed enabled indexes are rewritten once; type labels that once rendered as the same heading remain separate. Remove the obsolete `headingContentIdentity` core export.
diff --git a/.changeset/parser-escape-provenance.md b/.changeset/parser-escape-provenance.md
new file mode 100644
index 000000000..c7ec826c3
--- /dev/null
+++ b/.changeset/parser-escape-provenance.md
@@ -0,0 +1,5 @@
+---
+'@inkeep/open-knowledge': patch
+---
+
+Keep escaped Markdown punctuation literal beside automatic emails and in multiline containers.
diff --git a/docs/content/plugins/okf.mdx b/docs/content/plugins/okf.mdx
index d5ce38c69..42ec4c098 100644
--- a/docs/content/plugins/okf.mdx
+++ b/docs/content/plugins/okf.mdx
@@ -26,7 +26,7 @@ The skill is optional. The plugin's checks are the same with or without it.
**Maintain index.md** creates and maintains a navigation file in every folder that contains Markdown, listing that folder's documents by type and linking its subfolders.
-Generated indexes are machine-owned: OpenKnowledge rewrites them as documents change, so manual edits are replaced. Turning the setting on shows a disclosure before any files are written. In a Git project it also adds a scoped `.gitattributes` rule (`merge=union`). A local or CLI merge then combines concurrent index changes instead of conflicting. A merge run on GitHub or GitLab ignores `.gitattributes`, so index changes there can still conflict and need resolving by hand. After a merge across a layout change the combined file can list a document twice until the index is next rebuilt. If another attribute already controls those paths, index maintenance pauses and the plugin page explains what to resolve.
+Generated indexes are machine-owned: OpenKnowledge rewrites them as documents change, so manual edits are replaced. Titles, descriptions, folder labels, and type headings render as literal Markdown text; the generated Markdown source backslash-escapes ASCII punctuation in those values. Control characters in those values are replaced with the Unicode replacement character. Parser-reserved Private Use Area code points U+E005–U+E009 and U+E100–U+E103 are also replaced with the Unicode replacement character. Type groups compare single-line, NFC-normalized labels, so whitespace- or normalization-equivalent values share a section while punctuation-distinct values remain separate. Turning the setting on shows a disclosure before any files are written. In a Git project it also adds a scoped `.gitattributes` rule (`merge=union`). A local or CLI merge then combines concurrent index changes instead of conflicting. A merge run on GitHub or GitLab ignores `.gitattributes`, so index changes there can still conflict and need resolving by hand. After a merge across a layout change the combined file can list a document twice until the index is next rebuilt. If another attribute already controls those paths, index maintenance pauses and the plugin page explains what to resolve.
Turning it off stops rebuilds and removes the Git rule, but leaves the generated files in place.
diff --git a/packages/core/schema-snapshot.json b/packages/core/schema-snapshot.json
index fc0f4837e..b7689e5a6 100644
--- a/packages/core/schema-snapshot.json
+++ b/packages/core/schema-snapshot.json
@@ -886,6 +886,11 @@
"version": "11.0.0",
"hasOptions": false
},
+ {
+ "name": "escape-provenance",
+ "version": null,
+ "hasOptions": false
+ },
{
"name": "remark-mdx-agnostic",
"version": null,
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index a6cd6ad58..b7227837f 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -863,7 +863,6 @@ export {
MIN_CARRIED_LEADING_EMPTIES,
MIN_CARRIED_TRAILING_EMPTIES,
} from './markdown/doc-edge-blank-runs.ts';
-export { headingContentIdentity } from './markdown/heading-content-identity.ts';
export {
HTML_MAX_BYTES,
HtmlPayloadTooLargeError,
@@ -1021,6 +1020,7 @@ export {
maskNonRenderingContexts,
skipInlineCode,
} from './markdown/non-rendering-contexts.ts';
+export { isMutatingParserReservation } from './markdown/parser-reservations.ts';
export { normalizeReferenceLabel } from './markdown/reference-label.ts';
export { normalizeDocRelativeAssetUrl } from './markdown/resolve-image-url.ts';
export {
diff --git a/packages/core/src/markdown/autolink-promotion.ts b/packages/core/src/markdown/autolink-promotion.ts
index dc6d874b5..8be571ac2 100644
--- a/packages/core/src/markdown/autolink-promotion.ts
+++ b/packages/core/src/markdown/autolink-promotion.ts
@@ -1,5 +1,10 @@
import type { Link, Parent, Text } from 'mdast';
-import { deriveFragmentPosition } from './promoter-position.ts';
+import {
+ deriveFragmentPosition,
+ escapedValueOffsets,
+ isEscapeDerivedRun,
+ sliceTextWithProvenance,
+} from './promoter-position.ts';
const AUTOLINK_IN_TEXT_RE = /<([a-zA-Z][a-zA-Z0-9+.-]*:[^\s<>]+)>/g;
@@ -14,6 +19,7 @@ export function promoteInParent(parent: Parent, source: string = ''): void {
}
const text = (child as Text).value;
+ const escaped = escapedValueOffsets(child as Text);
AUTOLINK_IN_TEXT_RE.lastIndex = 0;
const segments: Parent['children'] = [];
@@ -25,22 +31,18 @@ export function promoteInParent(parent: Parent, source: string = ''): void {
const fullMatch = match[0];
const uri = match[1];
const matchStart = match.index;
+ if (isEscapeDerivedRun(escaped, matchStart, 1)) continue;
if (matchStart > lastIndex) {
- const lead: Text = { type: 'text', value: text.slice(lastIndex, matchStart) };
- const pos = deriveFragmentPosition(source, child as Text, lastIndex, matchStart);
- if (pos) lead.position = pos;
- segments.push(lead);
+ segments.push(sliceTextWithProvenance(source, child as Text, lastIndex, matchStart));
}
- const innerText: Text = { type: 'text', value: uri };
- const innerPos = deriveFragmentPosition(
+ const innerText = sliceTextWithProvenance(
source,
child as Text,
matchStart + 1,
matchStart + 1 + uri.length,
);
- if (innerPos) innerText.position = innerPos;
const linkNode: Link & { data: { sourceStyle: string } } = {
type: 'link',
url: uri,
@@ -65,10 +67,7 @@ export function promoteInParent(parent: Parent, source: string = ''): void {
newChildren.push(child);
} else {
if (lastIndex < text.length) {
- const tail: Text = { type: 'text', value: text.slice(lastIndex) };
- const pos = deriveFragmentPosition(source, child as Text, lastIndex, text.length);
- if (pos) tail.position = pos;
- segments.push(tail);
+ segments.push(sliceTextWithProvenance(source, child as Text, lastIndex, text.length));
}
newChildren.push(...segments);
}
diff --git a/packages/core/src/markdown/backslash-escape-guard.ts b/packages/core/src/markdown/backslash-escape-guard.ts
index 69f415f4b..dbd4923bf 100644
--- a/packages/core/src/markdown/backslash-escape-guard.ts
+++ b/packages/core/src/markdown/backslash-escape-guard.ts
@@ -2,6 +2,7 @@ import type { Nodes, Root } from 'mdast';
import type { Plugin } from 'unified';
import { visit } from 'unist-util-visit';
import { protectPattern } from './entity-ref-guard.ts';
+import type { EntityReferenceSpan, EscapeProvenanceEntry } from './mdast-augmentation.ts';
const BACKSLASH_ESCAPE_PUA_MARK = '';
@@ -22,7 +23,50 @@ export function restoreBackslashEscapesPlugin(): ReturnType<Plugin<[], Root>> {
visit(tree, (node: Nodes) => {
const rec = node as unknown as Record<string, unknown>;
if (typeof rec.value === 'string' && rec.value.includes(BACKSLASH_ESCAPE_PUA_MARK)) {
- rec.value = (rec.value as string).split(BACKSLASH_ESCAPE_PUA_MARK).join('');
+ if (node.type === 'text') {
+ const oldValue = node.value;
+ const sourceOffsets = new Array<number>(oldValue.length + 1);
+ const restored: EscapeProvenanceEntry[] = [];
+ let value = '';
+ for (let index = 0; index < oldValue.length; index += 1) {
+ sourceOffsets[index] = value.length;
+ if (oldValue[index] === BACKSLASH_ESCAPE_PUA_MARK) {
+ restored.push({ offset: value.length, char: oldValue[index + 1] ?? '' });
+ } else {
+ value += oldValue[index];
+ }
+ }
+ sourceOffsets[oldValue.length] = value.length;
+ const existing = node.data?.escapedChars ?? [];
+ const rebasedExisting = existing.map((entry) => ({
+ ...entry,
+ offset: sourceOffsets[entry.offset] ?? entry.offset,
+ }));
+ const escapedChars = [...rebasedExisting, ...restored]
+ .map((entry) => ({
+ ...entry,
+ }))
+ .sort(
+ (left, right) => left.offset - right.offset || left.char.localeCompare(right.char),
+ )
+ .filter(
+ (entry, index, entries) =>
+ index === 0 ||
+ entry.offset !== entries[index - 1]?.offset ||
+ entry.char !== entries[index - 1]?.char,
+ );
+ const entityRefSpans = node.data?.entityRefSpans?.map((span): EntityReferenceSpan => {
+ const start = sourceOffsets[span.offset] ?? span.offset;
+ const end = sourceOffsets[span.offset + span.length] ?? span.offset + span.length;
+ return { ...span, offset: start, length: end - start };
+ });
+ node.value = value;
+ node.data ??= {};
+ node.data.escapedChars = escapedChars;
+ if (entityRefSpans) node.data.entityRefSpans = entityRefSpans;
+ } else {
+ rec.value = (rec.value as string).split(BACKSLASH_ESCAPE_PUA_MARK).join('');
+ }
}
for (const key of ['url', 'title', 'alt'] as const) {
const v = rec[key];
diff --git a/packages/core/src/markdown/comment-promoter.ts b/packages/core/src/markdown/comment-promoter.ts
index bfbd5a2e8..fc267cbea 100644
--- a/packages/core/src/markdown/comment-promoter.ts
+++ b/packages/core/src/markdown/comment-promoter.ts
@@ -6,6 +6,7 @@ import {
deriveFragmentPosition,
escapedValueOffsets,
isEscapeDerivedRun,
+ sliceTextWithProvenance,
} from './promoter-position.ts';
const PERCENT_COMMENT_RE = /(?<!%)%%([^\n]*?[^\n%])%%(?!%)/g;
@@ -23,19 +24,27 @@ export function commentPromoterPlugin() {
const value = node.value;
if (value.indexOf('%%') === -1 && value.indexOf('<!--') === -1) return;
- const matches = collectInlineCommentMatches(value, escapedValueOffsets(source, node));
+ const matches = collectInlineCommentMatches(value, escapedValueOffsets(node));
if (matches.length === 0) return;
const replacements: PhrasingContent[] = [];
let cursor = 0;
for (const match of matches) {
if (match.start > cursor) {
- const lead: Text = { type: 'text', value: value.slice(cursor, match.start) };
- const pos = deriveFragmentPosition(source, node, cursor, match.start);
- if (pos) lead.position = pos;
- replacements.push(lead);
+ replacements.push(sliceTextWithProvenance(source, node, cursor, match.start));
}
- const innerText: Text = { type: 'text', value: match.body };
+ const openingLength = match.sourceForm === 'html' ? 4 : 2;
+ const innerStart = match.start + openingLength;
+ const bodyStart =
+ match.sourceForm === 'html'
+ ? innerStart + value.slice(innerStart, match.end - 3).indexOf(match.body)
+ : innerStart;
+ const innerText = sliceTextWithProvenance(
+ source,
+ node,
+ bodyStart,
+ bodyStart + match.body.length,
+ );
const commentNode: CommentMdast = {
type: 'comment',
children: [innerText],
@@ -47,10 +56,7 @@ export function commentPromoterPlugin() {
cursor = match.end;
}
if (cursor < value.length) {
- const tail: Text = { type: 'text', value: value.slice(cursor) };
- const pos = deriveFragmentPosition(source, node, cursor, value.length);
- if (pos) tail.position = pos;
- replacements.push(tail);
+ replacements.push(sliceTextWithProvenance(source, node, cursor, value.length));
}
const arr = (parent as { children: PhrasingContent[] }).children;
@@ -140,7 +146,7 @@ function handleBlockCommentsAtRoot(tree: Root, source: string): void {
let onlyEscapedCache: ReadonlySet<number> | null | undefined;
const onlyEscapes = (): ReadonlySet<number> | null => {
if (onlyEscapedCache === undefined) {
- onlyEscapedCache = escapedValueOffsets(source, onlyText);
+ onlyEscapedCache = escapedValueOffsets(onlyText);
}
return onlyEscapedCache;
};
@@ -188,8 +194,8 @@ function handleBlockCommentsAtRoot(tree: Root, source: string): void {
}
}
- const strippedHtml = stripHtmlCommentDelimiters(child);
- if (strippedHtml !== null && !boundaryDelimitersEscaped(source, child, 4, 3)) {
+ const strippedHtml = stripHtmlCommentDelimiters(child, source);
+ if (strippedHtml !== null && !boundaryDelimitersEscaped(child, 4, 3)) {
const block: CommentBlockMdast = {
type: 'commentBlock',
children: [strippedHtml],
@@ -200,8 +206,8 @@ function handleBlockCommentsAtRoot(tree: Root, source: string): void {
continue;
}
- const strippedPercent = stripPercentDelimiters(child);
- if (strippedPercent !== null && !boundaryDelimitersEscaped(source, child, 2, 2)) {
+ const strippedPercent = stripPercentDelimiters(child, source);
+ if (strippedPercent !== null && !boundaryDelimitersEscaped(child, 2, 2)) {
const block: CommentBlockMdast = {
type: 'commentBlock',
children: [strippedPercent],
@@ -213,11 +219,11 @@ function handleBlockCommentsAtRoot(tree: Root, source: string): void {
}
}
- if (child.type === 'paragraph' && isFenceOnlyParagraph(child, source)) {
+ if (child.type === 'paragraph' && isFenceOnlyParagraph(child)) {
let j = i + 1;
while (j < children.length) {
const sibling = children[j];
- if (sibling.type === 'paragraph' && isFenceOnlyParagraph(sibling, source)) break;
+ if (sibling.type === 'paragraph' && isFenceOnlyParagraph(sibling)) break;
j += 1;
}
if (j < children.length && j > i + 1) {
@@ -237,24 +243,19 @@ function handleBlockCommentsAtRoot(tree: Root, source: string): void {
}
}
-function runEscaped(source: string, node: Text, valueOffset: number, length: number): boolean {
- return isEscapeDerivedRun(escapedValueOffsets(source, node), valueOffset, length);
+function runEscaped(node: Text, valueOffset: number, length: number): boolean {
+ return isEscapeDerivedRun(escapedValueOffsets(node), valueOffset, length);
}
-function boundaryDelimitersEscaped(
- source: string,
- p: Paragraph,
- openLength: number,
- closeLength: number,
-): boolean {
+function boundaryDelimitersEscaped(p: Paragraph, openLength: number, closeLength: number): boolean {
const first = p.children[0];
const last = p.children[p.children.length - 1];
if (first.type !== 'text' || last.type !== 'text') return false;
const openAt = first.value.length - first.value.trimStart().length;
const closeAt = last.value.trimEnd().length - closeLength;
- const firstEscaped = escapedValueOffsets(source, first);
+ const firstEscaped = escapedValueOffsets(first);
if (isEscapeDerivedRun(firstEscaped, openAt, openLength)) return true;
- const lastEscaped = last === first ? firstEscaped : escapedValueOffsets(source, last);
+ const lastEscaped = last === first ? firstEscaped : escapedValueOffsets(last);
return isEscapeDerivedRun(lastEscaped, closeAt, closeLength);
}
@@ -280,7 +281,7 @@ function matchHtmlCommentBlock(value: string): string | null {
return trimmedBody;
}
-function stripHtmlCommentDelimiters(p: Paragraph): Paragraph | null {
+function stripHtmlCommentDelimiters(p: Paragraph, source: string): Paragraph | null {
if (p.children.length < 2) return null;
const first = p.children[0];
const last = p.children[p.children.length - 1];
@@ -304,27 +305,27 @@ function stripHtmlCommentDelimiters(p: Paragraph): Paragraph | null {
}
}
- let strippedFirst = firstTrimmed.slice(4);
- if (strippedFirst.startsWith(' ')) strippedFirst = strippedFirst.slice(1);
- let strippedLast = lastTrimmed.slice(0, -3);
- if (strippedLast.endsWith(' ')) strippedLast = strippedLast.slice(0, -1);
+ let firstFrom = first.value.length - firstTrimmed.length + 4;
+ if (first.value[firstFrom] === ' ') firstFrom += 1;
+ let lastTo = lastTrimmed.length - 3;
+ if (last.value[lastTo - 1] === ' ') lastTo -= 1;
const newChildren: Paragraph['children'] = [];
- if (strippedFirst.length > 0) {
- newChildren.push({ ...first, value: strippedFirst } as Text);
+ if (firstFrom < first.value.length) {
+ newChildren.push(sliceTextWithProvenance(source, first, firstFrom, first.value.length));
}
for (let i = 1; i < p.children.length - 1; i++) {
newChildren.push(p.children[i]);
}
- if (strippedLast.length > 0) {
- newChildren.push({ ...last, value: strippedLast } as Text);
+ if (lastTo > 0) {
+ newChildren.push(sliceTextWithProvenance(source, last, 0, lastTo));
}
if (newChildren.length === 0) return null;
return { type: 'paragraph', children: newChildren };
}
-function stripPercentDelimiters(p: Paragraph): Paragraph | null {
+function stripPercentDelimiters(p: Paragraph, source: string): Paragraph | null {
if (p.children.length < 2) return null;
const first = p.children[0];
const last = p.children[p.children.length - 1];
@@ -335,10 +336,12 @@ function stripPercentDelimiters(p: Paragraph): Paragraph | null {
const lastTrimmed = last.value.trimEnd();
if (!lastTrimmed.endsWith('%%') || lastTrimmed.endsWith('%%%')) return null;
- let strippedFirst = firstTrimmed.slice(2);
- if (strippedFirst.startsWith(' ')) strippedFirst = strippedFirst.slice(1);
- let strippedLast = lastTrimmed.slice(0, -2);
- if (strippedLast.endsWith(' ')) strippedLast = strippedLast.slice(0, -1);
+ let firstFrom = first.value.length - firstTrimmed.length + 2;
+ if (first.value[firstFrom] === ' ') firstFrom += 1;
+ let lastTo = lastTrimmed.length - 2;
+ if (last.value[lastTo - 1] === ' ') lastTo -= 1;
+ const strippedFirst = first.value.slice(firstFrom);
+ const strippedLast = last.value.slice(0, lastTo);
if (strippedFirst.indexOf('%%') !== -1) return null;
if (strippedLast.indexOf('%%') !== -1) return null;
@@ -357,13 +360,13 @@ function stripPercentDelimiters(p: Paragraph): Paragraph | null {
const newChildren: Paragraph['children'] = [];
if (strippedFirst.length > 0) {
- newChildren.push({ ...first, value: strippedFirst } as Text);
+ newChildren.push(sliceTextWithProvenance(source, first, firstFrom, first.value.length));
}
for (let i = 1; i < p.children.length - 1; i++) {
newChildren.push(p.children[i]);
}
if (strippedLast.length > 0) {
- newChildren.push({ ...last, value: strippedLast } as Text);
+ newChildren.push(sliceTextWithProvenance(source, last, 0, lastTo));
}
if (newChildren.length === 0) return null;
@@ -383,11 +386,11 @@ function countOccurrences(haystack: string, needle: string): number {
return count;
}
-function isFenceOnlyParagraph(p: Paragraph, source: string): boolean {
+function isFenceOnlyParagraph(p: Paragraph): boolean {
const text = isSingleTextParagraph(p);
if (text === null) return false;
if (text.trim() !== '%%') return false;
- return !runEscaped(source, p.children[0] as Text, text.indexOf('%%'), 2);
+ return !runEscaped(p.children[0] as Text, text.indexOf('%%'), 2);
}
function promoteSpanningInlineComments(tree: Root, source: string): void {
@@ -471,8 +474,8 @@ function promoteSpanningForm(
if (spanCrossesLineBreak(children, spec, open, close)) return false;
- if (runEscaped(source, openChild, open.offset, spec.open.length)) return false;
- if (runEscaped(source, closeChild, close.offset, spec.close.length)) return false;
+ if (runEscaped(openChild, open.offset, spec.open.length)) return false;
+ if (runEscaped(closeChild, close.offset, spec.close.length)) return false;
for (let i = open.index + 1; i < close.index; i++) {
if (containsClaimedComment(children[i])) return false;
@@ -491,7 +494,9 @@ function promoteSpanningForm(
const replacements: PhrasingContent[] = [];
const lead = openChild.value.slice(0, open.offset);
- if (lead.length > 0) replacements.push(sliceTextNode(source, openChild, 0, open.offset));
+ if (lead.length > 0) {
+ replacements.push(sliceTextWithProvenance(source, openChild, 0, open.offset));
+ }
const commentNode: CommentMdast = {
type: 'comment',
@@ -508,7 +513,9 @@ function promoteSpanningForm(
const tailFrom = close.offset + spec.close.length;
const tail = closeChild.value.slice(tailFrom);
if (tail.length > 0) {
- replacements.push(sliceTextNode(source, closeChild, tailFrom, closeChild.value.length));
+ replacements.push(
+ sliceTextWithProvenance(source, closeChild, tailFrom, closeChild.value.length),
+ );
}
children.splice(open.index, close.index - open.index + 1, ...replacements);
@@ -537,11 +544,11 @@ function buildSpanningBody(
const body: PhrasingContent[] = [];
if (headFrom < headTo) {
- body.push(sliceTextNode(source, openChild, headFrom, headTo));
+ body.push(sliceTextWithProvenance(source, openChild, headFrom, headTo));
}
for (let i = open.index + 1; i < close.index; i++) body.push(children[i]);
if (footFrom < footTo) {
- body.push(sliceTextNode(source, closeChild, footFrom, footTo));
+ body.push(sliceTextWithProvenance(source, closeChild, footFrom, footTo));
}
if (body.length === 0) return null;
@@ -585,26 +592,3 @@ function containsClaimedComment(node: unknown): boolean {
const kids = (node as { children?: unknown }).children;
return Array.isArray(kids) && kids.some(containsClaimedComment);
}
-
-interface EntityRefSpan {
- offset: number;
- length: number;
- raw: string;
-}
-
-function sliceTextNode(source: string, node: Text, from: number, to: number): Text {
- const sliced: Text = { type: 'text', value: node.value.slice(from, to) };
-
- const pos = deriveFragmentPosition(source, node, from, to);
- if (pos) sliced.position = pos;
-
- const spans = (node.data as { entityRefSpans?: EntityRefSpan[] } | undefined)?.entityRefSpans;
- if (spans?.length) {
- const inside = spans
- .filter((span) => span.offset >= from && span.offset + span.length <= to)
- .map((span) => ({ ...span, offset: span.offset - from }));
- if (inside.length > 0) sliced.data = { entityRefSpans: inside } as Text['data'];
- }
-
- return sliced;
-}
diff --git a/packages/core/src/markdown/escape-provenance.ts b/packages/core/src/markdown/escape-provenance.ts
new file mode 100644
index 000000000..3ccb52b14
--- /dev/null
+++ b/packages/core/src/markdown/escape-provenance.ts
@@ -0,0 +1,155 @@
+import { decodeNamedCharacterReference } from 'decode-named-character-reference';
+import type { Root, Text } from 'mdast';
+import type { CompileContext, Extension } from 'mdast-util-from-markdown';
+import { decodeNumericCharacterReference } from 'micromark-util-decode-numeric-character-reference';
+import type { Parser, Plugin, Processor } from 'unified';
+import { visit } from 'unist-util-visit';
+import type { EscapeProvenanceEntry } from './mdast-augmentation.ts';
+
+const MARKER_RANGE_START = 0xe200;
+const MARKER_RANGE_END = 0xf8ff;
+const CHARACTER_REFERENCE_RE = /&(?:[A-Za-z][A-Za-z0-9]*|#[0-9]+|#[xX][0-9A-Fa-f]+);/g;
+
+function decodedReferenceProjection(source: string): string {
+ CHARACTER_REFERENCE_RE.lastIndex = 0;
+ return source.replace(CHARACTER_REFERENCE_RE, (reference) => {
+ const body = reference.slice(1, -1);
+ if (body.startsWith('#')) {
+ const hexadecimal = body[1] === 'x' || body[1] === 'X';
+ return decodeNumericCharacterReference(
+ body.slice(hexadecimal ? 2 : 1),
+ hexadecimal ? 16 : 10,
+ );
+ }
+ const decoded = decodeNamedCharacterReference(body);
+ return decoded === false ? reference : decoded;
+ });
+}
+
+function inspectMarkerOccupancy(values: readonly string[]): {
+ occupied: ReadonlySet<number>;
+ longestStartRun: number;
+} {
+ const occupied = new Set<number>();
+ let longestStartRun = 0;
+ for (const value of values) {
+ let currentStartRun = 0;
+ for (const char of value) {
+ const codePoint = char.codePointAt(0);
+ if (codePoint === undefined) continue;
+ if (codePoint >= MARKER_RANGE_START && codePoint <= MARKER_RANGE_END) {
+ occupied.add(codePoint);
+ }
+ if (codePoint === MARKER_RANGE_START) {
+ currentStartRun += 1;
+ longestStartRun = Math.max(longestStartRun, currentStartRun);
+ } else {
+ currentStartRun = 0;
+ }
+ }
+ }
+ return { occupied, longestStartRun };
+}
+
+export function selectEscapeProvenanceMarker(source: string): string {
+ const projection = decodedReferenceProjection(source);
+ const { occupied, longestStartRun } = inspectMarkerOccupancy([source, projection]);
+ for (let codePoint = MARKER_RANGE_START; codePoint <= MARKER_RANGE_END; codePoint += 1) {
+ if (!occupied.has(codePoint)) return String.fromCharCode(codePoint);
+ }
+ const markerUnit = String.fromCharCode(MARKER_RANGE_START);
+ const terminator = String.fromCharCode(MARKER_RANGE_START + 1);
+ return markerUnit.repeat(longestStartRun + 1) + terminator;
+}
+
+function createEscapeExtension(marker: string): Extension {
+ return {
+ exit: {
+ characterEscapeValue(this: CompileContext, token): void {
+ const tail = this.stack.pop() as Text;
+ tail.value += marker + this.sliceSerialize(token);
+ if (tail.position) {
+ tail.position.end = {
+ line: token.end.line,
+ column: token.end.column,
+ offset: token.end.offset,
+ };
+ }
+ },
+ },
+ transforms: [
+ (tree) => {
+ visit(tree, (node) => {
+ if (node.type === 'text') {
+ const entries: EscapeProvenanceEntry[] = [];
+ let value = '';
+ let cursor = 0;
+ for (;;) {
+ const markerAt = node.value.indexOf(marker, cursor);
+ if (markerAt === -1) break;
+ value += node.value.slice(cursor, markerAt);
+ const escapedChar = node.value.slice(
+ markerAt + marker.length,
+ markerAt + marker.length + 1,
+ );
+ entries.push({ offset: value.length, char: escapedChar });
+ cursor = markerAt + marker.length;
+ }
+ value += node.value.slice(cursor);
+ node.value = value;
+ node.data ??= {};
+ node.data.escapedChars = entries;
+ return;
+ }
+ const record = node as unknown as Record<string, unknown>;
+ for (const [key, field] of Object.entries(record)) {
+ if (typeof field === 'string') record[key] = field.replaceAll(marker, '');
+ }
+ });
+ return tree;
+ },
+ ],
+ };
+}
+
+function withSynchronousReentrantEscapeProvenanceParser(
+ baseParser: Parser<Root>,
+ processor: Processor,
+): Parser<Root> {
+ let activeExtension: Extension | undefined;
+ return (document, file) => {
+ const data = processor.data() as { fromMarkdownExtensions?: Extension[] };
+ data.fromMarkdownExtensions ??= [];
+ const extensions = data.fromMarkdownExtensions;
+ const extension = createEscapeExtension(selectEscapeProvenanceMarker(document));
+ const outerExtension = activeExtension;
+ const outerExtensionIndex = outerExtension ? extensions.lastIndexOf(outerExtension) : -1;
+ if (outerExtensionIndex >= 0) {
+ extensions.splice(outerExtensionIndex, 1, extension);
+ } else {
+ extensions.push(extension);
+ }
+ activeExtension = extension;
+ try {
+ return baseParser(document, file);
+ } finally {
+ const extensionIndex = extensions.lastIndexOf(extension);
+ if (extensionIndex >= 0) {
+ if (outerExtension && outerExtensionIndex >= 0) {
+ extensions.splice(extensionIndex, 1, outerExtension);
+ } else {
+ extensions.splice(extensionIndex, 1);
+ }
+ }
+ activeExtension = outerExtension;
+ }
+ };
+}
+
+export const escapeProvenancePlugin: Plugin<[], Root> = function () {
+ const processor = this as Processor;
+ processor.parser = withSynchronousReentrantEscapeProvenanceParser(
+ processor.parser as Parser<Root>,
+ processor,
+ );
+};
diff --git a/packages/core/src/markdown/heading-content-identity.ts b/packages/core/src/markdown/heading-content-identity.ts
deleted file mode 100644
index dd4433204..000000000
--- a/packages/core/src/markdown/heading-content-identity.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-import { fromMarkdown } from 'mdast-util-from-markdown';
-import { toString as mdastToString } from 'mdast-util-to-string';
-
-export function headingContentIdentity(heading: string): string {
- const singleLine = heading.replace(/\s+/g, ' ').trim();
- const tree = fromMarkdown(`## ${singleLine}`);
- const node = tree.children[0];
- if (node === undefined || node.type !== 'heading') return '';
- return mdastToString(node, { includeHtml: false }).replace(/\s+/g, ' ').trim().normalize('NFC');
-}
diff --git a/packages/core/src/markdown/highlight-promoter.ts b/packages/core/src/markdown/highlight-promoter.ts
index a4ee36118..501ee8e3a 100644
--- a/packages/core/src/markdown/highlight-promoter.ts
+++ b/packages/core/src/markdown/highlight-promoter.ts
@@ -7,6 +7,7 @@ import {
deriveFragmentPosition,
escapedValueOffsets,
isEscapeDerivedRun,
+ sliceTextWithProvenance,
} from './promoter-position.ts';
const HIGHLIGHT_RE = /(?<!=)==(?=\S)([^\n]*?[^\s=])==(?!=)/g;
@@ -45,35 +46,13 @@ function getLastChar(node: PhrasingContent): string | null {
return null;
}
-interface EntityRefSpan {
- offset: number;
- length: number;
- raw: string;
-}
-
-function sliceTextNode(source: Text, from: number, to: number, rawSource?: string): Text {
- const out: Text = { type: 'text', value: source.value.slice(from, to) };
- if (rawSource !== undefined) {
- const pos = deriveFragmentPosition(rawSource, source, from, to);
- if (pos) out.position = pos;
- }
- const spans = (source.data as { entityRefSpans?: EntityRefSpan[] } | undefined)?.entityRefSpans;
- if (spans?.length) {
- const inside = spans
- .filter((s) => s.offset >= from && s.offset + s.length <= to)
- .map((s) => ({ ...s, offset: s.offset - from }));
- if (inside.length > 0) out.data = { entityRefSpans: inside } as Text['data'];
- }
- return out;
-}
-
function promoteCrossChildren(parent: Parent, source: string): void {
const children = parent.children as PhrasingContent[];
const escapeSets = new Map<Text, ReadonlySet<number> | null>();
const escapesOf = (node: Text): ReadonlySet<number> | null => {
const cached = escapeSets.get(node);
if (cached !== undefined) return cached;
- const set = escapedValueOffsets(source, node);
+ const set = escapedValueOffsets(node);
escapeSets.set(node, set);
return set;
};
@@ -184,10 +163,12 @@ function promoteCrossChildren(parent: Parent, source: string): void {
const bodyChildren: PhrasingContent[] = [];
if (openTrailing.length > 0) {
- bodyChildren.push(sliceTextNode(openChild, openPos + 2, openValue.length, source));
+ bodyChildren.push(sliceTextWithProvenance(source, openChild, openPos + 2, openValue.length));
}
for (let k = outerI + 1; k < closeChildIdx; k++) bodyChildren.push(children[k]);
- if (closeLeading.length > 0) bodyChildren.push(sliceTextNode(closeChild, 0, closePos, source));
+ if (closeLeading.length > 0) {
+ bodyChildren.push(sliceTextWithProvenance(source, closeChild, 0, closePos));
+ }
if (!bodyCrossesInline) {
outerI++;
@@ -200,10 +181,14 @@ function promoteCrossChildren(parent: Parent, source: string): void {
};
const replacement: PhrasingContent[] = [];
- if (leadValue.length > 0) replacement.push(sliceTextNode(openChild, 0, openPos, source));
+ if (leadValue.length > 0) {
+ replacement.push(sliceTextWithProvenance(source, openChild, 0, openPos));
+ }
replacement.push(markNode as unknown as PhrasingContent);
if (tailValue.length > 0) {
- replacement.push(sliceTextNode(closeChild, closePos + 2, closeChild.value.length, source));
+ replacement.push(
+ sliceTextWithProvenance(source, closeChild, closePos + 2, closeChild.value.length),
+ );
}
children.splice(outerI, closeChildIdx - outerI + 1, ...replacement);
@@ -229,7 +214,7 @@ export function highlightPromoterPlugin() {
const value = node.value;
if (value.indexOf('==') === -1) return;
- const escaped = escapedValueOffsets(source, node);
+ const escaped = escapedValueOffsets(node);
HIGHLIGHT_RE.lastIndex = 0;
const matches: RegExpExecArray[] = [];
let m: RegExpExecArray | null;
@@ -251,14 +236,9 @@ export function highlightPromoterPlugin() {
const start = match.index;
const end = start + match[0].length;
if (start > cursor) {
- const lead = sliceTextNode(node, cursor, start);
- const pos = deriveFragmentPosition(source, node, cursor, start);
- if (pos) lead.position = pos;
- replacements.push(lead);
+ replacements.push(sliceTextWithProvenance(source, node, cursor, start));
}
- const innerText = sliceTextNode(node, start + 2, end - 2);
- const innerPos = deriveFragmentPosition(source, node, start + 2, end - 2);
- if (innerPos) innerText.position = innerPos;
+ const innerText = sliceTextWithProvenance(source, node, start + 2, end - 2);
const markNode: MarkMdast = {
type: 'mark',
children: [innerText],
@@ -269,10 +249,7 @@ export function highlightPromoterPlugin() {
cursor = end;
}
if (cursor < value.length) {
- const tail = sliceTextNode(node, cursor, value.length);
- const pos = deriveFragmentPosition(source, node, cursor, value.length);
- if (pos) tail.position = pos;
- replacements.push(tail);
+ replacements.push(sliceTextWithProvenance(source, node, cursor, value.length));
}
const arr = (parent as { children: PhrasingContent[] }).children;
diff --git a/packages/core/src/markdown/mdast-augmentation.ts b/packages/core/src/markdown/mdast-augmentation.ts
index 116fced97..7548916b7 100644
--- a/packages/core/src/markdown/mdast-augmentation.ts
+++ b/packages/core/src/markdown/mdast-augmentation.ts
@@ -1,5 +1,48 @@
import type { Position } from 'unist';
+export interface EscapeProvenanceEntry {
+ offset: number;
+ char: string;
+}
+
+export interface EntityReferenceSpan {
+ offset: number;
+ length: number;
+ raw: string;
+}
+
+export interface EscapeProvenanceTextData {
+ escapedChars: EscapeProvenanceEntry[];
+}
+
+export type EscapeProvenanceValidation =
+ | { kind: 'absent' }
+ | { kind: 'malformed' }
+ | { kind: 'valid'; data: EscapeProvenanceTextData };
+
+export function validateEscapeProvenance(data: unknown): EscapeProvenanceValidation {
+ if (typeof data !== 'object' || data === null || !Object.hasOwn(data, 'escapedChars')) {
+ return { kind: 'absent' };
+ }
+ const escapedChars = (data as { escapedChars?: unknown }).escapedChars;
+ if (!Array.isArray(escapedChars)) return { kind: 'malformed' };
+ const valid = escapedChars.every((entry) => {
+ if (typeof entry !== 'object' || entry === null) return false;
+ const offset = (entry as { offset?: unknown }).offset;
+ return (
+ typeof offset === 'number' &&
+ Number.isInteger(offset) &&
+ offset >= 0 &&
+ typeof (entry as { char?: unknown }).char === 'string'
+ );
+ });
+ return valid ? { kind: 'valid', data: { escapedChars } } : { kind: 'malformed' };
+}
+
+export function hasEscapeProvenance<T>(data: T): data is T & EscapeProvenanceTextData {
+ return validateEscapeProvenance(data).kind === 'valid';
+}
+
export const PROMOTED_MDAST_TYPES = [
'wikiLink',
'wikiLinkEmbed',
@@ -121,9 +164,9 @@ declare module 'mdast' {
sourceDocBoundary?: SourceDocBoundary;
}
interface TextData {
- escapedChars?: Array<{ offset: number; char: string }>;
+ escapedChars?: EscapeProvenanceEntry[];
sourceRaw?: string;
- entityRefSpans?: Array<{ offset: number; length: number; raw: string }>;
+ entityRefSpans?: EntityReferenceSpan[];
}
interface EmphasisData {
sourceDelimiter?: '*' | '_';
diff --git a/packages/core/src/markdown/parser-reservations.ts b/packages/core/src/markdown/parser-reservations.ts
new file mode 100644
index 000000000..e8960a126
--- /dev/null
+++ b/packages/core/src/markdown/parser-reservations.ts
@@ -0,0 +1,62 @@
+import { protectFromMdx, R23_SENTINEL_ESCAPE_SUBSTITUTIONS } from './autolink-void-html-guard.ts';
+import { BACKSLASH_GUARD_SUBSTITUTIONS, encodeBackslashEscapes } from './backslash-escape-guard.ts';
+import { ENTITY_REF_GUARD_SUBSTITUTIONS, encodeEntityRefs } from './entity-ref-guard.ts';
+import { TYPED_WS_REF_PUA } from './to-markdown-handlers.ts';
+
+type ParserMutationSource =
+ | {
+ mode: 'protecting';
+ protect: (source: string) => string;
+ reservations: ReadonlyArray<{ to: string }>;
+ }
+ | {
+ mode: 'serializer-only';
+ protect?: never;
+ reservations: ReadonlyArray<{ to: string }>;
+ };
+
+export const PARSER_MUTATION_SOURCES: readonly ParserMutationSource[] = [
+ {
+ mode: 'protecting',
+ protect: encodeBackslashEscapes,
+ reservations: BACKSLASH_GUARD_SUBSTITUTIONS,
+ },
+ {
+ mode: 'protecting',
+ protect: protectFromMdx,
+ reservations: R23_SENTINEL_ESCAPE_SUBSTITUTIONS,
+ },
+ {
+ mode: 'protecting',
+ protect: encodeEntityRefs,
+ reservations: ENTITY_REF_GUARD_SUBSTITUTIONS,
+ },
+ {
+ mode: 'serializer-only',
+ reservations: [{ to: TYPED_WS_REF_PUA }],
+ },
+];
+
+const MUTATING_PARSER_RESERVATIONS: ReadonlySet<string> = new Set(
+ PARSER_MUTATION_SOURCES.flatMap(({ reservations }) => reservations.map(({ to }) => to)),
+);
+
+function protectWithMutationSource(
+ protectedSource: string,
+ mutationSource: ParserMutationSource,
+): string {
+ switch (mutationSource.mode) {
+ case 'protecting':
+ return mutationSource.protect(protectedSource);
+ case 'serializer-only':
+ return protectedSource;
+ }
+}
+
+export function protectParserSource(source: string): string {
+ return PARSER_MUTATION_SOURCES.reduce(protectWithMutationSource, source);
+}
+
+export function isMutatingParserReservation(value: string): boolean {
+ return MUTATING_PARSER_RESERVATIONS.has(value);
+}
diff --git a/packages/core/src/markdown/pipeline.ts b/packages/core/src/markdown/pipeline.ts
index 62799d32e..94d37efb6 100644
--- a/packages/core/src/markdown/pipeline.ts
+++ b/packages/core/src/markdown/pipeline.ts
@@ -5,7 +5,7 @@ import {
remarkProseMirror,
} from '@handlewithcare/remark-prosemirror';
import type { Node as PmNode, Schema } from '@tiptap/pm/model';
-import type { Root as MdastRoot } from 'mdast';
+import type { Nodes as MdastNode, Root as MdastRoot } from 'mdast';
import remarkFrontmatter from 'remark-frontmatter';
import remarkGfm from 'remark-gfm';
import remarkGithubAlerts from 'remark-github-alerts';
@@ -16,8 +16,8 @@ import { type Processor, unified } from 'unified';
import { VFile } from 'vfile';
import './mdast-augmentation.ts';
-import { protectFromMdx, restoreFromMdx } from './autolink-void-html-guard.ts';
-import { encodeBackslashEscapes, restoreBackslashEscapesPlugin } from './backslash-escape-guard.ts';
+import { restoreFromMdx } from './autolink-void-html-guard.ts';
+import { restoreBackslashEscapesPlugin } from './backslash-escape-guard.ts';
import { calloutTransformerPlugin, REMARK_GITHUB_ALERTS_OPTIONS } from './callout-transformer.ts';
import { commentPromoterPlugin } from './comment-promoter.ts';
import { dedentBlockJsxClose } from './dedent-block-jsx-close.ts';
@@ -25,17 +25,24 @@ import { detailsAccordionPromoterPlugin } from './details-accordion-promoter.ts'
import { divAlignPromoterPlugin } from './div-align-promoter.ts';
import { materializeDocEdgeBlankRuns } from './doc-edge-blank-runs.ts';
import { emptyTaskItemUnmintPlugin, mintEmptyTaskItemContent } from './empty-task-item.ts';
-import { encodeEntityRefs, restoreEntityRefsPlugin } from './entity-ref-guard.ts';
+import { restoreEntityRefsPlugin } from './entity-ref-guard.ts';
+import { escapeProvenancePlugin } from './escape-provenance.ts';
import { highlightPromoterPlugin } from './highlight-promoter.ts';
import { imagePromoterPlugin } from './image-promoter.ts';
import { indentedCodePromoterPlugin } from './indented-code-promoter.ts';
import { insertInteriorBlankRunParagraphs } from './interior-blank-runs.ts';
import { linkReferenceDestinationPlugin } from './link-reference-destination.ts';
import { mathPromoterPlugin } from './math-promoter.ts';
-import type { SourceDocBoundary } from './mdast-augmentation.ts';
+import {
+ type EntityReferenceSpan,
+ type EscapeProvenanceEntry,
+ hasEscapeProvenance,
+ type SourceDocBoundary,
+} from './mdast-augmentation.ts';
import { mergedPostParseWalkerPlugin } from './merged-walker.ts';
import { mermaidPromoterPlugin } from './mermaid-promoter.ts';
import { nonRenderingContextDemotePlugin } from './non-rendering-context-demote.ts';
+import { protectParserSource } from './parser-reservations.ts';
import { positionAwareBlankLineJoin } from './position-aware-join.ts';
import { remarkMdxAgnostic } from './remark-mdx-agnostic.ts';
import { singleDollarMathPromoterPlugin } from './single-dollar-math-promoter.ts';
@@ -74,6 +81,7 @@ function ensureNonEmptyDoc(tree: MdastRoot): MdastRoot {
export const ACTIVE_MDAST_PLUGINS = [
{ name: 'remark-parse', plugin: remarkParse },
+ { name: 'escape-provenance', plugin: escapeProvenancePlugin },
{ name: 'remark-mdx-agnostic', plugin: remarkMdxAgnostic },
{ name: 'remark-gfm', plugin: remarkGfm },
{ name: 'remark-math', plugin: remarkMath, options: { singleDollarTextMath: false } },
@@ -211,9 +219,7 @@ function readDocBoundary(value: unknown): SourceDocBoundary | undefined {
export function parseMd(rawSource: string, processor: Processor): PmNode {
const { source: rawAfterBom, hadBom } = splitDocumentHeadBom(rawSource);
const source = dedentBlockJsxClose(rawAfterBom);
- const protectedFr14 = encodeBackslashEscapes(source);
- const protectedR23 = protectFromMdx(protectedFr14);
- const protected_ = encodeEntityRefs(protectedR23);
+ const protected_ = protectParserSource(source);
const file = new VFile(protected_);
const tree = processor.parse(file);
@@ -242,7 +248,7 @@ function parseToMdast(
): MdastRoot {
const { source: rawAfterBom, hadBom } = splitDocumentHeadBom(rawSource);
const source = dedentBlockJsxClose(rawAfterBom);
- const protected_ = encodeEntityRefs(protectFromMdx(encodeBackslashEscapes(source)));
+ const protected_ = protectParserSource(source);
const file = new VFile(protected_);
const tree = processor.parse(file);
file.value = source;
@@ -260,6 +266,8 @@ export function serializeMd(doc: PmNode, processor: Processor, opts: SerializeMd
markHandlers: opts.pmMarkHandlers,
});
+ coalesceEscapedTextRuns(mdast);
+
mintEmptyTaskItemContent(mdast);
stripTrailingEdge(mdast);
@@ -276,3 +284,46 @@ export function serializeMd(doc: PmNode, processor: Processor, opts: SerializeMd
if (boundary?.bom) out = `\uFEFF${out}`;
return out;
}
+
+function coalesceEscapedTextRuns(node: MdastNode): void {
+ if (!('children' in node) || !Array.isArray(node.children)) return;
+ for (const child of node.children) coalesceEscapedTextRuns(child as MdastNode);
+ for (let index = 0; index + 1 < node.children.length; ) {
+ const left = node.children[index];
+ const right = node.children[index + 1];
+ const leftData = left?.data;
+ const rightData = right?.data;
+ const leftEscapedChars = hasEscapeProvenance(leftData) ? leftData.escapedChars : null;
+ const rightEscapedChars = hasEscapeProvenance(rightData) ? rightData.escapedChars : null;
+ if (
+ left?.type !== 'text' ||
+ right?.type !== 'text' ||
+ typeof left.data?.sourceRaw === 'string' ||
+ typeof right.data?.sourceRaw === 'string' ||
+ (leftEscapedChars === null && rightEscapedChars === null)
+ ) {
+ index += 1;
+ continue;
+ }
+ const leftLength = left.value.length;
+ left.value += right.value;
+ left.data ??= {};
+ left.data.escapedChars = [
+ ...(leftEscapedChars ?? []),
+ ...(rightEscapedChars ?? []).map((entry: EscapeProvenanceEntry) => ({
+ ...entry,
+ offset: entry.offset + leftLength,
+ })),
+ ];
+ if (right.data?.entityRefSpans?.length) {
+ left.data.entityRefSpans = [
+ ...(left.data.entityRefSpans ?? []),
+ ...right.data.entityRefSpans.map((span: EntityReferenceSpan) => ({
+ ...span,
+ offset: span.offset + leftLength,
+ })),
+ ];
+ }
+ node.children.splice(index + 1, 1);
+ }
+}
diff --git a/packages/core/src/markdown/position-slice.ts b/packages/core/src/markdown/position-slice.ts
index 93a301e7e..5e4455436 100644
--- a/packages/core/src/markdown/position-slice.ts
+++ b/packages/core/src/markdown/position-slice.ts
@@ -1,14 +1,10 @@
import type { Nodes, Root } from 'mdast';
import { visit } from 'unist-util-visit';
import type { VFile } from 'vfile';
+import { type EscapeProvenanceEntry, hasEscapeProvenance } from './mdast-augmentation.ts';
const ESCAPABLE_CHARS = new Set('!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'.split(''));
-interface EscapedChar {
- offset: number;
- char: string;
-}
-
export function splitGfmCellSegments(rowSrc: string): string[] {
const segments: string[] = [];
let cur = '';
@@ -66,8 +62,8 @@ export function applyPositionSliceToNode(
case 'text': {
const raw = source.slice(startOff, endOff);
const value: string = node.value ?? '';
- if (raw.length > value.length && raw.includes('\\')) {
- const escaped: EscapedChar[] = [];
+ if (!hasEscapeProvenance(node.data) && raw.length > value.length && raw.includes('\\')) {
+ const escaped: EscapeProvenanceEntry[] = [];
let rawIdx = 0;
let valIdx = 0;
while (rawIdx < raw.length && valIdx < value.length) {
diff --git a/packages/core/src/markdown/promoter-position.ts b/packages/core/src/markdown/promoter-position.ts
index 0c64df048..cb070fe7c 100644
--- a/packages/core/src/markdown/promoter-position.ts
+++ b/packages/core/src/markdown/promoter-position.ts
@@ -1,5 +1,10 @@
import type { Text } from 'mdast';
import type { Point, Position } from 'unist';
+import {
+ type EntityReferenceSpan,
+ type EscapeProvenanceEntry,
+ hasEscapeProvenance,
+} from './mdast-augmentation.ts';
function consumesEscape(source: string, i: number, valueChar: string): boolean {
return source[i] === '\\' && i + 1 < source.length && source[i + 1] === valueChar;
@@ -14,52 +19,90 @@ function valueOffsetToSourceOffset(
let srcCursor = parentSourceStart;
let valCursor = 0;
while (valCursor < targetValueOffset && srcCursor < source.length) {
+ const continuationEnd =
+ valCursor > 0 && valueText[valCursor - 1] === '\n'
+ ? containerContinuationEnd(source, srcCursor, valueText[valCursor])
+ : null;
+ if (continuationEnd !== null) srcCursor = continuationEnd;
srcCursor += consumesEscape(source, srcCursor, valueText[valCursor]) ? 2 : 1;
valCursor += 1;
}
return srcCursor;
}
-const NO_ESCAPES: ReadonlySet<number> = new Set();
+function sourceMatchesValueAt(source: string, offset: number, valueChar: string): boolean {
+ return source[offset] === valueChar || consumesEscape(source, offset, valueChar);
+}
-export function escapedValueOffsets(source: string, node: Text): ReadonlySet<number> | null {
- if (!source || !node.position || typeof node.position.start?.offset !== 'number') {
- return null;
- }
- const start = node.position.start.offset;
- const value = node.value;
- const spanEnd = Math.min(
- node.position.end?.offset ?? source.length,
- start + 2 * value.length,
- source.length,
- );
- let hasBackslash = false;
- for (let i = start; i < spanEnd; i++) {
- if (source.charCodeAt(i) === 92) {
- hasBackslash = true;
- break;
+function containerContinuationEnd(
+ source: string,
+ sourceOffset: number,
+ valueChar: string,
+): number | null {
+ let cursor = sourceOffset;
+ let consumed = false;
+ for (;;) {
+ const indentStart = cursor;
+ while (cursor < source.length && (source[cursor] === ' ' || source[cursor] === '\t')) {
+ cursor += 1;
}
+ if (source[cursor] === '>') {
+ cursor += 1;
+ if (source[cursor] === ' ' || source[cursor] === '\t') cursor += 1;
+ consumed = true;
+ continue;
+ }
+ if (cursor > indentStart) consumed = true;
+ break;
}
- if (!hasBackslash) {
- return NO_ESCAPES;
- }
+ return consumed && sourceMatchesValueAt(source, cursor, valueChar) ? cursor : null;
+}
- let escaped: Set<number> | null = null;
- let srcCursor = start;
- let valCursor = 0;
- while (valCursor < value.length && srcCursor < source.length) {
- if (consumesEscape(source, srcCursor, value[valCursor])) {
- if (escaped === null) escaped = new Set();
- escaped.add(valCursor);
- srcCursor += 2;
- } else if (source[srcCursor] === value[valCursor]) {
- srcCursor += 1;
- } else {
- break;
- }
- valCursor += 1;
+export function escapedValueOffsets(node: Text): ReadonlySet<number> | null {
+ if (!hasEscapeProvenance(node.data)) return null;
+ return new Set(node.data.escapedChars.map((entry) => entry.offset));
+}
+
+function sliceEscapedChars(
+ entries: readonly EscapeProvenanceEntry[],
+ from: number,
+ to: number,
+): EscapeProvenanceEntry[] {
+ return entries
+ .filter((entry) => entry.offset >= from && entry.offset < to)
+ .map((entry) => ({ ...entry, offset: entry.offset - from }));
+}
+
+function sliceEntityRefSpans(
+ spans: readonly EntityReferenceSpan[],
+ from: number,
+ to: number,
+): EntityReferenceSpan[] {
+ return spans
+ .filter((span) => span.offset >= from && span.offset + span.length <= to)
+ .map((span) => ({ ...span, offset: span.offset - from }));
+}
+
+export function sliceTextWithProvenance(
+ source: string,
+ node: Text,
+ from: number,
+ to: number,
+): Text {
+ const sliced: Text = { type: 'text', value: node.value.slice(from, to) };
+ const position = deriveFragmentPosition(source, node, from, to);
+ if (position) sliced.position = position;
+
+ let data: Text['data'];
+ if (hasEscapeProvenance(node.data)) {
+ data = { escapedChars: sliceEscapedChars(node.data.escapedChars, from, to) };
+ }
+ if (node.data?.entityRefSpans?.length) {
+ const entityRefSpans = sliceEntityRefSpans(node.data.entityRefSpans, from, to);
+ if (entityRefSpans.length > 0) data = { ...data, entityRefSpans };
}
- return escaped ?? NO_ESCAPES;
+ if (data) sliced.data = data;
+ return sliced;
}
export function isEscapeDerivedRun(
diff --git a/packages/core/src/markdown/single-dollar-math-promoter.ts b/packages/core/src/markdown/single-dollar-math-promoter.ts
index b012a5aff..5ab49fb40 100644
--- a/packages/core/src/markdown/single-dollar-math-promoter.ts
+++ b/packages/core/src/markdown/single-dollar-math-promoter.ts
@@ -6,6 +6,7 @@ import {
deriveFragmentPosition,
escapedValueOffsets,
isEscapeDerivedRun,
+ sliceTextWithProvenance,
} from './promoter-position.ts';
const SINGLE_DOLLAR_MATH_RE = /(?<!\\)\$(?=\S)([^$\n]*?[^\s$])\$(?!\d)/g;
@@ -19,7 +20,7 @@ export function singleDollarMathPromoterPlugin() {
const value = node.value;
if (value.indexOf('$') === -1) return;
- const escaped = escapedValueOffsets(source, node);
+ const escaped = escapedValueOffsets(node);
SINGLE_DOLLAR_MATH_RE.lastIndex = 0;
const matches: RegExpExecArray[] = [];
let m: RegExpExecArray | null;
@@ -41,10 +42,7 @@ export function singleDollarMathPromoterPlugin() {
const start = match.index;
const end = start + match[0].length;
if (start > cursor) {
- const lead: Text = { type: 'text', value: value.slice(cursor, start) };
- const pos = deriveFragmentPosition(source, node, cursor, start);
- if (pos) lead.position = pos;
- replacements.push(lead);
+ replacements.push(sliceTextWithProvenance(source, node, cursor, start));
}
const mathNode: InlineMath = { type: 'inlineMath', value: match[1] };
const fullPos = deriveFragmentPosition(source, node, start, end);
@@ -53,10 +51,7 @@ export function singleDollarMathPromoterPlugin() {
cursor = end;
}
if (cursor < value.length) {
- const tail: Text = { type: 'text', value: value.slice(cursor) };
- const pos = deriveFragmentPosition(source, node, cursor, value.length);
- if (pos) tail.position = pos;
- replacements.push(tail);
+ replacements.push(sliceTextWithProvenance(source, node, cursor, value.length));
}
const arr = (parent as { children: PhrasingContent[] }).children;
diff --git a/packages/core/src/markdown/tag-promotion.ts b/packages/core/src/markdown/tag-promotion.ts
index 9e05a9ac4..59a327a18 100644
--- a/packages/core/src/markdown/tag-promotion.ts
+++ b/packages/core/src/markdown/tag-promotion.ts
@@ -1,6 +1,11 @@
import type { Parent, Text } from 'mdast';
import type { TagMdast } from './mdast-augmentation.ts';
-import { deriveFragmentPosition } from './promoter-position.ts';
+import {
+ deriveFragmentPosition,
+ escapedValueOffsets,
+ isEscapeDerivedRun,
+ sliceTextWithProvenance,
+} from './promoter-position.ts';
export const TAG_IN_TEXT_PATTERN_SOURCE = '(^|\\s)#([a-zA-Z][\\w/-]*)';
export function createTagInTextRegex(): RegExp {
@@ -10,18 +15,6 @@ const TAG_IN_TEXT_RE = createTagInTextRegex();
export const INLINE_TAG_VALUE_RE = /^[a-zA-Z][\w/-]*$/;
-function isEscapedHash(source: string, sourceOffsetOfHash: number): boolean {
- if (sourceOffsetOfHash <= 0) return false;
- if (source[sourceOffsetOfHash - 1] !== '\\') return false;
- let cursor = sourceOffsetOfHash - 1;
- let count = 0;
- while (cursor >= 0 && source[cursor] === '\\') {
- count++;
- cursor--;
- }
- return count % 2 === 1;
-}
-
export function promoteTagsInParent(parent: Parent, source: string = ''): void {
const newChildren: Parent['children'] = [];
let changed = false;
@@ -33,10 +26,7 @@ export function promoteTagsInParent(parent: Parent, source: string = ''): void {
}
const text = (child as Text).value;
- const childTextStart =
- typeof (child as Text).position?.start?.offset === 'number'
- ? ((child as Text).position?.start?.offset ?? 0)
- : 0;
+ const escaped = escapedValueOffsets(child as Text);
TAG_IN_TEXT_RE.lastIndex = 0;
const segments: Parent['children'] = [];
@@ -49,33 +39,10 @@ export function promoteTagsInParent(parent: Parent, source: string = ''): void {
const tagValue = match[2] ?? '';
const tagStart = match.index + boundary.length;
- if (source && childTextStart >= 0) {
- let sourceCursor = childTextStart;
- let valueCursor = 0;
- let hashSourceOffset = -1;
- while (valueCursor <= tagStart && sourceCursor < source.length) {
- const isEscape =
- source[sourceCursor] === '\\' &&
- sourceCursor + 1 < source.length &&
- text[valueCursor] === source[sourceCursor + 1];
- const valueByteSourceOffset = isEscape ? sourceCursor + 1 : sourceCursor;
- if (valueCursor === tagStart) {
- hashSourceOffset = valueByteSourceOffset;
- break;
- }
- sourceCursor = isEscape ? sourceCursor + 2 : sourceCursor + 1;
- valueCursor += 1;
- }
- if (hashSourceOffset >= 0 && isEscapedHash(source, hashSourceOffset)) {
- continue;
- }
- }
+ if (isEscapeDerivedRun(escaped, tagStart, 1)) continue;
if (tagStart > lastIndex) {
- const lead: Text = { type: 'text', value: text.slice(lastIndex, tagStart) };
- const pos = deriveFragmentPosition(source, child as Text, lastIndex, tagStart);
- if (pos) lead.position = pos;
- segments.push(lead);
+ segments.push(sliceTextWithProvenance(source, child as Text, lastIndex, tagStart));
}
const tagNode: TagMdast = { type: 'tag', value: tagValue };
@@ -96,10 +63,7 @@ export function promoteTagsInParent(parent: Parent, source: string = ''): void {
newChildren.push(child);
} else {
if (lastIndex < text.length) {
- const tail: Text = { type: 'text', value: text.slice(lastIndex) };
- const pos = deriveFragmentPosition(source, child as Text, lastIndex, text.length);
- if (pos) tail.position = pos;
- segments.push(tail);
+ segments.push(sliceTextWithProvenance(source, child as Text, lastIndex, text.length));
}
newChildren.push(...segments);
}
diff --git a/packages/core/src/markdown/to-markdown-handlers.ts b/packages/core/src/markdown/to-markdown-handlers.ts
index 102e6b632..168fdcbe5 100644
--- a/packages/core/src/markdown/to-markdown-handlers.ts
+++ b/packages/core/src/markdown/to-markdown-handlers.ts
@@ -5,7 +5,7 @@ import { classifyCharacter } from 'micromark-util-classify-character';
import { isValidSourceLiteralRaw } from '../extensions/source-literal-mark.ts';
import { scanBraceSpans } from './autolink-void-html-guard.ts';
import { widenFenceLength } from './code-fence.ts';
-import type { RawMdxFallbackMdast } from './mdast-augmentation.ts';
+import { type RawMdxFallbackMdast, validateEscapeProvenance } from './mdast-augmentation.ts';
import { TO_MARKDOWN_EXT } from './remark-mdx-agnostic.ts';
import { isInlineWhitespaceNumericCharRef } from './whitespace-char-ref.ts';
import { escapeDecodedTableCellPipes } from './wiki-escape.ts';
@@ -76,27 +76,57 @@ export const toMarkdownHandlers = {
}
}
- if (node.data?.escapedChars?.length) {
+ const escapeProvenance = validateEscapeProvenance(node.data);
+ if (
+ escapeProvenance.kind === 'malformed' &&
+ typeof process !== 'undefined' &&
+ process.env?.OK_DEBUG_POSITION_SLICE === '1'
+ ) {
+ console.warn(
+ '[escape-provenance] malformed escapedChars metadata: CommonMark-safe serialization applies',
+ );
+ }
+
+ if (escapeProvenance.kind === 'valid' && escapeProvenance.data.escapedChars.length > 0) {
const value: string = node.value ?? '';
- const escaped: Array<{ offset: number; char: string }> = node.data.escapedChars;
+ const escaped = escapeProvenance.data.escapedChars;
let result = '';
let lastIdx = 0;
- for (const { offset, char } of escaped) {
- if (offset > lastIdx) {
- result += safeText(state, value.slice(lastIdx, offset), {
+ let previousEscapedChar: string | undefined;
+ const appendPlain = (segment: string, after: string): void => {
+ if (!segment) return;
+ const escapedAngleEnd = previousEscapedChar === '<' ? segment.indexOf('>') : -1;
+ if (escapedAngleEnd >= 0) {
+ result += safeText(
+ state,
+ segment.slice(0, escapedAngleEnd + 1),
+ {
+ ...info,
+ before: info.before + result,
+ after: segment.slice(escapedAngleEnd + 1, escapedAngleEnd + 2) || after,
+ },
+ true,
+ );
+ segment = segment.slice(escapedAngleEnd + 1);
+ }
+ if (segment) {
+ result += safeText(state, segment, {
...info,
before: info.before + result,
- after: '\\',
+ after,
});
}
+ };
+ for (const { offset, char } of escaped) {
+ if (offset > lastIdx) {
+ appendPlain(value.slice(lastIdx, offset), '\\');
+ }
result += `\\${char}`;
lastIdx = offset + 1;
+ previousEscapedChar = char;
}
if (lastIdx < value.length) {
- result += safeText(state, value.slice(lastIdx), {
- ...info,
- before: info.before + result,
- });
+ appendPlain(value.slice(lastIdx), info.after);
}
return result;
}
@@ -874,7 +904,7 @@ function isWhitespaceNumericCharRef(body: string): boolean {
);
}
-const TYPED_WS_REF_PUA = '';
+export const TYPED_WS_REF_PUA = '';
const NUMERIC_CHAR_REF_TOKEN_RE = /&#(?:x[0-9A-Fa-f]+|X[0-9A-Fa-f]+|[0-9]+);/g;
@@ -888,20 +918,32 @@ function emitsAtLineStart(emittedPrefix: string): boolean {
return /[\r\n][\t ]*$/.test(emittedPrefix);
}
-function safeText(state: State, value: string, info: Info): string {
+function safeText(
+ state: State,
+ value: string,
+ info: Info,
+ suppressGfmAutolinkLiteral: boolean = false,
+): string {
const { matched } = scanBraceSpans(value, { escapeAware: false });
- if (matched.length === 0) return safeTextSegment(state, value, info);
+ if (matched.length === 0) {
+ return safeTextSegment(state, value, info, suppressGfmAutolinkLiteral);
+ }
let result = '';
let pending = 0;
const flushNormal = (end: number): void => {
if (end > pending) {
- result += safeTextSegment(state, value.slice(pending, end), {
- ...info,
- before: info.before + result,
- after: value.slice(end, end + 1) || info.after,
- });
+ result += safeTextSegment(
+ state,
+ value.slice(pending, end),
+ {
+ ...info,
+ before: info.before + result,
+ after: value.slice(end, end + 1) || info.after,
+ },
+ suppressGfmAutolinkLiteral,
+ );
}
pending = end;
};
@@ -916,7 +958,12 @@ function safeText(state: State, value: string, info: Info): string {
return result;
}
-function safeTextSegment(state: State, value: string, info: Info): string {
+function safeTextSegment(
+ state: State,
+ value: string,
+ info: Info,
+ suppressGfmAutolinkLiteral: boolean = false,
+): string {
const originalUnsafe = state.unsafe;
state.unsafe = originalUnsafe.filter((u) => {
if (u.character === '&' && u.after === '[#A-Za-z]') return false;
@@ -925,6 +972,14 @@ function safeTextSegment(state: State, value: string, info: Info): string {
if (u.character === '(') return false;
if (u.character === '!' && u.after === '\\[' && u.inConstruct === 'phrasing') return false;
if (u.character === '@' && u.before === '[+\\-.\\w]') return false;
+ if (
+ suppressGfmAutolinkLiteral &&
+ u.character === ':' &&
+ u.before === '[ps]' &&
+ u.after === '\\/'
+ ) {
+ return false;
+ }
if (u.character === '{' && Array.isArray(u.inConstruct) && u.inConstruct.includes('phrasing')) {
return false;
}
diff --git a/packages/core/src/markdown/void-br-promoter.ts b/packages/core/src/markdown/void-br-promoter.ts
index a0aecd430..1a76d6a38 100644
--- a/packages/core/src/markdown/void-br-promoter.ts
+++ b/packages/core/src/markdown/void-br-promoter.ts
@@ -1,7 +1,7 @@
import type { Break, Parent, Root, Text } from 'mdast';
import { SKIP, visit } from 'unist-util-visit';
import type { VFile } from 'vfile';
-import { deriveFragmentPosition } from './promoter-position.ts';
+import { deriveFragmentPosition, sliceTextWithProvenance } from './promoter-position.ts';
const VOID_BR_IN_TEXT_RE = /(?<!\uE102)<br[ \t]*\/?>/g;
@@ -29,10 +29,7 @@ function promoteVoidBrInParent(parent: Parent, source: string): void {
const matchStart = match.index;
if (matchStart > lastIndex) {
- const lead: Text = { type: 'text', value: text.slice(lastIndex, matchStart) };
- const pos = deriveFragmentPosition(source, child as Text, lastIndex, matchStart);
- if (pos) lead.position = pos;
- segments.push(lead);
+ segments.push(sliceTextWithProvenance(source, child as Text, lastIndex, matchStart));
}
const brNode: Break = {
@@ -56,10 +53,7 @@ function promoteVoidBrInParent(parent: Parent, source: string): void {
newChildren.push(child);
} else {
if (lastIndex < text.length) {
- const tail: Text = { type: 'text', value: text.slice(lastIndex) };
- const pos = deriveFragmentPosition(source, child as Text, lastIndex, text.length);
- if (pos) tail.position = pos;
- segments.push(tail);
+ segments.push(sliceTextWithProvenance(source, child as Text, lastIndex, text.length));
}
newChildren.push(...segments);
}
diff --git a/packages/server/src/content/generate-index-conformance.test.ts b/packages/server/src/content/generate-index-conformance.test.ts
index 01efd4a66..abe856381 100644
--- a/packages/server/src/content/generate-index-conformance.test.ts
+++ b/packages/server/src/content/generate-index-conformance.test.ts
@@ -39,7 +39,7 @@ const ADMITTED = ['index', ...ENTRIES.map((e) => e.path.replace(/\.md$/, ''))];
describe('generated index — OKF conformance', () => {
test('the generated root index produces zero okf findings', async () => {
- const markdown = buildIndexMarkdown(ENTRIES, { isRoot: true });
+ const markdown = buildIndexMarkdown(ENTRIES, { warningScope: false, isRoot: true });
const findings = await lintDocument(markdown, OKF_LINT_CONFIG, 'index.md');
const detail = findings
@@ -50,20 +50,20 @@ describe('generated index — OKF conformance', () => {
});
test('a generated non-root index produces zero okf findings', async () => {
- const markdown = buildIndexMarkdown(ENTRIES, { isRoot: false });
+ const markdown = buildIndexMarkdown(ENTRIES, { warningScope: false, isRoot: false });
const findings = await lintDocument(markdown, OKF_LINT_CONFIG, 'concepts/index.md');
expect(findings.map((f) => f.code)).toEqual([]);
});
test('an empty bundle still yields a conformant root index', async () => {
- const markdown = buildIndexMarkdown([], { isRoot: true });
+ const markdown = buildIndexMarkdown([], { warningScope: false, isRoot: true });
const findings = await lintDocument(markdown, OKF_LINT_CONFIG, 'index.md');
expect(findings.map((f) => f.code)).toEqual([]);
});
test('every link in the generated index resolves', () => {
- const markdown = buildIndexMarkdown(ENTRIES, { isRoot: true });
+ const markdown = buildIndexMarkdown(ENTRIES, { warningScope: false, isRoot: true });
const broken = computeBrokenOutboundLinks(markdown, 'index', ADMITTED);
expect(
@@ -99,19 +99,27 @@ async function findingCodes(markdown: string, docName: string): Promise<string[]
describe('generated index — default lint profile', () => {
test('the generated root index is clean under markdownlint defaults', async () => {
- expect(await findingCodes(buildIndexMarkdown(ENTRIES, { isRoot: true }), 'index.md')).toEqual(
- [],
- );
+ expect(
+ await findingCodes(
+ buildIndexMarkdown(ENTRIES, { warningScope: false, isRoot: true }),
+ 'index.md',
+ ),
+ ).toEqual([]);
});
test('a generated non-root index is clean under markdownlint defaults', async () => {
expect(
- await findingCodes(buildIndexMarkdown(ENTRIES, { isRoot: false }), 'concepts/index.md'),
+ await findingCodes(
+ buildIndexMarkdown(ENTRIES, { warningScope: false, isRoot: false }),
+ 'concepts/index.md',
+ ),
).toEqual([]);
});
test('an empty bundle is clean under markdownlint defaults', async () => {
- expect(await findingCodes(buildIndexMarkdown([], { isRoot: true }), 'index.md')).toEqual([]);
+ expect(
+ await findingCodes(buildIndexMarkdown([], { warningScope: false, isRoot: true }), 'index.md'),
+ ).toEqual([]);
});
test('a Subdirectories document type plus a child directory is clean under markdownlint defaults', async () => {
@@ -124,9 +132,10 @@ describe('generated index — default lint profile', () => {
},
],
{
+ warningScope: false,
isRoot: false,
directory: 'concepts',
- subdirectories: [{ path: 'concepts/nested/index.md', title: 'nested' }],
+ subdirectories: [{ directory: 'concepts/nested', title: 'nested' }],
},
);
@@ -136,7 +145,7 @@ describe('generated index — default lint profile', () => {
test('an Index document type is clean under markdownlint defaults', async () => {
const readmeOnly = buildIndexMarkdown(
[{ path: 'testing/README.md', title: 'Testing', type: 'Index' }],
- { isRoot: false, directory: 'testing' },
+ { warningScope: false, isRoot: false, directory: 'testing' },
);
expect(await findingCodes(readmeOnly, 'testing/index.md')).toEqual([]);
@@ -149,7 +158,7 @@ describe('generated index — default lint profile', () => {
{ path: 'auth/login-flow.md', title: 'Login flow', type: 'Flow' },
{ path: 'auth/sso.md', title: 'SSO', type: 'Feature Doc' },
],
- { isRoot: false, directory: 'auth' },
+ { warningScope: false, isRoot: false, directory: 'auth' },
);
expect(await findingCodes(multiSection, 'auth/index.md')).toEqual([]);
@@ -161,7 +170,7 @@ describe('generated index — default lint profile', () => {
{ path: 'README.md', title: 'Knowledge base', type: 'Index' },
{ path: 'welcome.md', title: 'Welcome', type: 'note' },
],
- { isRoot: true },
+ { warningScope: false, isRoot: true },
);
expect(await findingCodes(markdown, 'index.md')).toEqual([]);
@@ -214,48 +223,63 @@ describe('generated index — heading-identity agreement with MD024', () => {
{ path: 'readme.md', title: 'Overview', description: undefined, type },
{ path: 'flow.md', title: 'Login flow', description: undefined, type: 'Flow' },
],
- { isRoot: false, directory: '' },
+ { warningScope: false, isRoot: false, directory: '' },
);
- test.each([
- ['Index'],
- ['Index #'],
- ['Index ##'],
- ['<b></b>Index'],
- ['Index<!--x-->'],
- ['Index<!-- a>b -->'],
- ['Index<a href="x>y">'],
- ])('a type reducing to the title merges into it: %j', async (type) => {
- const markdown = withType(type);
+ test('the exact Index type merges into the title', async () => {
+ const markdown = withType('Index');
expect(mdOnly(await findingCodes(markdown, 'index.md')), `emitted:\n${markdown}`).toEqual([]);
expect(markdown, `emitted:\n${markdown}`).toContain('](./readme.md)');
+ expect(markdown).not.toContain('## Index');
});
test.each([
- ['index'],
- ['INDEX'],
- ['Indexes'],
- ['Index Notes'],
- ['Flow'],
- ['Index <b>#</b>'],
- ['Index<b> #</b>'],
- ])('a type distinct from the title keeps its own section: %j', async (type) => {
- const markdown = withType(type);
-
- expect(markdown, `emitted:\n${markdown}`).toContain(`## ${type}`);
- expect(mdOnly(await findingCodes(markdown, 'index.md')), `emitted:\n${markdown}`).toEqual([]);
- });
+ ['Index #', 'Index \\#'],
+ ['Index ##', 'Index \\#\\#'],
+ ['<b></b>Index', '\\<b\\>\\<\\/b\\>Index'],
+ ['Index<!--x-->', 'Index\\<\\!\\-\\-x\\-\\-\\>'],
+ ['Index<!-- a>b -->', 'Index\\<\\!\\-\\- a\\>b \\-\\-\\>'],
+ ['Index<a href="x>y">', 'Index\\<a href\\=\\"x\\>y\\"\\>'],
+ ['index', 'index'],
+ ['INDEX', 'INDEX'],
+ ['Indexes', 'Indexes'],
+ ['Index Notes', 'Index Notes'],
+ ['Flow', 'Flow'],
+ ['Index <b>#</b>', 'Index \\<b\\>\\#\\<\\/b\\>'],
+ ['Index<b> #</b>', 'Index\\<b\\> \\#\\<\\/b\\>'],
+ ])(
+ 'a literal type distinct from the title keeps its own section: %j',
+ async (type, emittedHeading) => {
+ const markdown = withType(type);
+
+ expect(markdown, `emitted:\n${markdown}`).toContain(`## ${emittedHeading}`);
+ expect(mdOnly(await findingCodes(markdown, 'index.md')), `emitted:\n${markdown}`).toEqual([]);
+ },
+ );
- test.each([[['#', '##']], [['#', '<b></b>']], [['<b></b>', '<i></i>']]])(
- 'types whose rendered heading text is empty merge with each other: %j',
- async (pair) => {
+ test.each([
+ [
+ ['#', '##'],
+ ['\\#', '\\#\\#'],
+ ],
+ [
+ ['#', '<b></b>'],
+ ['\\#', '\\<b\\>\\<\\/b\\>'],
+ ],
+ [
+ ['<b></b>', '<i></i>'],
+ ['\\<b\\>\\<\\/b\\>', '\\<i\\>\\<\\/i\\>'],
+ ],
+ ])(
+ 'punctuation and HTML-shaped literal types remain distinct: %j',
+ async (pair, emittedHeadings) => {
const markdown = buildIndexMarkdown(
[
{ path: 'a.md', title: 'A', description: undefined, type: pair[0] },
{ path: 'b.md', title: 'B', description: undefined, type: pair[1] },
],
- { isRoot: false, directory: '' },
+ { warningScope: false, isRoot: false, directory: '' },
);
expect(mdOnly(await findingCodes(markdown, 'index.md')), `emitted:\n${markdown}`).toEqual([]);
@@ -263,21 +287,22 @@ describe('generated index — heading-identity agreement with MD024', () => {
expect(markdown).toContain('](./b.md)');
const sections = (markdown.match(/^## .+$/gm) ?? []).map((line) => line.slice(3));
- expect(sections, `emitted:\n${markdown}`).toHaveLength(1);
- expect(pair, `emitted:\n${markdown}`).toContain(sections[0]);
+ expect(sections, `emitted:\n${markdown}`).toEqual(emittedHeadings);
},
);
- test('two derived sections reducing to the same content merge with each other', async () => {
+ test('two literal sections remain distinct when Markdown would reduce them to one heading', async () => {
const markdown = buildIndexMarkdown(
[
{ path: 'a.md', title: 'A', description: undefined, type: 'Flow' },
{ path: 'b.md', title: 'B', description: undefined, type: 'Flow #' },
],
- { isRoot: false, directory: '' },
+ { warningScope: false, isRoot: false, directory: '' },
);
expect(mdOnly(await findingCodes(markdown, 'index.md')), `emitted:\n${markdown}`).toEqual([]);
+ expect(markdown).toContain('## Flow\n');
+ expect(markdown).toContain('## Flow \\#\n');
expect(markdown).toContain('](./a.md)');
expect(markdown).toContain('](./b.md)');
});
diff --git a/packages/server/src/content/generate-index-literal-metadata.test.ts b/packages/server/src/content/generate-index-literal-metadata.test.ts
new file mode 100644
index 000000000..d14ca61ce
--- /dev/null
+++ b/packages/server/src/content/generate-index-literal-metadata.test.ts
@@ -0,0 +1,1012 @@
+import assert from 'node:assert/strict';
+import {
+ MarkdownManager,
+ resolveInternalHref,
+ sharedExtensions,
+} from '@inkeep/open-knowledge-core';
+import type { JSONContent } from '@tiptap/core';
+import { describe, expect, test, vi } from 'vitest';
+import * as Y from 'yjs';
+import { getLogger } from '../logger.ts';
+import { extractPageDescription, extractPageTitle, extractPageType } from '../page-identity.ts';
+import {
+ buildIndexMarkdown,
+ createGeneratedIndexWarningScope,
+ type IndexEntry,
+} from './generate-index.ts';
+import { type GeneratedArtifactEnv, writeGeneratedArtifact } from './generated-artifact.ts';
+import { planDirectoryIndexRegenerations } from './regenerate-index.ts';
+
+const markdownManager = new MarkdownManager({ extensions: sharedExtensions });
+
+const MARKDOWN_METACHARACTER_CORPUS = [
+ 'stars *emphasis* and **strong**',
+ 'underscore _under_',
+ 'strike ~~gone~~',
+ 'code `value`',
+ 'highlight ==marked==',
+ 'math $x+1$',
+ 'wiki [[Target]]',
+ 'HTML <u>raw</u>',
+ 'comment <!--hidden-->',
+ 'MDX <Widget />',
+ 'link [extra](https://example.test)',
+ 'image ![pixel](https://example.test/pixel.png)',
+ 'autolink <https://example.test/auto>',
+ 'entities &amp; and &#65;',
+ 'backslash \\',
+ 'brackets [literal]',
+ 'heading close #',
+].join('; ');
+
+const PORTABLE_PATH_CORPUS = 'folder _under_ ~~gone~~ ==marked== $x+1$ [literal]';
+
+const RESIDUAL_CONTROL_CODE_POINTS = [
+ ...Array.from({ length: 0x20 }, (_, codePoint) => codePoint).filter(
+ (codePoint) => codePoint < 0x09 || codePoint > 0x0d,
+ ),
+ ...Array.from({ length: 0x21 }, (_, offset) => offset + 0x7f),
+];
+
+const RESIDUAL_CONTROL_CORPUS = String.fromCodePoint(...RESIDUAL_CONTROL_CODE_POINTS);
+
+const BIDI_OVERRIDE_PENDING_POLICY = '\u202e';
+
+const ZERO_WIDTH_AND_JOINING_CONTROLS = '\u200b\u200c\u200d';
+
+const CONTROL_BEARING_LITERAL = `Cafe\u0301\t\n\v\f\rLabel${RESIDUAL_CONTROL_CORPUS}`;
+
+const NORMALIZED_CONTROL_LITERAL = `Cafe\u0301 Label${'\ufffd'.repeat(RESIDUAL_CONTROL_CODE_POINTS.length)}`;
+
+const PARSER_RESERVATION_TRACER = {
+ literal: 'Note\ue102x',
+ normalizedLiteral: 'Note\ufffdx',
+};
+
+const ADDITIONAL_MUTATING_PARSER_RESERVATION_CASES = [
+ {
+ reservation: 'R23 secondary U+E005',
+ literal: 'Note\ue005x',
+ normalizedLiteral: 'Note\ufffdx',
+ },
+ {
+ reservation: 'R23 secondary U+E006',
+ literal: 'Note\ue006x',
+ normalizedLiteral: 'Note\ufffdx',
+ },
+ {
+ reservation: 'R23 secondary U+E007',
+ literal: 'Note\ue007x',
+ normalizedLiteral: 'Note\ufffdx',
+ },
+ {
+ reservation: 'R23 secondary U+E008',
+ literal: 'Note\ue008x',
+ normalizedLiteral: 'Note\ufffdx',
+ },
+ {
+ reservation: 'R23 secondary U+E009',
+ literal: 'Note\ue009x',
+ normalizedLiteral: 'Note\ufffdx',
+ },
+ {
+ reservation: 'entity delimiter pair U+E100-U+E101',
+ literal: 'Note\ue100amp\ue101x',
+ normalizedLiteral: 'Note\ufffdamp\ufffdx',
+ },
+ {
+ reservation: 'typed whitespace U+E103',
+ literal: 'Note\ue103x',
+ normalizedLiteral: 'Note\ufffdx',
+ },
+];
+
+const COLLISION_DEFENDED_PARSER_RESERVATION_CORPUS = `before${String.fromCodePoint(
+ ...Array.from({ length: 5 }, (_, offset) => 0xe000 + offset),
+ ...Array.from({ length: 0xf8ff - 0xe200 + 1 }, (_, offset) => 0xe200 + offset),
+)}after`;
+
+const GENERATED_NODE_TYPES = new Set(['doc', 'heading', 'list', 'listItem', 'paragraph', 'text']);
+const GENERATED_MARK_TYPES = new Set(['escapeMark', 'link', 'sourceLiteral']);
+
+const GENERATED_ARTIFACT_ORIGIN = Object.freeze({
+ source: 'local' as const,
+ skipStoreHooks: false,
+ context: Object.freeze({ origin: 'generated-index-literal-metadata', paired: true as const }),
+});
+
+const GENERATED_ARTIFACT_WRITER: GeneratedArtifactEnv['writer'] = {
+ id: 'ok-generator',
+ name: 'OpenKnowledge (generated)',
+ email: '[email protected]',
+};
+
+function walkJson(root: JSONContent): JSONContent[] {
+ const nodes: JSONContent[] = [];
+ const visit = (node: JSONContent): void => {
+ nodes.push(node);
+ node.content?.forEach(visit);
+ };
+ visit(root);
+ return nodes;
+}
+
+function textContent(root: JSONContent): string {
+ if (root.text !== undefined) return root.text;
+ return root.content?.map(textContent).join('') ?? '';
+}
+
+function hasResidualControl(value: string): boolean {
+ return Array.from(value).some((character) => {
+ const codePoint = character.charCodeAt(0);
+ return (
+ codePoint <= 0x08 ||
+ (codePoint >= 0x0e && codePoint <= 0x1f) ||
+ (codePoint >= 0x7f && codePoint <= 0x9f)
+ );
+ });
+}
+
+function linkHrefs(root: JSONContent): string[] {
+ return walkJson(root).flatMap((node) =>
+ (node.marks ?? [])
+ .filter((mark) => mark.type === 'link')
+ .map((mark) => String(mark.attrs?.href ?? '')),
+ );
+}
+
+function expectLiteralThroughEditor(
+ markdown: string,
+ literal: string,
+ intendedHrefs: readonly string[],
+): void {
+ const parsed = markdownManager.parse(markdown);
+ const serializedOnce = markdownManager.serialize(parsed);
+ const reparsed = markdownManager.parse(serializedOnce);
+ const stages = [
+ ['initial parse', parsed],
+ ['parse after serialization', reparsed],
+ ] as const;
+
+ for (const [stage, document] of stages) {
+ const nodes = walkJson(document);
+ const unexpectedNodeTypes = [
+ ...new Set(
+ nodes
+ .map((node) => node.type ?? '(missing)')
+ .filter((type) => !GENERATED_NODE_TYPES.has(type)),
+ ),
+ ].toSorted();
+ const unexpectedMarkTypes = [
+ ...new Set(
+ nodes
+ .flatMap((node) => node.marks ?? [])
+ .map((mark) => mark.type)
+ .filter((type) => !GENERATED_MARK_TYPES.has(type)),
+ ),
+ ].toSorted();
+
+ expect.soft(textContent(document), stage).toContain(literal);
+ expect.soft(unexpectedNodeTypes, stage).toEqual([]);
+ expect.soft(unexpectedMarkTypes, stage).toEqual([]);
+ expect
+ .soft([...new Set(linkHrefs(document))].toSorted(), stage)
+ .toEqual([...new Set(intendedHrefs)].toSorted());
+ }
+
+ expect(markdownManager.serialize(reparsed)).toBe(serializedOnce);
+}
+
+function expectInitialEditorStability(
+ markdown: string,
+ expectedMarkdown: string,
+ literal: string,
+ intendedHrefs: readonly string[],
+): void {
+ expectLiteralThroughEditor(markdown, literal, intendedHrefs);
+ expect(markdown).toBe(expectedMarkdown);
+ expect(markdownManager.serialize(markdownManager.parse(markdown))).toBe(expectedMarkdown);
+}
+
+interface ParsedSection {
+ level: number;
+ heading: string;
+ hrefs: string[];
+}
+
+function parsedSections(root: JSONContent): ParsedSection[] {
+ const sections: ParsedSection[] = [];
+ let current: ParsedSection | undefined;
+
+ for (const node of root.content ?? []) {
+ if (node.type === 'heading') {
+ current = {
+ level: Number(node.attrs?.level),
+ heading: textContent(node),
+ hrefs: [],
+ };
+ sections.push(current);
+ continue;
+ }
+ current?.hrefs.push(...linkHrefs(node));
+ }
+
+ return sections;
+}
+
+function expectGroupingThroughEditor(
+ markdown: string,
+ expectedSections: readonly ParsedSection[],
+ intendedHrefs: readonly string[],
+): void {
+ const parsed = markdownManager.parse(markdown);
+ const serializedOnce = markdownManager.serialize(parsed);
+ const reparsed = markdownManager.parse(serializedOnce);
+ const stages = [
+ ['initial parse', parsed],
+ ['parse after serialization', reparsed],
+ ] as const;
+
+ for (const [stage, document] of stages) {
+ const nodes = walkJson(document);
+ const unexpectedNodeTypes = [
+ ...new Set(
+ nodes
+ .map((node) => node.type ?? '(missing)')
+ .filter((type) => !GENERATED_NODE_TYPES.has(type)),
+ ),
+ ].toSorted();
+ const unexpectedMarkTypes = [
+ ...new Set(
+ nodes
+ .flatMap((node) => node.marks ?? [])
+ .map((mark) => mark.type)
+ .filter((type) => !GENERATED_MARK_TYPES.has(type)),
+ ),
+ ].toSorted();
+
+ expect.soft(parsedSections(document), stage).toEqual(expectedSections);
+ expect.soft(unexpectedNodeTypes, stage).toEqual([]);
+ expect.soft(unexpectedMarkTypes, stage).toEqual([]);
+ expect
+ .soft([...new Set(linkHrefs(document))].toSorted(), stage)
+ .toEqual([...new Set(intendedHrefs)].toSorted());
+ }
+
+ expect(markdownManager.serialize(reparsed)).toBe(serializedOnce);
+}
+
+function generatedArtifactEnv(
+ document: Y.Doc | undefined,
+ captureDiskMarkdown: (markdown: string) => void,
+): GeneratedArtifactEnv {
+ return {
+ origin: GENERATED_ARTIFACT_ORIGIN,
+ writer: GENERATED_ARTIFACT_WRITER,
+ isConflict: () => false,
+ getDocument: () => document,
+ writeDisk: (_absPath, markdown) => captureDiskMarkdown(markdown),
+ registerWrite: () => undefined,
+ noteFileIndex: () => undefined,
+ signalFiles: () => undefined,
+ attribute: () => Promise.resolve(),
+ };
+}
+
+const metadataSource = [
+ '---',
+ `title: '${MARKDOWN_METACHARACTER_CORPUS}'`,
+ `description: '${MARKDOWN_METACHARACTER_CORPUS}'`,
+ `type: '${MARKDOWN_METACHARACTER_CORPUS}'`,
+ '---',
+ '',
+ '# Fallback',
+ '',
+].join('\n');
+
+const extractedTitle = extractPageTitle(metadataSource, 'fallback');
+const extractedDescription = extractPageDescription(metadataSource);
+const extractedType = extractPageType(metadataSource);
+
+const FIELD_CASES: Array<{ field: string; entry: IndexEntry }> = [
+ {
+ field: 'title',
+ entry: {
+ path: 'guides/entry.md',
+ title: extractedTitle,
+ description: 'Plain description',
+ type: 'Guide',
+ },
+ },
+ {
+ field: 'description',
+ entry: {
+ path: 'guides/entry.md',
+ title: 'Plain title',
+ description: extractedDescription,
+ type: 'Guide',
+ },
+ },
+ {
+ field: 'type',
+ entry: {
+ path: 'guides/entry.md',
+ title: 'Plain title',
+ description: 'Plain description',
+ type: extractedType,
+ },
+ },
+];
+
+interface LiteralSurfaceCase {
+ surface: string;
+ render: (literal: string) => { markdown: string; hrefs: string[] };
+ expected: (literal: string, pathLiteral: string) => string;
+}
+
+const LITERAL_SURFACE_CASES: LiteralSurfaceCase[] = [
+ {
+ surface: 'title metadata',
+ render: (literal) => ({
+ markdown: buildIndexMarkdown(
+ [{ path: 'guides/entry.md', title: literal, description: 'Plain', type: 'Guide' }],
+ { warningScope: false, isRoot: false, directory: 'guides' },
+ ),
+ hrefs: ['./entry.md'],
+ }),
+ expected: (literal) => `# Index\n\n## Guide\n\n* [${literal}](./entry.md) - Plain\n`,
+ },
+ {
+ surface: 'description metadata',
+ render: (literal) => ({
+ markdown: buildIndexMarkdown(
+ [{ path: 'guides/entry.md', title: 'Plain', description: literal, type: 'Guide' }],
+ { warningScope: false, isRoot: false, directory: 'guides' },
+ ),
+ hrefs: ['./entry.md'],
+ }),
+ expected: (literal) => `# Index\n\n## Guide\n\n* [Plain](./entry.md) - ${literal}\n`,
+ },
+ {
+ surface: 'type metadata',
+ render: (literal) => ({
+ markdown: buildIndexMarkdown(
+ [{ path: 'guides/entry.md', title: 'Plain', description: 'Plain', type: literal }],
+ { warningScope: false, isRoot: false, directory: 'guides' },
+ ),
+ hrefs: ['./entry.md'],
+ }),
+ expected: (literal) => `# Index\n\n## ${literal}\n\n* [Plain](./entry.md) - Plain\n`,
+ },
+ {
+ surface: 'derived folder label',
+ render: (literal) => {
+ const guides = planDirectoryIndexRegenerations({
+ warningScope: false,
+ docs: [[`guides/${literal}/entry`, { title: 'Entry', type: 'Guide' }]],
+ docExtension: () => '.md',
+ currentMarkdownFor: () => null,
+ }).find((decision) => decision.directory === 'guides');
+ assert(guides);
+ return {
+ markdown: guides.markdown,
+ hrefs: [`./${encodeURIComponent(literal)}/index.md`],
+ };
+ },
+ expected: (literal, pathLiteral) =>
+ `# Index\n\n## Subdirectories\n\n* [${literal}](./${encodeURIComponent(pathLiteral)}/index.md)\n`,
+ },
+];
+
+describe('generated index literal metadata', () => {
+ test.each([
+ {
+ kind: 'control',
+ folderLabel: 'control\u007ffolder',
+ message: 'generated index metadata contained a Cc control; replaced it with U+FFFD',
+ },
+ {
+ kind: 'parser-reservation',
+ folderLabel: `parser${PARSER_RESERVATION_TRACER.literal}folder`,
+ message:
+ 'generated index metadata contained a parser-reserved private-use code point; replaced it with U+FFFD',
+ },
+ ])(
+ '$kind substitution attributes and rearms the source folder label',
+ ({ kind, folderLabel, message }) => {
+ const warn = vi.spyOn(getLogger('generated-index'), 'warn').mockImplementation(() => {});
+ const warningScope = createGeneratedIndexWarningScope('/projects/folder-label/content');
+ const sourceDirectory = `guides/${folderLabel}`;
+ const corruptDocs = [
+ [`${sourceDirectory}/entry`, { title: 'Entry', type: 'Guide' }],
+ ] as const;
+ const cleanDocs = [['guides/clean-folder/entry', { title: 'Entry', type: 'Guide' }]] as const;
+ const plan = (docs: typeof corruptDocs | typeof cleanDocs) =>
+ planDirectoryIndexRegenerations({
+ docs,
+ docExtension: () => '.md',
+ currentMarkdownFor: () => null,
+ warningScope,
+ });
+
+ try {
+ plan(corruptDocs);
+ plan(corruptDocs);
+ expect(warn).toHaveBeenCalledOnce();
+ expect(warn).toHaveBeenCalledWith(
+ {
+ contentDir: '/projects/folder-label/content',
+ path: sourceDirectory,
+ field: 'folder-label',
+ kind,
+ },
+ message,
+ );
+
+ plan(cleanDocs);
+ plan(corruptDocs);
+ expect(warn).toHaveBeenCalledTimes(2);
+ } finally {
+ warn.mockRestore();
+ }
+ },
+ );
+
+ test('content-directory scopes isolate both warning kinds and their clean episodes', () => {
+ const warn = vi.spyOn(getLogger('generated-index'), 'warn').mockImplementation(() => {});
+ const scopeA = createGeneratedIndexWarningScope('/projects/a/content');
+ const scopeB = createGeneratedIndexWarningScope('/projects/b/content');
+ const path = 'guides/shared-warning.md';
+ const corruptEntry: IndexEntry = {
+ path,
+ title: `Title\u0000${PARSER_RESERVATION_TRACER.literal}`,
+ description: 'Plain',
+ type: 'Guide',
+ };
+ const cleanEntry = { ...corruptEntry, title: 'Clean title' };
+ const render = (warningScope: typeof scopeA, entry: IndexEntry) =>
+ buildIndexMarkdown([entry], {
+ isRoot: false,
+ directory: 'guides',
+ warningScope,
+ });
+
+ try {
+ render(scopeA, corruptEntry);
+ render(scopeB, corruptEntry);
+ expect(warn).toHaveBeenCalledTimes(4);
+
+ for (const contentDir of ['/projects/a/content', '/projects/b/content']) {
+ expect(warn).toHaveBeenCalledWith(
+ { contentDir, path, field: 'title', kind: 'control' },
+ 'generated index metadata contained a Cc control; replaced it with U+FFFD',
+ );
+ expect(warn).toHaveBeenCalledWith(
+ { contentDir, path, field: 'title', kind: 'parser-reservation' },
+ 'generated index metadata contained a parser-reserved private-use code point; replaced it with U+FFFD',
+ );
+ }
+
+ render(scopeA, cleanEntry);
+ render(scopeB, corruptEntry);
+ expect(warn).toHaveBeenCalledTimes(4);
+
+ render(scopeA, corruptEntry);
+ expect(warn).toHaveBeenCalledTimes(6);
+
+ render(scopeA, { ...corruptEntry, title: PARSER_RESERVATION_TRACER.literal });
+ render(scopeA, corruptEntry);
+ expect(warn).toHaveBeenCalledTimes(7);
+ expect(
+ warn.mock.calls.filter(
+ ([payload, message]) =>
+ payload.contentDir === '/projects/a/content' &&
+ message === 'generated index metadata contained a Cc control; replaced it with U+FFFD',
+ ),
+ ).toHaveLength(3);
+ expect(
+ warn.mock.calls.filter(
+ ([payload, message]) =>
+ payload.contentDir === '/projects/a/content' &&
+ message ===
+ 'generated index metadata contained a parser-reserved private-use code point; replaced it with U+FFFD',
+ ),
+ ).toHaveLength(2);
+ } finally {
+ warn.mockRestore();
+ }
+ });
+
+ test('control substitution warns once per source path and metadata field', () => {
+ const warn = vi.spyOn(getLogger('generated-index'), 'warn').mockImplementation(() => {});
+ const warningScope = createGeneratedIndexWarningScope('/projects/control/content');
+ const entry: IndexEntry = {
+ path: 'guides/control-warning.md',
+ title: 'Title\u0000',
+ description: 'Description\u0001',
+ type: 'Type\u0002',
+ };
+
+ try {
+ buildIndexMarkdown([entry], { isRoot: false, directory: 'guides', warningScope });
+ expect(warn).toHaveBeenCalledTimes(3);
+ for (const field of ['title', 'description', 'type']) {
+ expect(warn).toHaveBeenCalledWith(
+ {
+ contentDir: '/projects/control/content',
+ path: 'guides/control-warning.md',
+ field,
+ kind: 'control',
+ },
+ 'generated index metadata contained a Cc control; replaced it with U+FFFD',
+ );
+ }
+
+ buildIndexMarkdown([entry], { isRoot: false, directory: 'guides', warningScope });
+ expect(warn).toHaveBeenCalledTimes(3);
+
+ buildIndexMarkdown(
+ [
+ {
+ ...entry,
+ title: 'Clean title',
+ description: 'Clean description',
+ type: 'Clean type',
+ },
+ ],
+ { isRoot: false, directory: 'guides', warningScope },
+ );
+ buildIndexMarkdown([entry], { isRoot: false, directory: 'guides', warningScope });
+ expect(warn).toHaveBeenCalledTimes(6);
+ } finally {
+ warn.mockRestore();
+ }
+ });
+
+ test('parser-reservation substitution warns once per source path and metadata field', () => {
+ const warn = vi.spyOn(getLogger('generated-index'), 'warn').mockImplementation(() => {});
+ const warningScope = createGeneratedIndexWarningScope('/projects/parser-reservation/content');
+ const entry: IndexEntry = {
+ path: 'guides/parser-reservation-warning.md',
+ title: PARSER_RESERVATION_TRACER.literal,
+ description: 'Plain',
+ type: 'Guide',
+ };
+
+ try {
+ buildIndexMarkdown([entry], { isRoot: false, directory: 'guides', warningScope });
+ expect(warn).toHaveBeenCalledOnce();
+ expect(warn).toHaveBeenCalledWith(
+ {
+ contentDir: '/projects/parser-reservation/content',
+ path: 'guides/parser-reservation-warning.md',
+ field: 'title',
+ kind: 'parser-reservation',
+ },
+ 'generated index metadata contained a parser-reserved private-use code point; replaced it with U+FFFD',
+ );
+
+ buildIndexMarkdown([entry], { isRoot: false, directory: 'guides', warningScope });
+ expect(warn).toHaveBeenCalledOnce();
+
+ buildIndexMarkdown([{ ...entry, title: 'Clean title' }], {
+ isRoot: false,
+ directory: 'guides',
+ warningScope,
+ });
+ buildIndexMarkdown([entry], { isRoot: false, directory: 'guides', warningScope });
+ expect(warn).toHaveBeenCalledTimes(2);
+ } finally {
+ warn.mockRestore();
+ }
+ });
+
+ test('generation snapshots rearm warnings for metadata sources that disappear', () => {
+ const warn = vi.spyOn(getLogger('generated-index'), 'warn').mockImplementation(() => {});
+ const warningScope = createGeneratedIndexWarningScope('/projects/snapshot/content');
+ const corruptDocs = [
+ [
+ 'guides/removed-description',
+ { title: 'Present', description: 'Description\u0000', type: 'Guide' },
+ ],
+ ['guides/removed-document', { title: PARSER_RESERVATION_TRACER.literal, type: 'Guide' }],
+ ] as const;
+ const cleanDocs = [
+ ['guides/removed-description', { title: 'Present', type: 'Guide' }],
+ ] as const;
+ const plan = (docs: typeof corruptDocs | typeof cleanDocs) =>
+ planDirectoryIndexRegenerations({
+ docs,
+ docExtension: () => '.md',
+ currentMarkdownFor: () => null,
+ warningScope,
+ });
+
+ try {
+ plan(corruptDocs);
+ plan(corruptDocs);
+ expect(warn).toHaveBeenCalledTimes(2);
+
+ plan(cleanDocs);
+ plan(corruptDocs);
+ expect(warn).toHaveBeenCalledTimes(4);
+ expect(
+ warn.mock.calls.filter(
+ ([, message]) =>
+ message === 'generated index metadata contained a Cc control; replaced it with U+FFFD',
+ ),
+ ).toHaveLength(2);
+ expect(
+ warn.mock.calls.filter(
+ ([, message]) =>
+ message ===
+ 'generated index metadata contained a parser-reserved private-use code point; replaced it with U+FFFD',
+ ),
+ ).toHaveLength(2);
+ } finally {
+ warn.mockRestore();
+ }
+ });
+
+ test.each(LITERAL_SURFACE_CASES)(
+ '$surface neutralizes the parser reservation tracer before its first editor cycle',
+ ({ render, expected }) => {
+ const { markdown, hrefs } = render(PARSER_RESERVATION_TRACER.literal);
+ const expectedMarkdown = expected(
+ PARSER_RESERVATION_TRACER.normalizedLiteral,
+ PARSER_RESERVATION_TRACER.literal,
+ );
+
+ expect(markdown).toContain(PARSER_RESERVATION_TRACER.normalizedLiteral);
+ expectInitialEditorStability(
+ markdown,
+ expectedMarkdown,
+ PARSER_RESERVATION_TRACER.normalizedLiteral,
+ hrefs,
+ );
+ },
+ );
+
+ test.each(ADDITIONAL_MUTATING_PARSER_RESERVATION_CASES)(
+ 'title metadata neutralizes $reservation before its first editor cycle',
+ ({ literal, normalizedLiteral }) => {
+ const { markdown, hrefs } = LITERAL_SURFACE_CASES[0].render(literal);
+ const expectedMarkdown = LITERAL_SURFACE_CASES[0].expected(normalizedLiteral, literal);
+
+ expect(markdown).toContain(normalizedLiteral);
+ expectInitialEditorStability(markdown, expectedMarkdown, normalizedLiteral, hrefs);
+ },
+ );
+
+ test('title metadata preserves collision-defended parser reservations', () => {
+ const { markdown, hrefs } = LITERAL_SURFACE_CASES[0].render(
+ COLLISION_DEFENDED_PARSER_RESERVATION_CORPUS,
+ );
+ const expectedMarkdown = LITERAL_SURFACE_CASES[0].expected(
+ COLLISION_DEFENDED_PARSER_RESERVATION_CORPUS,
+ COLLISION_DEFENDED_PARSER_RESERVATION_CORPUS,
+ );
+
+ expectInitialEditorStability(
+ markdown,
+ expectedMarkdown,
+ COLLISION_DEFENDED_PARSER_RESERVATION_CORPUS,
+ hrefs,
+ );
+ });
+
+ test.each(LITERAL_SURFACE_CASES)(
+ '$surface replaces residual C0, DEL, and C1 controls after whitespace folding',
+ ({ render, expected }) => {
+ const { markdown, hrefs } = render(CONTROL_BEARING_LITERAL);
+ const expectedMarkdown = expected(NORMALIZED_CONTROL_LITERAL, CONTROL_BEARING_LITERAL);
+
+ expect.soft(hasResidualControl(markdown)).toBe(false);
+ expectInitialEditorStability(markdown, expectedMarkdown, NORMALIZED_CONTROL_LITERAL, hrefs);
+ },
+ );
+
+ test.each(LITERAL_SURFACE_CASES)(
+ '$surface currently preserves the bidi override pending policy',
+ ({ render, expected }) => {
+ const literal = `before${BIDI_OVERRIDE_PENDING_POLICY}after`;
+ const { markdown, hrefs } = render(literal);
+ const expectedMarkdown = expected(literal, literal);
+
+ expect(markdown).toContain(literal);
+ expectInitialEditorStability(markdown, expectedMarkdown, literal, hrefs);
+ },
+ );
+
+ test.each(LITERAL_SURFACE_CASES)(
+ '$surface preserves zero-width and joining controls',
+ ({ render, expected }) => {
+ const literal = `before${ZERO_WIDTH_AND_JOINING_CONTROLS}after`;
+ const { markdown, hrefs } = render(literal);
+ const expectedMarkdown = expected(literal, literal);
+
+ expect(markdown).toContain(literal);
+ expectInitialEditorStability(markdown, expectedMarkdown, literal, hrefs);
+ },
+ );
+
+ test('control normalization preserves NFC type grouping and first-cycle stability', () => {
+ const entries: IndexEntry[] = [
+ { path: 'guides/composed.md', title: 'Composed', type: 'Café\tGuide\u0000' },
+ { path: 'guides/decomposed.md', title: 'Decomposed', type: 'Cafe\u0301\nGuide\u009b' },
+ ];
+ const build = (orderedEntries: readonly IndexEntry[]) =>
+ buildIndexMarkdown(orderedEntries, {
+ warningScope: false,
+ isRoot: false,
+ directory: 'guides',
+ });
+ const forward = build(entries);
+ const reversed = build(entries.toReversed());
+ const expectedHrefs = ['./composed.md', './decomposed.md'];
+ const expectedMarkdown =
+ '# Index\n\n## Cafe\u0301 Guide�\n\n* [Composed](./composed.md)\n* [Decomposed](./decomposed.md)\n';
+
+ expect(forward).toBe(expectedMarkdown);
+ expect(reversed).toBe(expectedMarkdown);
+
+ for (const markdown of [forward, reversed]) {
+ const parsed = markdownManager.parse(markdown);
+ const sections = parsedSections(parsed);
+
+ expect(hasResidualControl(markdown)).toBe(false);
+ expect(markdownManager.serialize(parsed)).toBe(expectedMarkdown);
+ expect(sections).toHaveLength(2);
+ expect(sections[0]).toEqual({ level: 1, heading: 'Index', hrefs: [] });
+ expect(sections[1]?.level).toBe(2);
+ expect(sections[1]?.heading.normalize('NFC')).toBe('Café Guide�');
+ expect(sections[1]?.hrefs).toEqual(expectedHrefs);
+ }
+ });
+
+ test.each(FIELD_CASES)(
+ '$field metadata remains literal through the editor Markdown pipeline',
+ ({ entry }) => {
+ const markdown = buildIndexMarkdown([entry], {
+ warningScope: false,
+ isRoot: false,
+ directory: 'guides',
+ });
+
+ expectLiteralThroughEditor(markdown, MARKDOWN_METACHARACTER_CORPUS, ['./entry.md']);
+ },
+ );
+
+ test.each([
+ {
+ source: 'first H1',
+ title: extractPageTitle(`# ${MARKDOWN_METACHARACTER_CORPUS}\n`, 'fallback'),
+ literal: MARKDOWN_METACHARACTER_CORPUS,
+ },
+ {
+ source: 'filename fallback',
+ title: extractPageTitle('Plain body without a heading.\n', PORTABLE_PATH_CORPUS),
+ literal: PORTABLE_PATH_CORPUS,
+ },
+ ])('$source titles remain literal through the editor Markdown pipeline', ({ title, literal }) => {
+ const markdown = buildIndexMarkdown(
+ [{ path: 'guides/entry.md', title, description: 'Plain description', type: 'Guide' }],
+ { warningScope: false, isRoot: false, directory: 'guides' },
+ );
+
+ expectLiteralThroughEditor(markdown, literal, ['./entry.md']);
+ });
+
+ test.each([
+ { cycle: 'initial', currentMarkdown: null },
+ { cycle: 'incremental', currentMarkdown: '# Stale index\n' },
+ ])(
+ '$cycle planning keeps a derived subdirectory label literal and its target resolvable',
+ ({ currentMarkdown }) => {
+ const guides = planDirectoryIndexRegenerations({
+ warningScope: false,
+ docs: [[`guides/${PORTABLE_PATH_CORPUS}/entry`, { title: 'Entry', type: 'Guide' }]],
+ docExtension: () => '.md',
+ currentMarkdownFor: () => currentMarkdown,
+ }).find((decision) => decision.directory === 'guides');
+
+ expect(guides).toBeDefined();
+ expect(guides?.changed).toBe(true);
+ expectLiteralThroughEditor(guides?.markdown ?? '', PORTABLE_PATH_CORPUS, [
+ `./${encodeURIComponent(PORTABLE_PATH_CORPUS)}/index.md`,
+ ]);
+
+ const href = linkHrefs(markdownManager.parse(guides?.markdown ?? ''))[0] ?? '';
+ expect(resolveInternalHref(href, 'guides/index')?.docName).toBe(
+ `guides/${PORTABLE_PATH_CORPUS}/index`,
+ );
+ },
+ );
+
+ test.each([
+ { destination: 'disk', loaded: false },
+ { destination: 'loaded document', loaded: true },
+ ])('$destination artifacts preserve generated metadata semantics', async ({ loaded }) => {
+ const markdown = buildIndexMarkdown(
+ [
+ {
+ path: 'guides/entry.md',
+ title: extractedTitle,
+ description: 'Plain description',
+ type: 'Guide',
+ },
+ ],
+ { warningScope: false, isRoot: false, directory: 'guides' },
+ );
+ const document = loaded ? new Y.Doc() : undefined;
+ let diskMarkdown = '';
+ const outcome = await writeGeneratedArtifact(
+ {
+ docName: 'guides/index',
+ absPath: '/project/content/guides/index.md',
+ markdown,
+ currentMarkdown: null,
+ },
+ generatedArtifactEnv(document, (written) => {
+ diskMarkdown = written;
+ }),
+ );
+ const storedMarkdown = document?.getText('source').toString() ?? diskMarkdown;
+
+ expect(outcome).toBe(loaded ? 'document' : 'disk');
+ expect(storedMarkdown).toBe(markdown);
+ expectLiteralThroughEditor(storedMarkdown, MARKDOWN_METACHARACTER_CORPUS, ['./entry.md']);
+ });
+
+ test('generated headings, lists, link ordering, and directory-relative targets remain meaningful', () => {
+ const markdown = buildIndexMarkdown(
+ [
+ { path: 'guides/zeta.md', title: 'Zeta', description: 'Last', type: 'Guide' },
+ { path: 'guides/alpha.md', title: 'Alpha', description: 'First', type: 'Guide' },
+ ],
+ {
+ warningScope: false,
+ isRoot: false,
+ directory: 'guides',
+ subdirectories: [{ directory: 'guides/nested', title: 'Nested' }],
+ },
+ );
+ const document = markdownManager.parse(
+ markdownManager.serialize(markdownManager.parse(markdown)),
+ );
+ const topLevel = document.content ?? [];
+ const headings = topLevel
+ .filter((node) => node.type === 'heading')
+ .map((node) => ({ level: node.attrs?.level, text: textContent(node) }));
+ const links = walkJson(document)
+ .filter((node) => node.text !== undefined && node.marks?.some((mark) => mark.type === 'link'))
+ .map((node) => ({ href: linkHrefs(node)[0], text: node.text }));
+
+ expect(topLevel.map((node) => node.type)).toEqual([
+ 'heading',
+ 'heading',
+ 'list',
+ 'heading',
+ 'list',
+ ]);
+ expect(headings).toEqual([
+ { level: 1, text: 'Index' },
+ { level: 2, text: 'Guide' },
+ { level: 2, text: 'Subdirectories' },
+ ]);
+ expect(links).toEqual([
+ { href: './alpha.md', text: 'Alpha' },
+ { href: './zeta.md', text: 'Zeta' },
+ { href: './nested/index.md', text: 'Nested' },
+ ]);
+ });
+
+ test('literal type labels keep distinct buckets while exact owned labels retain their behavior', () => {
+ const entries: IndexEntry[] = [
+ { path: 'guides/exact-index.md', title: 'Exact index', type: 'Index' },
+ { path: 'guides/emphasis-index.md', title: 'Emphasis index', type: '*Index*' },
+ { path: 'guides/index-closer.md', title: 'Index closer', type: 'Index #' },
+ { path: 'guides/exact-other.md', title: 'Exact other', type: 'Other' },
+ { path: 'guides/untyped.md', title: 'Untyped' },
+ { path: 'guides/strong-other.md', title: 'Strong other', type: '**Other**' },
+ { path: 'guides/html-other.md', title: 'HTML other', type: '<b></b>Other' },
+ {
+ path: 'guides/exact-subdirectories.md',
+ title: 'Exact subdirectories',
+ type: 'Subdirectories',
+ },
+ {
+ path: 'guides/link-subdirectories.md',
+ title: 'Link subdirectories',
+ type: '[Subdirectories](./x)',
+ },
+ { path: 'guides/guide.md', title: 'Plain guide', type: 'Guide' },
+ { path: 'guides/emphasis-guide.md', title: 'Emphasis guide', type: '*Guide*' },
+ ];
+ const subdirectories = [{ directory: 'guides/nested', title: 'Nested' }];
+ const build = (orderedEntries: readonly IndexEntry[]) =>
+ buildIndexMarkdown(orderedEntries, {
+ warningScope: false,
+ isRoot: false,
+ directory: 'guides',
+ subdirectories,
+ });
+ const forward = build(entries);
+ const reversed = build(entries.toReversed());
+
+ expect(reversed).toBe(forward);
+ expectGroupingThroughEditor(
+ forward,
+ [
+ { level: 1, heading: 'Index', hrefs: ['./exact-index.md'] },
+ { level: 2, heading: '**Other**', hrefs: ['./strong-other.md'] },
+ { level: 2, heading: '*Guide*', hrefs: ['./emphasis-guide.md'] },
+ { level: 2, heading: '*Index*', hrefs: ['./emphasis-index.md'] },
+ { level: 2, heading: '<b></b>Other', hrefs: ['./html-other.md'] },
+ {
+ level: 2,
+ heading: '[Subdirectories](./x)',
+ hrefs: ['./link-subdirectories.md'],
+ },
+ { level: 2, heading: 'Guide', hrefs: ['./guide.md'] },
+ { level: 2, heading: 'Index #', hrefs: ['./index-closer.md'] },
+ { level: 2, heading: 'Other', hrefs: ['./exact-other.md', './untyped.md'] },
+ {
+ level: 2,
+ heading: 'Subdirectories',
+ hrefs: ['./exact-subdirectories.md', './nested/index.md'],
+ },
+ ],
+ [
+ './emphasis-guide.md',
+ './emphasis-index.md',
+ './exact-index.md',
+ './exact-other.md',
+ './exact-subdirectories.md',
+ './guide.md',
+ './html-other.md',
+ './index-closer.md',
+ './link-subdirectories.md',
+ './nested/index.md',
+ './strong-other.md',
+ './untyped.md',
+ ],
+ );
+ });
+
+ test('type bucket identity retains whitespace and NFC normalization independent of input order', () => {
+ const entries: IndexEntry[] = [
+ { path: 'guides/composed.md', title: 'Composed', type: 'Café Guide' },
+ { path: 'guides/decomposed.md', title: 'Decomposed', type: 'Café\nGuide' },
+ ];
+ const build = (orderedEntries: readonly IndexEntry[]) =>
+ buildIndexMarkdown(orderedEntries, {
+ warningScope: false,
+ isRoot: false,
+ directory: 'guides',
+ });
+ const forward = build(entries);
+ const reversed = build(entries.toReversed());
+ const expectedHrefs = ['./composed.md', './decomposed.md'];
+
+ expect(reversed).toBe(forward);
+
+ for (const markdown of [forward, reversed]) {
+ const parsed = markdownManager.parse(markdown);
+ const serializedOnce = markdownManager.serialize(parsed);
+ const reparsed = markdownManager.parse(serializedOnce);
+
+ for (const document of [parsed, reparsed]) {
+ const sections = parsedSections(document);
+ expect(sections).toHaveLength(2);
+ expect(sections[0]).toEqual({ level: 1, heading: 'Index', hrefs: [] });
+ expect(sections[1]?.level).toBe(2);
+ expect(sections[1]?.heading.normalize('NFC')).toBe('Café Guide');
+ expect(sections[1]?.hrefs).toEqual(expectedHrefs);
+ expect([...new Set(linkHrefs(document))].toSorted()).toEqual(expectedHrefs);
+ }
+
+ expect(markdownManager.serialize(reparsed)).toBe(serializedOnce);
+ }
+ });
+});
diff --git a/packages/server/src/content/generate-index.test.ts b/packages/server/src/content/generate-index.test.ts
index f76053052..cf377f393 100644
--- a/packages/server/src/content/generate-index.test.ts
+++ b/packages/server/src/content/generate-index.test.ts
@@ -14,7 +14,7 @@ describe('buildIndexMarkdown', () => {
entry({ path: 'notes/b.md', title: 'Beta', type: 'note' }),
entry({ path: 'concepts/c.md', title: 'Gamma', type: 'concept' }),
],
- { isRoot: false },
+ { warningScope: false, isRoot: false },
);
expect(out).toBe(
@@ -45,7 +45,7 @@ describe('buildIndexMarkdown', () => {
],
]) {
for (const isRoot of [true, false]) {
- const out = buildIndexMarkdown(entries, { isRoot });
+ const out = buildIndexMarkdown(entries, { warningScope: false, isRoot });
expect(out.match(/^# .+$/gm), `isRoot=${isRoot}, ${entries.length} entries`).toEqual([
'# Index',
]);
@@ -55,6 +55,7 @@ describe('buildIndexMarkdown', () => {
test('a document with no type lands in Other', () => {
const out = buildIndexMarkdown([entry({ path: 'scratch.md', title: 'Scratch' })], {
+ warningScope: false,
isRoot: false,
});
expect(out).toContain('## Other');
@@ -66,7 +67,7 @@ describe('buildIndexMarkdown', () => {
entry({ path: 'a.md', title: 'A', type: ' ' }),
entry({ path: 'b.md', title: 'B', type: '' }),
],
- { isRoot: false },
+ { warningScope: false, isRoot: false },
);
expect(out.match(/^## .+$/gm)).toEqual(['## Other']);
});
@@ -78,7 +79,7 @@ describe('buildIndexMarkdown', () => {
entry({ path: 'a.md', title: 'Apple', type: 'note' }),
entry({ path: 'm.md', title: 'mango', type: 'concept' }),
],
- { isRoot: false },
+ { warningScope: false, isRoot: false },
);
const headings = out.match(/^## .+$/gm);
@@ -92,7 +93,7 @@ describe('buildIndexMarkdown', () => {
entry({ path: 'a.md', title: 'A', type: 'note', description: 'has one' }),
entry({ path: 'b.md', title: 'B', type: 'note' }),
],
- { isRoot: false },
+ { warningScope: false, isRoot: false },
);
expect(out).toContain('* [A](./a.md) - has one');
@@ -100,10 +101,20 @@ describe('buildIndexMarkdown', () => {
expect(out).not.toContain('* [B](./b.md) -');
});
+ test('the description suffix is omitted entirely when the description is blank', () => {
+ const out = buildIndexMarkdown(
+ [entry({ path: 'a.md', title: 'A', type: 'note', description: ' \n ' })],
+ { warningScope: false, isRoot: false },
+ );
+
+ expect(out).toContain('* [A](./a.md)\n');
+ expect(out).not.toContain('* [A](./a.md) -');
+ });
+
test('a multi-line description collapses to one line so it cannot break the list item', () => {
const out = buildIndexMarkdown(
[entry({ path: 'a.md', title: 'A', type: 'note', description: 'first\nsecond third' })],
- { isRoot: false },
+ { warningScope: false, isRoot: false },
);
expect(out).toContain('* [A](./a.md) - first second third');
@@ -113,17 +124,17 @@ describe('buildIndexMarkdown', () => {
test('the root index carries a quoted okf_version and a non-root index carries no frontmatter', () => {
const entries = [entry({ path: 'a.md', title: 'A', type: 'note' })];
- const root = buildIndexMarkdown(entries, { isRoot: true });
+ const root = buildIndexMarkdown(entries, { warningScope: false, isRoot: true });
expect(root.startsWith('---\nokf_version: "0.2"\n---\n\n')).toBe(true);
- const nested = buildIndexMarkdown(entries, { isRoot: false });
+ const nested = buildIndexMarkdown(entries, { warningScope: false, isRoot: false });
expect(nested.startsWith('---')).toBe(false);
});
test('links are relative with the extension retained, never a bare folder', () => {
const out = buildIndexMarkdown(
[entry({ path: 'concepts/nested/deep.md', title: 'Deep', type: 'concept' })],
- { isRoot: false },
+ { warningScope: false, isRoot: false },
);
expect(out).toContain('](./concepts/nested/deep.md)');
@@ -131,13 +142,13 @@ describe('buildIndexMarkdown', () => {
});
test('an empty project yields the title alone rather than an invented section', () => {
- expect(buildIndexMarkdown([], { isRoot: true })).toBe(
+ expect(buildIndexMarkdown([], { warningScope: false, isRoot: true })).toBe(
'---\nokf_version: "0.2"\n---\n\n# Index\n',
);
- expect(buildIndexMarkdown([], { isRoot: false })).toBe('# Index\n');
+ expect(buildIndexMarkdown([], { warningScope: false, isRoot: false })).toBe('# Index\n');
});
- test('a bucket whose members spell the heading differently renders the same either way', () => {
+ test('literal headings and normalized equivalents render the same in any input order', () => {
const mixed = [
entry({ path: 'a.md', title: 'Alpha', type: 'Flow' }),
entry({ path: 'b.md', title: 'Beta', type: 'Flow #' }),
@@ -146,29 +157,39 @@ describe('buildIndexMarkdown', () => {
];
for (const isRoot of [true, false]) {
- const forward = buildIndexMarkdown(mixed, { isRoot });
- const reversed = buildIndexMarkdown(mixed.toReversed(), { isRoot });
+ const forward = buildIndexMarkdown(mixed, { warningScope: false, isRoot });
+ const reversed = buildIndexMarkdown(mixed.toReversed(), { warningScope: false, isRoot });
expect(reversed, `isRoot=${isRoot}`).toBe(forward);
+ expect(headingContents(forward).map((heading) => heading.normalize('NFC'))).toEqual([
+ 'Index',
+ 'Café',
+ 'Flow',
+ 'Flow \\#',
+ ]);
for (const href of ['](./a.md)', '](./b.md)', '](./c.md)', '](./d.md)']) {
expect(forward, `isRoot=${isRoot} ${href}`).toContain(href);
}
}
});
- test('a generator-owned heading keeps its own spelling against a colliding type', () => {
+ test('a generator-owned heading remains distinct from a punctuation-shaped literal type', () => {
const out = buildIndexMarkdown(
[entry({ path: 'a.md', title: 'Alpha', type: 'Subdirectories #' })],
- { isRoot: false, subdirectories: [{ path: 'nested/index.md', title: 'nested' }] },
+ {
+ warningScope: false,
+ isRoot: false,
+ subdirectories: [{ directory: 'nested', title: 'nested' }],
+ },
);
expect(out).toContain('## Subdirectories\n');
- expect(out).not.toContain('## Subdirectories #');
+ expect(out).toContain('## Subdirectories \\#\n');
expect(out).toContain('](./a.md)');
expect(out).toContain('](./nested/index.md)');
});
- test('every heading the generator owns keeps its own spelling against a colliding type', () => {
+ test('every heading the generator owns remains distinct from HTML-shaped literal types', () => {
const out = buildIndexMarkdown(
[
entry({ path: 'untyped.md', title: 'Untyped' }),
@@ -176,13 +197,19 @@ describe('buildIndexMarkdown', () => {
entry({ path: 'b.md', title: 'Beta', type: '<b></b>Subdirectories' }),
entry({ path: 'c.md', title: 'Gamma', type: '<b></b>Index' }),
],
- { isRoot: false, subdirectories: [{ path: 'nested/index.md', title: 'nested' }] },
+ {
+ warningScope: false,
+ isRoot: false,
+ subdirectories: [{ directory: 'nested', title: 'nested' }],
+ },
);
expect(out).toContain('# Index\n');
expect(out).toContain('## Other\n');
expect(out).toContain('## Subdirectories\n');
- expect(out).not.toContain('<b>');
+ expect(out).toContain('## \\<b\\>\\<\\/b\\>Other\n');
+ expect(out).toContain('## \\<b\\>\\<\\/b\\>Subdirectories\n');
+ expect(out).toContain('## \\<b\\>\\<\\/b\\>Index\n');
for (const href of ['](./untyped.md)', '](./a.md)', '](./b.md)', '](./c.md)']) {
expect(out, href).toContain(href);
@@ -190,7 +217,7 @@ describe('buildIndexMarkdown', () => {
});
test.each([...GENERATOR_OWNED_HEADINGS])(
- 'a colliding type cannot rename the generator-owned heading %j, reached from the entry loop alone',
+ 'an HTML-shaped literal type remains distinct from generator-owned heading %j',
(owned) => {
for (const order of [
[
@@ -203,10 +230,10 @@ describe('buildIndexMarkdown', () => {
],
]) {
const label = `${owned} order=${order.map((e) => e.type).join(',')}`;
- const out = buildIndexMarkdown(order, { isRoot: false });
+ const out = buildIndexMarkdown(order, { warningScope: false, isRoot: false });
expect(headingContents(out), label).toContain(owned);
- expect(out, label).not.toContain('<b>');
+ expect(headingContents(out), label).toContain(`\\<b\\>\\<\\/b\\>${owned}`);
expect(out, label).toContain('](./a.md)');
expect(out, label).toContain('](./b.md)');
}
@@ -219,7 +246,7 @@ describe('buildIndexMarkdown', () => {
entry({ path: 'readme.md', title: 'Overview', type: 'Index' }),
entry({ path: 'login-flow.md', title: 'Login flow', type: 'Flow' }),
],
- { isRoot: false },
+ { warningScope: false, isRoot: false },
);
expect(out).toBe(
@@ -234,8 +261,11 @@ describe('buildIndexMarkdown', () => {
entry({ path: 'c.md', title: 'Gamma', type: 'note' }),
];
- const forward = buildIndexMarkdown(entries, { isRoot: true });
- const reversed = buildIndexMarkdown([...entries].reverse(), { isRoot: true });
+ const forward = buildIndexMarkdown(entries, { warningScope: false, isRoot: true });
+ const reversed = buildIndexMarkdown([...entries].reverse(), {
+ warningScope: false,
+ isRoot: true,
+ });
expect(reversed).toBe(forward);
});
@@ -248,7 +278,7 @@ describe('buildIndexMarkdown', () => {
type: 'note',
}),
],
- { isRoot: false },
+ { warningScope: false, isRoot: false },
);
expect(out).toContain(
@@ -262,8 +292,11 @@ describe('buildIndexMarkdown', () => {
entry({ path: 'alpha.md', title: 'same', type: 'note' }),
];
- const forward = buildIndexMarkdown(entries, { isRoot: false });
- const reversed = buildIndexMarkdown([...entries].reverse(), { isRoot: false });
+ const forward = buildIndexMarkdown(entries, { warningScope: false, isRoot: false });
+ const reversed = buildIndexMarkdown([...entries].reverse(), {
+ warningScope: false,
+ isRoot: false,
+ });
expect(reversed).toBe(forward);
expect(forward.indexOf('./alpha.md')).toBeLessThan(forward.indexOf('./zeta.md'));
@@ -271,6 +304,7 @@ describe('buildIndexMarkdown', () => {
test('collapses multiline section names into one heading', () => {
const out = buildIndexMarkdown([entry({ path: 'a.md', title: 'A', type: 'project\n plan' })], {
+ warningScope: false,
isRoot: false,
});
@@ -284,9 +318,10 @@ describe('buildIndexMarkdown', () => {
entry({ path: 'concepts/aggregate.md', title: 'Aggregate', type: 'concept' }),
],
{
+ warningScope: false,
isRoot: false,
directory: 'concepts',
- subdirectories: [{ path: 'concepts/nested/index.md', title: 'nested' }],
+ subdirectories: [{ directory: 'concepts/nested', title: 'nested' }],
},
);
@@ -317,9 +352,10 @@ describe('buildIndexMarkdown', () => {
}),
],
{
+ warningScope: false,
isRoot: false,
directory: 'concepts',
- subdirectories: [{ path: 'concepts/nested/index.md', title: 'nested' }],
+ subdirectories: [{ directory: 'concepts/nested', title: 'nested' }],
},
);
@@ -339,9 +375,10 @@ describe('buildIndexMarkdown', () => {
test('a container directory with markdown only below it lists subdirectories and no type section', () => {
const out = buildIndexMarkdown([], {
+ warningScope: false,
isRoot: false,
directory: 'concepts',
- subdirectories: [{ path: 'concepts/nested/index.md', title: 'nested' }],
+ subdirectories: [{ directory: 'concepts/nested', title: 'nested' }],
});
expect(out).toBe(
@@ -360,11 +397,12 @@ describe('buildIndexMarkdown', () => {
entry({ path: 'concepts/c.md', title: 'C' }),
],
{
+ warningScope: false,
isRoot,
directory: 'concepts',
subdirectories: [
- { path: 'concepts/x/index.md', title: 'x' },
- { path: 'concepts/y/index.md', title: 'y' },
+ { directory: 'concepts/x', title: 'x' },
+ { directory: 'concepts/y', title: 'y' },
],
},
);
@@ -380,16 +418,18 @@ describe('buildIndexMarkdown', () => {
entry({ path: 'concepts/a.md', title: 'Alpha', type: 'concept' }),
];
const subdirectories = [
- { path: 'concepts/zeta/index.md', title: 'zeta' },
- { path: 'concepts/alpha/index.md', title: 'alpha' },
+ { directory: 'concepts/zeta', title: 'zeta' },
+ { directory: 'concepts/alpha', title: 'alpha' },
];
const forward = buildIndexMarkdown(entries, {
+ warningScope: false,
isRoot: false,
directory: 'concepts',
subdirectories,
});
const reversed = buildIndexMarkdown([...entries].reverse(), {
+ warningScope: false,
isRoot: false,
directory: 'concepts',
subdirectories: [...subdirectories].reverse(),
@@ -408,18 +448,20 @@ describe('buildIndexMarkdown', () => {
entry({ path: 'concepts/aggregate.md', title: 'Aggregate', type: 'concept' }),
];
const subdirectories = [
- { path: 'concepts/zeta/index.md', title: 'zeta' },
- { path: 'concepts/alpha/index.md', title: 'alpha' },
+ { directory: 'concepts/zeta', title: 'zeta' },
+ { directory: 'concepts/alpha', title: 'alpha' },
];
for (const isRoot of [true, false]) {
const label = `isRoot=${isRoot}`;
const forward = buildIndexMarkdown(entries, {
+ warningScope: false,
isRoot,
directory: 'concepts',
subdirectories,
});
const reversed = buildIndexMarkdown([...entries].reverse(), {
+ warningScope: false,
isRoot,
directory: 'concepts',
subdirectories: [...subdirectories].reverse(),
@@ -450,7 +492,7 @@ describe('buildIndexMarkdown', () => {
];
const out = buildIndexMarkdown(
names.map((name) => entry({ path: `blogs/drafts/${name}.md`, title: name, type: 'note' })),
- { isRoot: false, directory: 'blogs/drafts' },
+ { warningScope: false, isRoot: false, directory: 'blogs/drafts' },
);
const hrefs = [...out.matchAll(/\]\(([^)]+)\)/g)].map((match) => match[1] ?? '');
@@ -474,7 +516,11 @@ describe('buildIndexMarkdown', () => {
entry({ path: 'readme.md', title: 'Readme', type }),
entry({ path: 'login-flow.md', title: 'Login flow', type: 'Flow' }),
],
- { isRoot, subdirectories: [{ path: 'nested/index.md', title: 'nested' }] },
+ {
+ warningScope: false,
+ isRoot,
+ subdirectories: [{ directory: 'nested', title: 'nested' }],
+ },
);
expect(out, label).toContain('](./readme.md)');
@@ -495,11 +541,12 @@ function headingContents(markdown: string): string[] {
function generatorOwnedHeadings(): string[] {
const probes = [false, true].flatMap((isRoot) => [
- buildIndexMarkdown([], { isRoot }),
- buildIndexMarkdown([entry({ path: 'a.md', title: 'A' })], { isRoot }),
+ buildIndexMarkdown([], { warningScope: false, isRoot }),
+ buildIndexMarkdown([entry({ path: 'a.md', title: 'A' })], { warningScope: false, isRoot }),
buildIndexMarkdown([], {
+ warningScope: false,
isRoot,
- subdirectories: [{ path: 'nested/index.md', title: 'nested' }],
+ subdirectories: [{ directory: 'nested', title: 'nested' }],
}),
]);
diff --git a/packages/server/src/content/generate-index.ts b/packages/server/src/content/generate-index.ts
index 38a1e5870..dcb4dcbba 100644
--- a/packages/server/src/content/generate-index.ts
+++ b/packages/server/src/content/generate-index.ts
@@ -1,4 +1,5 @@
-import { encodeHrefPath, headingContentIdentity } from '@inkeep/open-knowledge-core';
+import { encodeHrefPath, isMutatingParserReservation } from '@inkeep/open-knowledge-core';
+import { getLogger } from '../logger.ts';
export interface IndexEntry {
path: string;
@@ -8,7 +9,7 @@ export interface IndexEntry {
}
export interface SubdirectoryEntry {
- path: string;
+ directory: string;
title: string;
}
@@ -16,6 +17,7 @@ export interface BuildIndexOptions {
isRoot: boolean;
directory?: string | undefined;
subdirectories?: readonly SubdirectoryEntry[] | undefined;
+ warningScope: GeneratedIndexWarningScope | false;
}
const GENERATED_OKF_VERSION = '0.2';
@@ -34,13 +36,135 @@ export const GENERATOR_OWNED_HEADINGS: ReadonlySet<string> = new Set([
interface RenderableLink {
path: string;
- title: string;
- description?: string | undefined;
+ title: LiteralMarkdownText;
+ description?: LiteralMarkdownText | undefined;
}
-function sectionOf(entry: IndexEntry): string {
- const declared = entry.type === undefined ? '' : toSingleLine(entry.type);
- return declared ? declared : UNTYPED_SECTION;
+interface LiteralMarkdownText {
+ visible: string;
+ identity: string;
+ source: string;
+}
+
+interface LiteralMetadataSource {
+ path: string;
+ field: 'title' | 'description' | 'type' | 'folder-label';
+}
+
+type LiteralMarkdownTextOrigin =
+ | {
+ kind: 'attributed';
+ source: LiteralMetadataSource;
+ warningScope: GeneratedIndexWarningScope | false;
+ }
+ | { kind: 'generator-owned' };
+
+const SUBSTITUTION_WARNING_MESSAGES = {
+ control: 'generated index metadata contained a Cc control; replaced it with U+FFFD',
+ 'parser-reservation':
+ 'generated index metadata contained a parser-reserved private-use code point; replaced it with U+FFFD',
+} as const;
+
+type SubstitutionWarningKind = keyof typeof SUBSTITUTION_WARNING_MESSAGES;
+
+function metadataSourceKey(source: LiteralMetadataSource): string {
+ return `${source.path}\0${source.field}`;
+}
+
+export class GeneratedIndexWarningScope {
+ readonly contentDir: string;
+ readonly #warnedEpisodes = new Map<string, string>();
+
+ constructor(contentDir: string) {
+ this.contentDir = contentDir;
+ }
+
+ update(kind: SubstitutionWarningKind, source: LiteralMetadataSource, substituted: boolean): void {
+ const sourceKey = metadataSourceKey(source);
+ const episodeKey = `${kind}\0${sourceKey}`;
+ if (!substituted) {
+ this.#warnedEpisodes.delete(episodeKey);
+ return;
+ }
+ if (this.#warnedEpisodes.has(episodeKey)) return;
+ this.#warnedEpisodes.set(episodeKey, sourceKey);
+ getLogger('generated-index').warn(
+ { contentDir: this.contentDir, ...source, kind },
+ SUBSTITUTION_WARNING_MESSAGES[kind],
+ );
+ }
+
+ retain(activeSources: ReadonlySet<string>): void {
+ for (const [episodeKey, sourceKey] of this.#warnedEpisodes) {
+ if (!activeSources.has(sourceKey)) this.#warnedEpisodes.delete(episodeKey);
+ }
+ }
+}
+
+export function createGeneratedIndexWarningScope(contentDir: string): GeneratedIndexWarningScope {
+ return new GeneratedIndexWarningScope(contentDir);
+}
+
+export function retainGeneratedIndexSubstitutionWarningSources(
+ warningScope: GeneratedIndexWarningScope | false,
+ entries: readonly IndexEntry[],
+ subdirectories: readonly SubdirectoryEntry[],
+): void {
+ if (!warningScope) return;
+ const activeSources = new Set<string>();
+ for (const entry of entries) {
+ activeSources.add(metadataSourceKey({ path: entry.path, field: 'title' }));
+ activeSources.add(metadataSourceKey({ path: entry.path, field: 'type' }));
+ if (entry.description !== undefined) {
+ activeSources.add(metadataSourceKey({ path: entry.path, field: 'description' }));
+ }
+ }
+ for (const subdirectory of subdirectories) {
+ activeSources.add(metadataSourceKey({ path: subdirectory.directory, field: 'folder-label' }));
+ }
+ warningScope.retain(activeSources);
+}
+
+function literalMarkdownText(
+ value: string,
+ origin: LiteralMarkdownTextOrigin,
+): LiteralMarkdownText {
+ let controlSubstituted = false;
+ let parserReservationSubstituted = false;
+ const visible = Array.from(toSingleLine(value), (character) => {
+ if (/\p{Cc}/u.test(character)) {
+ controlSubstituted = true;
+ return '\uFFFD';
+ }
+ if (isMutatingParserReservation(character)) {
+ parserReservationSubstituted = true;
+ return '\uFFFD';
+ }
+ return character;
+ }).join('');
+ if (origin.kind === 'attributed' && origin.warningScope) {
+ origin.warningScope.update('control', origin.source, controlSubstituted);
+ origin.warningScope.update('parser-reservation', origin.source, parserReservationSubstituted);
+ }
+ return {
+ visible,
+ identity: visible.normalize('NFC'),
+ source: visible.replace(/[\u0021-\u002f\u003a-\u0040\u005b-\u0060\u007b-\u007e]/g, '\\$&'),
+ };
+}
+
+function sectionOf(
+ entry: IndexEntry,
+ warningScope: GeneratedIndexWarningScope | false,
+): LiteralMarkdownText {
+ const declared = literalMarkdownText(entry.type ?? '', {
+ kind: 'attributed',
+ source: { path: entry.path, field: 'type' },
+ warningScope,
+ });
+ return declared.visible
+ ? declared
+ : literalMarkdownText(UNTYPED_SECTION, { kind: 'generator-owned' });
}
function toSingleLine(value: string): string {
@@ -70,7 +194,10 @@ function prefersSpelling(candidate: string, current: string): boolean {
function compareLinks(left: RenderableLink, right: RenderableLink): number {
return (
- compareCodePoints(normalizedSortKey(left.title), normalizedSortKey(right.title)) ||
+ compareCodePoints(
+ normalizedSortKey(left.title.visible),
+ normalizedSortKey(right.title.visible),
+ ) ||
compareCodePoints(
left.path.replaceAll('\\', '/').normalize('NFC'),
right.path.replaceAll('\\', '/').normalize('NFC'),
@@ -78,8 +205,30 @@ function compareLinks(left: RenderableLink, right: RenderableLink): number {
);
}
-function escapeLinkLabel(value: string): string {
- return toSingleLine(value).replace(/[\\[\]]/g, '\\$&');
+function renderableLink(
+ entry: IndexEntry | SubdirectoryEntry,
+ warningScope: GeneratedIndexWarningScope | false,
+): RenderableLink {
+ const titleSource: LiteralMetadataSource =
+ 'directory' in entry
+ ? { path: entry.directory, field: 'folder-label' }
+ : { path: entry.path, field: 'title' };
+ return {
+ path: 'directory' in entry ? `${entry.directory}/index.md` : entry.path,
+ title: literalMarkdownText(entry.title, {
+ kind: 'attributed',
+ source: titleSource,
+ warningScope,
+ }),
+ description:
+ 'description' in entry && entry.description !== undefined
+ ? literalMarkdownText(entry.description, {
+ kind: 'attributed',
+ source: { path: entry.path, field: 'description' },
+ warningScope,
+ })
+ : undefined,
+ };
}
function relativeTo(directory: string, path: string): string {
@@ -100,9 +249,8 @@ function toHref(relativePath: string): string {
}
function renderLink(link: RenderableLink, directory: string): string {
- const anchor = `* [${escapeLinkLabel(link.title)}](${toHref(relativeTo(directory, link.path))})`;
- const description = link.description === undefined ? '' : toSingleLine(link.description);
- return description ? `${anchor} - ${description}` : anchor;
+ const anchor = `* [${link.title.source}](${toHref(relativeTo(directory, link.path))})`;
+ return link.description?.source ? `${anchor} - ${link.description.source}` : anchor;
}
function renderBody(links: readonly RenderableLink[], directory: string): string {
@@ -119,19 +267,20 @@ export function buildIndexMarkdown(
): string {
const directory = options.directory ?? '';
const subdirectories = options.subdirectories ?? [];
+ const warningScope = options.warningScope;
interface Bucket {
- heading: string;
+ heading: LiteralMarkdownText;
pinned: boolean;
links: RenderableLink[];
}
- const titleKey = headingContentIdentity(INDEX_TITLE);
+ const titleKey = literalMarkdownText(INDEX_TITLE, { kind: 'generator-owned' }).identity;
const grouped = new Map<string, Bucket>();
- const bucketFor = (heading: string): RenderableLink[] => {
- const key = headingContentIdentity(heading);
- const pinned = GENERATOR_OWNED_HEADINGS.has(heading);
+ const bucketFor = (heading: LiteralMarkdownText): RenderableLink[] => {
+ const key = heading.identity;
+ const pinned = GENERATOR_OWNED_HEADINGS.has(heading.visible);
const existing = grouped.get(key);
if (!existing) {
const created: Bucket = { heading, pinned, links: [] };
@@ -141,24 +290,30 @@ export function buildIndexMarkdown(
if (pinned && !existing.pinned) {
existing.heading = heading;
existing.pinned = true;
- } else if (!pinned && !existing.pinned && prefersSpelling(heading, existing.heading)) {
+ } else if (
+ !pinned &&
+ !existing.pinned &&
+ prefersSpelling(heading.visible, existing.heading.visible)
+ ) {
existing.heading = heading;
}
return existing.links;
};
for (const entry of entries) {
- bucketFor(sectionOf(entry)).push(entry);
+ bucketFor(sectionOf(entry, warningScope)).push(renderableLink(entry, warningScope));
}
if (subdirectories.length > 0) {
- bucketFor(SUBDIRECTORY_SECTION).push(...subdirectories);
+ bucketFor(literalMarkdownText(SUBDIRECTORY_SECTION, { kind: 'generator-owned' })).push(
+ ...subdirectories.map((entry) => renderableLink(entry, warningScope)),
+ );
}
const blocks = [...grouped]
.filter(([key]) => key !== titleKey)
- .sort(([, left], [, right]) => compareSections(left.heading, right.heading))
- .map(([, { heading, links }]) => `## ${heading}\n\n${renderBody(links, directory)}`);
+ .sort(([, left], [, right]) => compareSections(left.heading.visible, right.heading.visible))
+ .map(([, { heading, links }]) => `## ${heading.source}\n\n${renderBody(links, directory)}`);
const header = options.isRoot ? `---\nokf_version: "${GENERATED_OKF_VERSION}"\n---\n\n` : '';
diff --git a/packages/server/src/content/regenerate-index.test.ts b/packages/server/src/content/regenerate-index.test.ts
index f184a38b0..7b84d3453 100644
--- a/packages/server/src/content/regenerate-index.test.ts
+++ b/packages/server/src/content/regenerate-index.test.ts
@@ -61,7 +61,12 @@ describe('planDirectoryIndexRegenerations', () => {
docs: Record<string, IndexSourceDoc>,
currentMarkdownFor: (directory: string) => string | null = () => null,
): DirectoryIndexDeps {
- return { docs: Object.entries(docs), docExtension: () => '.md', currentMarkdownFor };
+ return {
+ docs: Object.entries(docs),
+ docExtension: () => '.md',
+ currentMarkdownFor,
+ warningScope: false,
+ };
}
const TREE: Record<string, IndexSourceDoc> = {
@@ -168,6 +173,7 @@ describe('planDirectoryIndexRegenerations', () => {
}
const decisions = planDirectoryIndexRegenerations({
+ warningScope: false,
docs: once(),
docExtension: () => '.md',
currentMarkdownFor: () => null,
diff --git a/packages/server/src/content/regenerate-index.ts b/packages/server/src/content/regenerate-index.ts
index 4d2c513b1..398c7ac89 100644
--- a/packages/server/src/content/regenerate-index.ts
+++ b/packages/server/src/content/regenerate-index.ts
@@ -1,5 +1,11 @@
import { extractPageDescription, extractPageTitle, extractPageType } from '../page-identity.ts';
-import { buildIndexMarkdown, type IndexEntry, type SubdirectoryEntry } from './generate-index.ts';
+import {
+ buildIndexMarkdown,
+ type GeneratedIndexWarningScope,
+ type IndexEntry,
+ retainGeneratedIndexSubstitutionWarningSources,
+ type SubdirectoryEntry,
+} from './generate-index.ts';
export const ROOT_INDEX_DOC_NAME = 'index';
@@ -21,6 +27,7 @@ export interface DirectoryIndexDeps {
docs: Iterable<readonly [string, IndexSourceDoc]>;
docExtension: (docName: string) => string;
currentMarkdownFor: (directory: string) => string | null;
+ warningScope: GeneratedIndexWarningScope | false;
}
export interface DirectoryIndexDecision {
@@ -120,19 +127,33 @@ export function planDirectoryIndexRegenerations(
else childrenByParent.set(parent, [directory]);
}
+ const renderedEntries: IndexEntry[] = [];
+ const renderedSubdirectories: SubdirectoryEntry[] = [];
const decisions = [...directories].map((directory): DirectoryIndexDecision => {
const entries = entriesByDirectory.get(directory) ?? [];
const subdirectories: SubdirectoryEntry[] = (childrenByParent.get(directory) ?? []).map(
- (child) => ({ path: `${child}/${ROOT_INDEX_DOC_NAME}.md`, title: basenameOf(child) }),
+ (child) => ({
+ directory: child,
+ title: basenameOf(child),
+ }),
);
+ renderedEntries.push(...entries);
+ renderedSubdirectories.push(...subdirectories);
const markdown = buildIndexMarkdown(entries, {
isRoot: directory === '',
directory,
subdirectories,
+ warningScope: deps.warningScope,
});
return { directory, changed: deps.currentMarkdownFor(directory) !== markdown, markdown };
});
+ retainGeneratedIndexSubstitutionWarningSources(
+ deps.warningScope,
+ renderedEntries,
+ renderedSubdirectories,
+ );
+
return decisions.sort(
(left, right) =>
depthOf(right.directory) - depthOf(left.directory) ||
diff --git a/packages/server/src/server-factory.test.ts b/packages/server/src/server-factory.test.ts
index 031ff23e1..60367ca3d 100644
--- a/packages/server/src/server-factory.test.ts
+++ b/packages/server/src/server-factory.test.ts
@@ -4212,6 +4212,21 @@ describe('createServer() — generated index wiring', () => {
expect(readIndex()).toContain('okf_version: "0.2"');
});
+ test('generated metadata warnings identify the production content directory', async () => {
+ const logs = captureAllLoggers();
+ writeDoc('warning.md', 'Parser\ue102warning', 'note');
+
+ await bootServer();
+
+ expect(logs.getLoggerEntries('generated-index')).toEqual([
+ {
+ level: 'warn',
+ msg: 'generated index metadata contained a parser-reserved private-use code point; replaced it with U+FFFD',
+ payload: { contentDir, path: 'warning.md', field: 'title', kind: 'parser-reservation' },
+ },
+ ]);
+ });
+
test('server readiness settles before the boot index sweep starts planning', async () => {
writeDoc('existing.md', 'Existing at boot', 'note');
@@ -4639,11 +4654,11 @@ describe('createServer() — generated index wiring', () => {
await bootServer();
writeDoc('only.md', 'Only', 'solo-type');
writeDoc('keep.md', 'Keep', 'note');
- await waitForIndex((md) => md.includes('## solo-type'));
+ await waitForIndex((md) => md.includes('## solo\\-type'));
unlinkSync(join(contentDir, 'only.md'));
- await waitForIndex((md) => !md.includes('## solo-type'));
+ await waitForIndex((md) => !md.includes('## solo\\-type'));
expect(readIndex()).toContain('## note');
});
diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.ts
index ee726caec..b80586525 100644
--- a/packages/server/src/server-factory.ts
+++ b/packages/server/src/server-factory.ts
@@ -85,6 +85,7 @@ import {
} from './config-file-watcher.ts';
import { applyExternalConfigChange, isConfigEcho } from './config-persistence.ts';
import { bindConflictAuthority, ConflictAuthority } from './conflict-authority.ts';
+import { createGeneratedIndexWarningScope } from './content/generate-index.ts';
import { type GeneratedArtifactEnv, writeGeneratedArtifact } from './content/generated-artifact.ts';
import {
type GeneratedIndexGitAttributesStatus,
@@ -483,6 +484,7 @@ export function createServer(options: ServerOptions): ServerInstance {
} = options;
const log = getLogger('server');
+ const generatedIndexWarningScope = createGeneratedIndexWarningScope(resolve(contentDir));
let cc1Broadcaster: CC1Broadcaster | null = null;
const initialBranch = readProjectHeadState(projectDir).branch ?? 'main';
const durabilityState = new DocumentDurabilityState(initialBranch, {
@@ -1408,6 +1410,7 @@ export function createServer(options: ServerOptions): ServerInstance {
docs: fileIndex,
docExtension: getDocExtension,
currentMarkdownFor: () => null,
+ warningScope: generatedIndexWarningScope,
});
const gitAttributes = inspectGeneratedIndexGitAttributes({
projectDir,
diff --git a/packages/server/src/validation-coverage-contract.test.ts b/packages/server/src/validation-coverage-contract.test.ts
index adf04574f..5bd7a50b0 100644
--- a/packages/server/src/validation-coverage-contract.test.ts
+++ b/packages/server/src/validation-coverage-contract.test.ts
@@ -1,6 +1,10 @@
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
-import { LINT_PLUGIN_IDS, VALIDATION_SOURCES } from '@inkeep/open-knowledge-core';
+import {
+ isMutatingParserReservation,
+ LINT_PLUGIN_IDS,
+ VALIDATION_SOURCES,
+} from '@inkeep/open-knowledge-core';
import { describe, expect, test } from 'vitest';
import { formatValidatorDegradationWarning } from './lint/validation-audit.ts';
import { DESCRIPTION as AUDIT_DESCRIPTION, AUDIT_WARNINGS_DESCRIPTION } from './mcp/tools/audit.ts';
@@ -18,6 +22,30 @@ const MCP_REFERENCE = readFileSync(
join(import.meta.dir, '../../../docs/content/reference/mcp.mdx'),
'utf8',
);
+const OKF_PLUGIN = readFileSync(
+ join(import.meta.dir, '../../../docs/content/plugins/okf.mdx'),
+ 'utf8',
+);
+
+function formatCodepoint(codepoint: number): string {
+ return `U+${codepoint.toString(16).toUpperCase().padStart(4, '0')}`;
+}
+
+function documentedParserReservations(): string[] {
+ const clause = /Parser-reserved Private Use Area code points (.+?) are also replaced/.exec(
+ OKF_PLUGIN,
+ )?.[1];
+ if (clause === undefined) throw new Error('OKF parser-reservation disclosure is missing');
+ return [...clause.matchAll(/U\+([0-9A-F]{4,6})(?:–U\+([0-9A-F]{4,6}))?/g)].flatMap(
+ ([, startHex, endHex]) => {
+ const start = Number.parseInt(startHex ?? '', 16);
+ const end = Number.parseInt(endHex ?? startHex ?? '', 16);
+ return Array.from({ length: end - start + 1 }, (_, offset) =>
+ formatCodepoint(start + offset),
+ );
+ },
+ );
+}
function mcpReferenceRow(tool: string): string {
const row = MCP_REFERENCE.split('\n').find((line) => line.startsWith(`| \`${tool}\` |`));
@@ -26,6 +54,13 @@ function mcpReferenceRow(tool: string): string {
}
describe('validation coverage source contract', () => {
+ test('OKF documents every parser reservation neutralized by generated indexes', () => {
+ const executable = Array.from({ length: 0xf8ff - 0xe000 + 1 }, (_, offset) => 0xe000 + offset)
+ .filter((codepoint) => isMutatingParserReservation(String.fromCodePoint(codepoint)))
+ .map(formatCodepoint);
+ expect(documentedParserReservations()).toEqual(executable);
+ });
+
test('tool descriptions name every source family they can select', () => {
for (const source of LINT_PLUGIN_IDS) expect(LINT_DESCRIPTION).toContain(`\`${source}\``);
for (const source of VALIDATION_SOURCES) expect(AUDIT_DESCRIPTION).toContain(`\`${source}\``);