Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): anchor comments across decoded character refs (PRD-8182) (#3708)

shagun-singh-inkeep2cff3419c381
diff --git a/.changeset/comment-anchor-decoded-char-refs.md b/.changeset/comment-anchor-decoded-char-refs.md
new file mode 100644
index 000000000..c025785a4
--- /dev/null
+++ b/.changeset/comment-anchor-decoded-char-refs.md
@@ -0,0 +1,9 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Fix comments being refused on any passage containing a boundary-whitespace character reference
+
+Selecting text and adding a comment failed with "The quoted passage is not in the document" whenever the selection crossed a space typed just inside bold, italic, or strikethrough, or a space-indented line. The passage matcher treated markdown syntax as elastic but did not know that the display pipeline decodes a narrow set of numeric character references — ` `, `	`, ` ` — which the byte-fidelity serializer mints to hold a phrasing-boundary space across re-parse. Six bytes on disk, one character on screen, and the match broke at the ampersand.
+
+Anchoring now decodes those references on whichever side carries them, in both directions, and context scoring makes the same allowance so a highlight no longer degrades near one. References the pipeline does not decode (`&`, ` `, `…`) still match literally, as they always did.
diff --git a/packages/app/src/comments/anchor-search.test.ts b/packages/app/src/comments/anchor-search.test.ts
index f5ea8538e..523fd5e06 100644
--- a/packages/app/src/comments/anchor-search.test.ts
+++ b/packages/app/src/comments/anchor-search.test.ts
@@ -449,3 +449,46 @@ describe('captureSelectionContext across an inline atom', () => {
expect(createAnchorResolver(doc)('done', context)?.from).toBe(target);
});
});
+
+/**
+ * A stored quote carrying a decoded character reference.
+ *
+ * The anchor's `exact` is sliced out of the markdown BODY, so it arrives with
+ * whatever the body spends on formatting — and with the boundary-whitespace
+ * char-refs the byte-fidelity serializer mints to hold a space just inside
+ * emphasis. The editor searches the text a reader can SEE, where those six bytes
+ * are one space. Server-side the passage resolves; here the highlight vanished.
+ */
+describe('findRangeInIndex — a stored quote carrying a boundary-whitespace char-ref', () => {
+ const RENDERED = 'Intro line. External apps [external action icon] follows.';
+ const STORED = 'External apps  ***[external action icon]';
+
+ test('resolves the passage the reader can see, on the quote alone', () => {
+ // No context, so the bracket recovery that would otherwise paper over this
+ // has nothing to work from — the quote itself has to match.
+ const range = findRangeInIndex(indexOf(RENDERED), STORED);
+ expect(range).not.toBeNull();
+ expect(RENDERED.slice((range?.from ?? 1) - 1, (range?.to ?? 1) - 1)).toBe(
+ 'External apps [external action icon]',
+ );
+ });
+
+ test('resolves it on the quote, not by recovering it from its brackets', () => {
+ // With context present the bracket recovery can reach the same span, but it
+ // reports a REWRITTEN anchor — the caller re-captures `exact` from it, so a
+ // passage nobody edited would churn its stored bytes on every re-find.
+ const range = findRangeInIndex(indexOf(RENDERED), STORED, {
+ prefix: 'Intro line. ',
+ suffix: ' follows.',
+ });
+ expect(range).not.toBeNull();
+ expect(RENDERED.slice((range?.from ?? 1) - 1, (range?.to ?? 1) - 1)).toBe(
+ 'External apps [external action icon]',
+ );
+ });
+
+ test('does not let the ref drag the match across neighbouring words', () => {
+ const rendered = 'alpha omega';
+ expect(findRangeInIndex(indexOf(rendered), 'alpha & omega')).toBeNull();
+ });
+});
diff --git a/packages/core/src/comments/leaf-text.test.ts b/packages/core/src/comments/leaf-text.test.ts
index 51afe4b6f..43f6c8a29 100644
--- a/packages/core/src/comments/leaf-text.test.ts
+++ b/packages/core/src/comments/leaf-text.test.ts
@@ -6,7 +6,7 @@
* hole in it (so a comment on the surrounding prose failed to anchor).
*/
-import { getSchema } from '@tiptap/core';
+import { getSchema, type JSONContent } from '@tiptap/core';
import { describe, expect, test } from 'vitest';
import { sharedExtensions } from '../extensions/shared.ts';
import { MarkdownManager } from '../markdown/index.ts';
@@ -128,6 +128,13 @@ describe('the captured quote anchors back to the body', () => {
'Before.\n\n```mermaid\ngraph TD;\n```\n\nAfter.',
'Before.\n\n$$\nx = 1\n$$\n\nAfter.',
'See [[page]] and #tag with ==mark== plus $$y$$ and ![a](i.png).',
+ // Boundary-whitespace char-refs. Nobody types these: the byte-fidelity
+ // serializer mints them to hold a phrasing-boundary space across re-parse,
+ // so an ordinary keystroke inside emphasis puts one in the body.
+ 'A   word.',
+ '**bold **tail',
+ '~~***External apps  ***[external action icon]~~',
+ '  indented line',
];
for (const md of CASES) {
@@ -135,6 +142,27 @@ describe('the captured quote anchors back to the body', () => {
expect(locate(md)).not.toBeNull();
});
}
+
+ test('the boundary-space fixtures above are what the serializer actually mints', () => {
+ // Keeps those cases honest: they are only worth pinning because a round trip
+ // through this pipeline produces them from an ordinary space.
+ const withBoundarySpace = {
+ type: 'doc',
+ content: [
+ {
+ type: 'paragraph',
+ content: [
+ { type: 'text', marks: [{ type: 'strong' }], text: 'bold ' },
+ { type: 'text', text: 'tail' },
+ ],
+ },
+ { type: 'paragraph', content: [{ type: 'text', text: ' indented line' }] },
+ ],
+ };
+ expect(mdManager.serialize(withBoundarySpace satisfies JSONContent)).toBe(
+ '**bold **tail\n\n  indented line\n',
+ );
+ });
});
describe('inlineOnly — for callers deciding whether text formatting applies', () => {
diff --git a/packages/core/src/comments/passage-match.test.ts b/packages/core/src/comments/passage-match.test.ts
index a6fed98c5..c0e7f9275 100644
--- a/packages/core/src/comments/passage-match.test.ts
+++ b/packages/core/src/comments/passage-match.test.ts
@@ -7,7 +7,14 @@
*/
import { describe, expect, test } from 'vitest';
-import { findAllPassages, findPassage, rewriteCeiling } from './passage-match.ts';
+import { isInlineWhitespaceNumericCharRef } from '../markdown/whitespace-char-ref.ts';
+import {
+ contextEvidenceFloor,
+ contextMatchScore,
+ findAllPassages,
+ findPassage,
+ rewriteCeiling,
+} from './passage-match.ts';
const BODY = `## Ingredients
@@ -526,3 +533,191 @@ describe('a needle ending in whitespace, at the end of the haystack', () => {
expect(findAllPassages(rendered, `${prefix}crema\n.`, { syntaxIn: 'needle' })).toEqual([]);
});
});
+
+/**
+ * Character references the display pipeline DECODES.
+ *
+ * The matcher treats markdown's own syntax as elastic, but markdown syntax is
+ * not the only place the body and the screen legally disagree. The byte-fidelity
+ * serializer mints a bare numeric char-ref to hold a phrasing-boundary space or
+ * tab across re-parse (CommonMark §6.4) — a space just inside `**…**` would
+ * otherwise dissolve the emphasis — and the display decode turns it back into
+ * the one character the reader sees. Six bytes on disk, one on screen.
+ *
+ * Nothing taught the matcher that, so a quote crossing one cannot find its passage
+ * in the body and the anchor is refused. It reaches ordinary documents because
+ * nobody types the entity: typing a space just inside bold, or indenting a
+ * paragraph, is enough to mint one.
+ *
+ * The decode set is deliberately narrow — `whitespace-char-ref.ts` owns it, and
+ * only SPACE, TAB, and NBSP are in it. Everything else (` `, `&`,
+ * `—`) survives to the screen as its own literal bytes and already
+ * matches; making those elastic would let a passage cross an ampersand or an em
+ * dash the quote never contained.
+ */
+describe('numeric character references the display pipeline decodes', () => {
+ /** The slice of `body` a caller's rendered quote resolves to. */
+ function locate(body: string, quote: string): string | null {
+ const hit = findPassage(body, quote, { syntaxIn: 'haystack' });
+ return hit ? body.slice(hit.start, hit.end) : null;
+ }
+
+ /**
+ * The fixtures below name refs on both sides of the decode boundary. This
+ * pins them to the predicate that actually draws it, so widening the decode
+ * set fails here rather than silently leaving the matcher a construct behind
+ * — the exact class of drift a decode-set widening would silently introduce.
+ */
+ /** U+00A0, spelled out — a literal one in a string is invisible to a reader. */
+ const NBSP = '\u00A0';
+
+ const DECODED = [' ', ' ', ' ', ' ', '	', ' ', ' '];
+ const LITERAL = [' ', '&', '…', '<', ' ', 'A', '&', '—'];
+
+ test('the fixtures agree with the decode contract they stand for', () => {
+ for (const ref of DECODED) expect(isInlineWhitespaceNumericCharRef(ref)).toBe(true);
+ for (const ref of LITERAL) expect(isInlineWhitespaceNumericCharRef(ref)).toBe(false);
+ });
+
+ test('every decoded ref is crossable, in either spelling', () => {
+ // Decimal and hex, upper and lower `X`, zero-padded — all the same character
+ // to a reader, so all the same to a quote.
+ for (const ref of [' ', ' ', ' ', ' ']) {
+ expect(locate(`A ${ref} B`, 'A B')).toBe(`A ${ref} B`);
+ }
+ });
+
+ test('a tab ref is crossable', () => {
+ expect(locate('A 	 B', 'A \t B')).toBe('A 	 B');
+ });
+
+ test('an NBSP ref is crossable, and its character still has to be there', () => {
+ // NBSP is the case that rules out skipping the run as invisible syntax:
+ // U+00A0 is not whitespace to this matcher, so the quote carries it as
+ // content and the body's six bytes have to answer for it.
+ expect(locate('A   B', `A ${NBSP} B`)).toBe('A   B');
+ expect(locate('A   B', `A ${NBSP} B`)).toBe('A   B');
+ // ...and a quote holding an ordinary space where the reader sees an NBSP
+ // is quoting a character the body does not have.
+ expect(locate('A   B', 'A B')).toBeNull();
+ });
+
+ test('crosses a ref with no spaces around it', () => {
+ expect(locate('foo bar', 'foo bar')).toBe('foo bar');
+ });
+
+ test('crosses a run of back-to-back refs', () => {
+ expect(locate('A  	  B', 'A \t B')).toBe('A  	  B');
+ });
+
+ test('the reported passage: a boundary space inside emphasis', () => {
+ // The line from the reporter's document, verbatim. The serializer minted the
+ // ` ` to hold the space the author typed before the closing `***`.
+ expect(
+ locate(
+ '~~***External apps  ***[external action icon]~~',
+ 'External apps [external action icon]',
+ ),
+ ).toBe('External apps  ***[external action icon]');
+ });
+
+ test('a ref that renders as itself stays content', () => {
+ // ` ` and friends are never decoded, so they are on screen and a quote
+ // that omits them is quoting something else.
+ for (const ref of LITERAL) {
+ expect(locate(`A ${ref} B`, 'A B')).toBeNull();
+ }
+ });
+
+ test('a decoded ref does not make the surrounding text elastic', () => {
+ expect(locate('A   B', 'A C')).toBeNull();
+ expect(locate('A   B', 'X B')).toBeNull();
+ });
+
+ /**
+ * The stored side of the same disagreement.
+ *
+ * An anchor's `exact` is sliced out of the body, so it arrives carrying the
+ * entity; the editor searches the text a reader can see. A comment that
+ * anchored server-side would otherwise lose its highlight here, and the
+ * context scorer would read the neighbourhood as changed.
+ */
+ test('the ref is crossable from the stored side too', () => {
+ expect(findPassage('A B', 'A   B', { syntaxIn: 'needle' })).not.toBeNull();
+ expect(findPassage(`A ${NBSP} B`, 'A   B', { syntaxIn: 'needle' })).not.toBeNull();
+ expect(findPassage('foo bar', 'foo bar', { syntaxIn: 'needle' })).not.toBeNull();
+ });
+
+ test('a literal ref is still content from the stored side', () => {
+ expect(findPassage('A B', 'A   B', { syntaxIn: 'needle' })).toBeNull();
+ });
+
+ test('a match does not open on a ref the caller did not select', () => {
+ // The span is stored and re-asserted against the body, so a range that
+ // opens on six bytes the reader never saw is a range that reads wrong.
+ expect(locate(' foo', 'foo')).toBe('foo');
+ });
+
+ test('a run stops at the first byte that is not a ref', () => {
+ // Guards the sticky flag on the token regex. Sticky anchors `exec` at
+ // `lastIndex`; a global one would happily return a ref from further down
+ // the text, so the scan would swallow whatever sat in between and report a
+ // run that never existed. Here that would make `abc` invisible and let the
+ // quote match across it.
+ expect(locate('x abc def', 'x def')).toBeNull();
+ expect(locate('x abc def', 'x abc def')).toBe('x abc def');
+ });
+
+ test('malformed refs are ordinary content', () => {
+ // Only a complete, well-formed numeric ref decodes. A bare ampersand, an
+ // unterminated ref, and a ref with a bad body are all just characters.
+ for (const body of ['A & B', 'A &#x20 B', 'A &#; B', 'A &#xZZ; B', 'A &# B']) {
+ expect(locate(body, 'A B')).toBeNull();
+ }
+ });
+
+ /**
+ * Context scoring reads the same two substrates and has to make the same
+ * allowance. Without it every candidate near a ref scores near zero, the
+ * evidence floor can no longer be met, and a passage orphans because text
+ * NEAR it — inside the stored context window, outside the quote — was edited.
+ */
+ test('context scoring sees through a ref the same way', () => {
+ const body = 'Intro. External apps  ***[icon]*** trails off here.';
+ const span = { start: body.indexOf('External'), end: body.indexOf(' trails') };
+ // Context as a client captures it: rendered text, no markdown syntax in it.
+ const score = contextMatchScore(body, span, { prefix: 'Intro. ' }, { syntaxIn: 'haystack' });
+ expect(score).toBeGreaterThanOrEqual(contextEvidenceFloor({ prefix: 'Intro. ' }));
+ });
+
+ test('a stored context carrying a ref still scores against the rendered side', () => {
+ // The ref sits at the seam, where the context meets the passage — the
+ // position that decides the score, since agreement is counted inward from
+ // there. Six undecoded bytes at the seam take the common run to zero on
+ // their own, however well the rest of the window agrees.
+ const rendered = 'Intro. External apps today trails off here.';
+ const span = { start: rendered.indexOf('trails'), end: rendered.length };
+ const score = contextMatchScore(
+ rendered,
+ span,
+ { prefix: 'External apps ***today*** ' },
+ { syntaxIn: 'none', syntaxInContext: true },
+ );
+ expect(score).toBeGreaterThanOrEqual('Externalappstoday'.length);
+ });
+
+ test('context scoring keeps a decoded NBSP as content, not whitespace', () => {
+ // NBSP must survive condense on the body side so both sides of the comparison
+ // carry it. If NBSP were dropped like a space the condensations would disagree
+ // and the score would fall below the floor.
+ const body = 'Intro . External apps trails off here.';
+ const span = { start: body.indexOf('External'), end: body.length };
+ const score = contextMatchScore(
+ body,
+ span,
+ { prefix: `Intro${NBSP}. ` },
+ { syntaxIn: 'haystack' },
+ );
+ expect(score).toBeGreaterThanOrEqual(contextEvidenceFloor({ prefix: `Intro${NBSP}. ` }));
+ });
+});
diff --git a/packages/core/src/comments/passage-match.ts b/packages/core/src/comments/passage-match.ts
index 420245933..515acda21 100644
--- a/packages/core/src/comments/passage-match.ts
+++ b/packages/core/src/comments/passage-match.ts
@@ -15,8 +15,15 @@
* treat markdown syntax as elastic on whichever side carries it. Every other
* character must still match, in order: this is not a fuzzy match and cannot
* land on different words.
+ *
+ * Markdown syntax is not the only place the two sides legally disagree. The
+ * display pipeline also DECODES a narrow set of numeric character references —
+ * six bytes on disk, one character on screen — and this file is a consumer of
+ * that contract; `whitespace-char-ref.ts` owns which refs are in it.
*/
+import { decodeInlineWhitespaceNumericCharRef } from '../markdown/whitespace-char-ref.ts';
+
/** A located passage as `[start, end)` offsets into the haystack. */
export interface PassageMatch {
readonly start: number;
@@ -225,6 +232,59 @@ function invisibleLineRunAt(text: string, i: number): number {
return line.length;
}
+/**
+ * One numeric character reference.
+ *
+ * LOCKSTEP: the PATTERN BODY is identical to `NUMERIC_CHAR_REF_TOKEN` in
+ * `whitespace-char-ref.ts` and `NUMERIC_CHAR_REF_TOKEN_RE` in
+ * `to-markdown-handlers.ts`; change all three together or they classify
+ * different ref sets. The FLAG differs on purpose and must not be "corrected"
+ * to match: sticky (`/y`) anchors `exec` at `lastIndex`, which is what makes
+ * the run scan below positional. With `/g` the same call would happily return
+ * a match from further down the text, and the scan would report a run that
+ * does not start where it was asked to look.
+ *
+ * This file owns the scan; `whitespace-char-ref.ts` owns which refs decode and
+ * to what, via `decodeInlineWhitespaceNumericCharRef`.
+ */
+const NUMERIC_CHAR_REF_TOKEN = /&#(?:x[0-9A-Fa-f]+|X[0-9A-Fa-f]+|[0-9]+);/y;
+
+/**
+ * A run of back-to-back numeric character references the display pipeline
+ * decodes, and the characters it shows in their place — or null when `i` opens
+ * no such run.
+ *
+ * The byte-fidelity serializer mints these to hold a phrasing-boundary space or
+ * tab across re-parse (CommonMark §6.4), so an ordinary space typed just inside
+ * `**…**` reaches disk as ` `. Nobody types the entity, which is why this
+ * reaches ordinary documents.
+ *
+ * Deliberately NOT a `syntaxRunAt` entry. A syntax run means "renders as
+ * nothing, skip it" — true of a space or tab, false of ` `: NBSP is not
+ * whitespace to `isSpace`, so the other side carries it as content and the run
+ * has to be decoded and compared rather than skipped. Refs outside the decoded
+ * set (`&`, `…`, `—`) reach the screen as their own bytes and
+ * must stay content, so they get null and match literally as they always did.
+ */
+function renderedCharRefRunAt(
+ text: string,
+ i: number,
+): { length: number; rendered: string } | null {
+ if (text[i] !== '&') return null;
+ let end = i;
+ let rendered = '';
+ for (;;) {
+ NUMERIC_CHAR_REF_TOKEN.lastIndex = end;
+ const token = NUMERIC_CHAR_REF_TOKEN.exec(text)?.[0];
+ if (token === undefined) break;
+ const char = decodeInlineWhitespaceNumericCharRef(token);
+ if (char === null) break;
+ rendered += char;
+ end += token.length;
+ }
+ return end === i ? null : { length: end - i, rendered };
+}
+
function isSpace(ch: string): boolean {
return ch === ' ' || ch === '\t' || ch === '\n' || ch === '\r' || ch === '\f' || ch === '\v';
}
@@ -326,7 +386,13 @@ export function findAllPassages(
// is content to them and the loop will match it literally. Inline HTML that
// survives into rendered text is the case: a quote of `<div>` must still be
// able to start on the `<` that `HTML_TAG` would otherwise call syntax.
- if (syntaxInHaystack && first !== needle[0] && syntaxRunAt(haystack, start) > 0) continue;
+ if (
+ syntaxInHaystack &&
+ first !== needle[0] &&
+ (syntaxRunAt(haystack, start) > 0 || renderedCharRefRunAt(haystack, start) !== null)
+ ) {
+ continue;
+ }
let hi = start;
let ni = 0;
@@ -352,6 +418,37 @@ export function findAllPassages(
else ni += run;
continue;
}
+ // A run of character references the markdown side spends on a character
+ // the other side simply has. Consume the run there, and what it renders
+ // here: a decoded space or tab asks nothing (whitespace is already
+ // elastic both ways), while a decoded NBSP is content the other side has
+ // to supply — its own whitespace in front of it stays elastic.
+ const decoded = syntaxInHaystack
+ ? renderedCharRefRunAt(haystack, hi)
+ : renderedCharRefRunAt(needle, ni);
+ if (decoded !== null) {
+ const other = syntaxInHaystack ? needle : haystack;
+ let oi = syntaxInHaystack ? ni : hi;
+ let supplied = true;
+ for (const ch of decoded.rendered) {
+ if (isSpace(ch)) continue;
+ while (oi < other.length && isSpace(other[oi] as string)) oi += 1;
+ if (other[oi] !== ch) {
+ supplied = false;
+ break;
+ }
+ oi += 1;
+ }
+ if (!supplied) break;
+ if (syntaxInHaystack) {
+ hi += decoded.length;
+ ni = oi;
+ } else {
+ ni += decoded.length;
+ hi = oi;
+ }
+ continue;
+ }
break;
}
// Whitespace left over in the needle once the haystack is exhausted. The
@@ -456,6 +553,17 @@ function condense(text: string, from: number, to: number, syntax: boolean): stri
i += run;
continue;
}
+ const decoded = renderedCharRefRunAt(text, i);
+ if (decoded !== null) {
+ // Whitespace is dropped from both sides here, so a decoded space adds
+ // nothing; an NBSP is content and has to survive on this side exactly
+ // as it does on the rendered one, or the two condensations disagree.
+ for (const ch of decoded.rendered) {
+ if (!isSpace(ch)) out += ch;
+ }
+ i += decoded.length;
+ continue;
+ }
}
out += ch;
i += 1;