Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): anchor comments across decoded character refs (PRD-8182) (#3708)
shagun-singh-inkeep2cff3419c381
diff --git a/.changeset/comment-anchor-decoded-char-refs.md b/.changeset/comment-anchor-decoded-char-refs.mdnew file mode 100644index 000000000..c025785a4--- /dev/null+++ b/.changeset/comment-anchor-decoded-char-refs.md@@ -0,0 +1,9 @@+---+"@inkeep/open-knowledge": patch+---++Fix comments being refused on any passage containing a boundary-whitespace character reference++Selecting text and adding a comment failed with "The quoted passage is not in the document" whenever the selection crossed a space typed just inside bold, italic, or strikethrough, or a space-indented line. The passage matcher treated markdown syntax as elastic but did not know that the display pipeline decodes a narrow set of numeric character references — ` `, `	`, ` ` — which the byte-fidelity serializer mints to hold a phrasing-boundary space across re-parse. Six bytes on disk, one character on screen, and the match broke at the ampersand.++Anchoring now decodes those references on whichever side carries them, in both directions, and context scoring makes the same allowance so a highlight no longer degrades near one. References the pipeline does not decode (`&`, ` `, `…`) still match literally, as they always did.diff --git a/packages/app/src/comments/anchor-search.test.ts b/packages/app/src/comments/anchor-search.test.tsindex f5ea8538e..523fd5e06 100644--- a/packages/app/src/comments/anchor-search.test.ts+++ b/packages/app/src/comments/anchor-search.test.ts@@ -449,3 +449,46 @@ describe('captureSelectionContext across an inline atom', () => {expect(createAnchorResolver(doc)('done', context)?.from).toBe(target);});});++/**+ * A stored quote carrying a decoded character reference.+ *+ * The anchor's `exact` is sliced out of the markdown BODY, so it arrives with+ * whatever the body spends on formatting — and with the boundary-whitespace+ * char-refs the byte-fidelity serializer mints to hold a space just inside+ * emphasis. The editor searches the text a reader can SEE, where those six bytes+ * are one space. Server-side the passage resolves; here the highlight vanished.+ */+describe('findRangeInIndex — a stored quote carrying a boundary-whitespace char-ref', () => {+ const RENDERED = 'Intro line. External apps [external action icon] follows.';+ const STORED = 'External apps  ***[external action icon]';++ test('resolves the passage the reader can see, on the quote alone', () => {+ // No context, so the bracket recovery that would otherwise paper over this+ // has nothing to work from — the quote itself has to match.+ const range = findRangeInIndex(indexOf(RENDERED), STORED);+ expect(range).not.toBeNull();+ expect(RENDERED.slice((range?.from ?? 1) - 1, (range?.to ?? 1) - 1)).toBe(+ 'External apps [external action icon]',+ );+ });++ test('resolves it on the quote, not by recovering it from its brackets', () => {+ // With context present the bracket recovery can reach the same span, but it+ // reports a REWRITTEN anchor — the caller re-captures `exact` from it, so a+ // passage nobody edited would churn its stored bytes on every re-find.+ const range = findRangeInIndex(indexOf(RENDERED), STORED, {+ prefix: 'Intro line. ',+ suffix: ' follows.',+ });+ expect(range).not.toBeNull();+ expect(RENDERED.slice((range?.from ?? 1) - 1, (range?.to ?? 1) - 1)).toBe(+ 'External apps [external action icon]',+ );+ });++ test('does not let the ref drag the match across neighbouring words', () => {+ const rendered = 'alpha omega';+ expect(findRangeInIndex(indexOf(rendered), 'alpha & omega')).toBeNull();+ });+});diff --git a/packages/core/src/comments/leaf-text.test.ts b/packages/core/src/comments/leaf-text.test.tsindex 51afe4b6f..43f6c8a29 100644--- a/packages/core/src/comments/leaf-text.test.ts+++ b/packages/core/src/comments/leaf-text.test.ts@@ -6,7 +6,7 @@* hole in it (so a comment on the surrounding prose failed to anchor).*/-import { getSchema } from '@tiptap/core';+import { getSchema, type JSONContent } from '@tiptap/core';import { describe, expect, test } from 'vitest';import { sharedExtensions } from '../extensions/shared.ts';import { MarkdownManager } from '../markdown/index.ts';@@ -128,6 +128,13 @@ describe('the captured quote anchors back to the body', () => {'Before.\n\n```mermaid\ngraph TD;\n```\n\nAfter.','Before.\n\n$$\nx = 1\n$$\n\nAfter.','See [[page]] and #tag with ==mark== plus $$y$$ and .',+ // Boundary-whitespace char-refs. Nobody types these: the byte-fidelity+ // serializer mints them to hold a phrasing-boundary space across re-parse,+ // so an ordinary keystroke inside emphasis puts one in the body.+ 'A   word.',+ '**bold **tail',+ '~~***External apps  ***[external action icon]~~',+ '  indented line',];for (const md of CASES) {@@ -135,6 +142,27 @@ describe('the captured quote anchors back to the body', () => {expect(locate(md)).not.toBeNull();});}++ test('the boundary-space fixtures above are what the serializer actually mints', () => {+ // Keeps those cases honest: they are only worth pinning because a round trip+ // through this pipeline produces them from an ordinary space.+ const withBoundarySpace = {+ type: 'doc',+ content: [+ {+ type: 'paragraph',+ content: [+ { type: 'text', marks: [{ type: 'strong' }], text: 'bold ' },+ { type: 'text', text: 'tail' },+ ],+ },+ { type: 'paragraph', content: [{ type: 'text', text: ' indented line' }] },+ ],+ };+ expect(mdManager.serialize(withBoundarySpace satisfies JSONContent)).toBe(+ '**bold **tail\n\n  indented line\n',+ );+ });});describe('inlineOnly — for callers deciding whether text formatting applies', () => {diff --git a/packages/core/src/comments/passage-match.test.ts b/packages/core/src/comments/passage-match.test.tsindex a6fed98c5..c0e7f9275 100644--- a/packages/core/src/comments/passage-match.test.ts+++ b/packages/core/src/comments/passage-match.test.ts@@ -7,7 +7,14 @@*/import { describe, expect, test } from 'vitest';-import { findAllPassages, findPassage, rewriteCeiling } from './passage-match.ts';+import { isInlineWhitespaceNumericCharRef } from '../markdown/whitespace-char-ref.ts';+import {+ contextEvidenceFloor,+ contextMatchScore,+ findAllPassages,+ findPassage,+ rewriteCeiling,+} from './passage-match.ts';const BODY = `## Ingredients@@ -526,3 +533,191 @@ describe('a needle ending in whitespace, at the end of the haystack', () => {expect(findAllPassages(rendered, `${prefix}crema\n.`, { syntaxIn: 'needle' })).toEqual([]);});});++/**+ * Character references the display pipeline DECODES.+ *+ * The matcher treats markdown's own syntax as elastic, but markdown syntax is+ * not the only place the body and the screen legally disagree. The byte-fidelity+ * serializer mints a bare numeric char-ref to hold a phrasing-boundary space or+ * tab across re-parse (CommonMark §6.4) — a space just inside `**…**` would+ * otherwise dissolve the emphasis — and the display decode turns it back into+ * the one character the reader sees. Six bytes on disk, one on screen.+ *+ * Nothing taught the matcher that, so a quote crossing one cannot find its passage+ * in the body and the anchor is refused. It reaches ordinary documents because+ * nobody types the entity: typing a space just inside bold, or indenting a+ * paragraph, is enough to mint one.+ *+ * The decode set is deliberately narrow — `whitespace-char-ref.ts` owns it, and+ * only SPACE, TAB, and NBSP are in it. Everything else (` `, `&`,+ * `—`) survives to the screen as its own literal bytes and already+ * matches; making those elastic would let a passage cross an ampersand or an em+ * dash the quote never contained.+ */+describe('numeric character references the display pipeline decodes', () => {+ /** The slice of `body` a caller's rendered quote resolves to. */+ function locate(body: string, quote: string): string | null {+ const hit = findPassage(body, quote, { syntaxIn: 'haystack' });+ return hit ? body.slice(hit.start, hit.end) : null;+ }++ /**+ * The fixtures below name refs on both sides of the decode boundary. This+ * pins them to the predicate that actually draws it, so widening the decode+ * set fails here rather than silently leaving the matcher a construct behind+ * — the exact class of drift a decode-set widening would silently introduce.+ */+ /** U+00A0, spelled out — a literal one in a string is invisible to a reader. */+ const NBSP = '\u00A0';++ const DECODED = [' ', ' ', ' ', ' ', '	', ' ', ' '];+ const LITERAL = [' ', '&', '…', '<', ' ', 'A', '&', '—'];++ test('the fixtures agree with the decode contract they stand for', () => {+ for (const ref of DECODED) expect(isInlineWhitespaceNumericCharRef(ref)).toBe(true);+ for (const ref of LITERAL) expect(isInlineWhitespaceNumericCharRef(ref)).toBe(false);+ });++ test('every decoded ref is crossable, in either spelling', () => {+ // Decimal and hex, upper and lower `X`, zero-padded — all the same character+ // to a reader, so all the same to a quote.+ for (const ref of [' ', ' ', ' ', ' ']) {+ expect(locate(`A ${ref} B`, 'A B')).toBe(`A ${ref} B`);+ }+ });++ test('a tab ref is crossable', () => {+ expect(locate('A 	 B', 'A \t B')).toBe('A 	 B');+ });++ test('an NBSP ref is crossable, and its character still has to be there', () => {+ // NBSP is the case that rules out skipping the run as invisible syntax:+ // U+00A0 is not whitespace to this matcher, so the quote carries it as+ // content and the body's six bytes have to answer for it.+ expect(locate('A   B', `A ${NBSP} B`)).toBe('A   B');+ expect(locate('A   B', `A ${NBSP} B`)).toBe('A   B');+ // ...and a quote holding an ordinary space where the reader sees an NBSP+ // is quoting a character the body does not have.+ expect(locate('A   B', 'A B')).toBeNull();+ });++ test('crosses a ref with no spaces around it', () => {+ expect(locate('foo bar', 'foo bar')).toBe('foo bar');+ });++ test('crosses a run of back-to-back refs', () => {+ expect(locate('A  	  B', 'A \t B')).toBe('A  	  B');+ });++ test('the reported passage: a boundary space inside emphasis', () => {+ // The line from the reporter's document, verbatim. The serializer minted the+ // ` ` to hold the space the author typed before the closing `***`.+ expect(+ locate(+ '~~***External apps  ***[external action icon]~~',+ 'External apps [external action icon]',+ ),+ ).toBe('External apps  ***[external action icon]');+ });++ test('a ref that renders as itself stays content', () => {+ // ` ` and friends are never decoded, so they are on screen and a quote+ // that omits them is quoting something else.+ for (const ref of LITERAL) {+ expect(locate(`A ${ref} B`, 'A B')).toBeNull();+ }+ });++ test('a decoded ref does not make the surrounding text elastic', () => {+ expect(locate('A   B', 'A C')).toBeNull();+ expect(locate('A   B', 'X B')).toBeNull();+ });++ /**+ * The stored side of the same disagreement.+ *+ * An anchor's `exact` is sliced out of the body, so it arrives carrying the+ * entity; the editor searches the text a reader can see. A comment that+ * anchored server-side would otherwise lose its highlight here, and the+ * context scorer would read the neighbourhood as changed.+ */+ test('the ref is crossable from the stored side too', () => {+ expect(findPassage('A B', 'A   B', { syntaxIn: 'needle' })).not.toBeNull();+ expect(findPassage(`A ${NBSP} B`, 'A   B', { syntaxIn: 'needle' })).not.toBeNull();+ expect(findPassage('foo bar', 'foo bar', { syntaxIn: 'needle' })).not.toBeNull();+ });++ test('a literal ref is still content from the stored side', () => {+ expect(findPassage('A B', 'A B', { syntaxIn: 'needle' })).toBeNull();+ });++ test('a match does not open on a ref the caller did not select', () => {+ // The span is stored and re-asserted against the body, so a range that+ // opens on six bytes the reader never saw is a range that reads wrong.+ expect(locate(' foo', 'foo')).toBe('foo');+ });++ test('a run stops at the first byte that is not a ref', () => {+ // Guards the sticky flag on the token regex. Sticky anchors `exec` at+ // `lastIndex`; a global one would happily return a ref from further down+ // the text, so the scan would swallow whatever sat in between and report a+ // run that never existed. Here that would make `abc` invisible and let the+ // quote match across it.+ expect(locate('x abc def', 'x def')).toBeNull();+ expect(locate('x abc def', 'x abc def')).toBe('x abc def');+ });++ test('malformed refs are ordinary content', () => {+ // Only a complete, well-formed numeric ref decodes. A bare ampersand, an+ // unterminated ref, and a ref with a bad body are all just characters.+ for (const body of ['A & B', 'A   B', 'A &#; B', 'A &#xZZ; B', 'A &# B']) {+ expect(locate(body, 'A B')).toBeNull();+ }+ });++ /**+ * Context scoring reads the same two substrates and has to make the same+ * allowance. Without it every candidate near a ref scores near zero, the+ * evidence floor can no longer be met, and a passage orphans because text+ * NEAR it — inside the stored context window, outside the quote — was edited.+ */+ test('context scoring sees through a ref the same way', () => {+ const body = 'Intro. External apps  ***[icon]*** trails off here.';+ const span = { start: body.indexOf('External'), end: body.indexOf(' trails') };+ // Context as a client captures it: rendered text, no markdown syntax in it.+ const score = contextMatchScore(body, span, { prefix: 'Intro. ' }, { syntaxIn: 'haystack' });+ expect(score).toBeGreaterThanOrEqual(contextEvidenceFloor({ prefix: 'Intro. ' }));+ });++ test('a stored context carrying a ref still scores against the rendered side', () => {+ // The ref sits at the seam, where the context meets the passage — the+ // position that decides the score, since agreement is counted inward from+ // there. Six undecoded bytes at the seam take the common run to zero on+ // their own, however well the rest of the window agrees.+ const rendered = 'Intro. External apps today trails off here.';+ const span = { start: rendered.indexOf('trails'), end: rendered.length };+ const score = contextMatchScore(+ rendered,+ span,+ { prefix: 'External apps ***today*** ' },+ { syntaxIn: 'none', syntaxInContext: true },+ );+ expect(score).toBeGreaterThanOrEqual('Externalappstoday'.length);+ });++ test('context scoring keeps a decoded NBSP as content, not whitespace', () => {+ // NBSP must survive condense on the body side so both sides of the comparison+ // carry it. If NBSP were dropped like a space the condensations would disagree+ // and the score would fall below the floor.+ const body = 'Intro . External apps trails off here.';+ const span = { start: body.indexOf('External'), end: body.length };+ const score = contextMatchScore(+ body,+ span,+ { prefix: `Intro${NBSP}. ` },+ { syntaxIn: 'haystack' },+ );+ expect(score).toBeGreaterThanOrEqual(contextEvidenceFloor({ prefix: `Intro${NBSP}. ` }));+ });+});diff --git a/packages/core/src/comments/passage-match.ts b/packages/core/src/comments/passage-match.tsindex 420245933..515acda21 100644--- a/packages/core/src/comments/passage-match.ts+++ b/packages/core/src/comments/passage-match.ts@@ -15,8 +15,15 @@* treat markdown syntax as elastic on whichever side carries it. Every other* character must still match, in order: this is not a fuzzy match and cannot* land on different words.+ *+ * Markdown syntax is not the only place the two sides legally disagree. The+ * display pipeline also DECODES a narrow set of numeric character references —+ * six bytes on disk, one character on screen — and this file is a consumer of+ * that contract; `whitespace-char-ref.ts` owns which refs are in it.*/+import { decodeInlineWhitespaceNumericCharRef } from '../markdown/whitespace-char-ref.ts';+/** A located passage as `[start, end)` offsets into the haystack. */export interface PassageMatch {readonly start: number;@@ -225,6 +232,59 @@ function invisibleLineRunAt(text: string, i: number): number {return line.length;}+/**+ * One numeric character reference.+ *+ * LOCKSTEP: the PATTERN BODY is identical to `NUMERIC_CHAR_REF_TOKEN` in+ * `whitespace-char-ref.ts` and `NUMERIC_CHAR_REF_TOKEN_RE` in+ * `to-markdown-handlers.ts`; change all three together or they classify+ * different ref sets. The FLAG differs on purpose and must not be "corrected"+ * to match: sticky (`/y`) anchors `exec` at `lastIndex`, which is what makes+ * the run scan below positional. With `/g` the same call would happily return+ * a match from further down the text, and the scan would report a run that+ * does not start where it was asked to look.+ *+ * This file owns the scan; `whitespace-char-ref.ts` owns which refs decode and+ * to what, via `decodeInlineWhitespaceNumericCharRef`.+ */+const NUMERIC_CHAR_REF_TOKEN = /&#(?:x[0-9A-Fa-f]+|X[0-9A-Fa-f]+|[0-9]+);/y;++/**+ * A run of back-to-back numeric character references the display pipeline+ * decodes, and the characters it shows in their place — or null when `i` opens+ * no such run.+ *+ * The byte-fidelity serializer mints these to hold a phrasing-boundary space or+ * tab across re-parse (CommonMark §6.4), so an ordinary space typed just inside+ * `**…**` reaches disk as ` `. Nobody types the entity, which is why this+ * reaches ordinary documents.+ *+ * Deliberately NOT a `syntaxRunAt` entry. A syntax run means "renders as+ * nothing, skip it" — true of a space or tab, false of ` `: NBSP is not+ * whitespace to `isSpace`, so the other side carries it as content and the run+ * has to be decoded and compared rather than skipped. Refs outside the decoded+ * set (`&`, `…`, `—`) reach the screen as their own bytes and+ * must stay content, so they get null and match literally as they always did.+ */+function renderedCharRefRunAt(+ text: string,+ i: number,+): { length: number; rendered: string } | null {+ if (text[i] !== '&') return null;+ let end = i;+ let rendered = '';+ for (;;) {+ NUMERIC_CHAR_REF_TOKEN.lastIndex = end;+ const token = NUMERIC_CHAR_REF_TOKEN.exec(text)?.[0];+ if (token === undefined) break;+ const char = decodeInlineWhitespaceNumericCharRef(token);+ if (char === null) break;+ rendered += char;+ end += token.length;+ }+ return end === i ? null : { length: end - i, rendered };+}+function isSpace(ch: string): boolean {return ch === ' ' || ch === '\t' || ch === '\n' || ch === '\r' || ch === '\f' || ch === '\v';}@@ -326,7 +386,13 @@ export function findAllPassages(// is content to them and the loop will match it literally. Inline HTML that// survives into rendered text is the case: a quote of `<div>` must still be// able to start on the `<` that `HTML_TAG` would otherwise call syntax.- if (syntaxInHaystack && first !== needle[0] && syntaxRunAt(haystack, start) > 0) continue;+ if (+ syntaxInHaystack &&+ first !== needle[0] &&+ (syntaxRunAt(haystack, start) > 0 || renderedCharRefRunAt(haystack, start) !== null)+ ) {+ continue;+ }let hi = start;let ni = 0;@@ -352,6 +418,37 @@ export function findAllPassages(else ni += run;continue;}+ // A run of character references the markdown side spends on a character+ // the other side simply has. Consume the run there, and what it renders+ // here: a decoded space or tab asks nothing (whitespace is already+ // elastic both ways), while a decoded NBSP is content the other side has+ // to supply — its own whitespace in front of it stays elastic.+ const decoded = syntaxInHaystack+ ? renderedCharRefRunAt(haystack, hi)+ : renderedCharRefRunAt(needle, ni);+ if (decoded !== null) {+ const other = syntaxInHaystack ? needle : haystack;+ let oi = syntaxInHaystack ? ni : hi;+ let supplied = true;+ for (const ch of decoded.rendered) {+ if (isSpace(ch)) continue;+ while (oi < other.length && isSpace(other[oi] as string)) oi += 1;+ if (other[oi] !== ch) {+ supplied = false;+ break;+ }+ oi += 1;+ }+ if (!supplied) break;+ if (syntaxInHaystack) {+ hi += decoded.length;+ ni = oi;+ } else {+ ni += decoded.length;+ hi = oi;+ }+ continue;+ }break;}// Whitespace left over in the needle once the haystack is exhausted. The@@ -456,6 +553,17 @@ function condense(text: string, from: number, to: number, syntax: boolean): strii += run;continue;}+ const decoded = renderedCharRefRunAt(text, i);+ if (decoded !== null) {+ // Whitespace is dropped from both sides here, so a decoded space adds+ // nothing; an NBSP is content and has to survive on this side exactly+ // as it does on the rendered one, or the two condensations disagree.+ for (const ch of decoded.rendered) {+ if (!isSpace(ch)) out += ch;+ }+ i += decoded.length;+ continue;+ }}out += ch;i += 1;