Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(atoms): keep quote provenance when the model drops Markdown or punctuation (#8371)
garrytaneef8bae574d2
diff --git a/src/core/cycle/extract-atoms.ts b/src/core/cycle/extract-atoms.tsindex f5fc349b6e..0acd2f5a02 100644--- a/src/core/cycle/extract-atoms.ts+++ b/src/core/cycle/extract-atoms.ts@@ -86,7 +86,7 @@ import { resolveAlias, resolveTierDefault } from '../model-config.ts';import { isMissingSchemaObject, warnOncePerProcess } from '../utils.ts';import { utcDate } from './cycle-date.ts';import { atomSourceDate } from './atom-source-date.ts';-import { normalizeForGrounding } from './synthesize-verify.ts';+import { normalizeForGrounding, isGroundingQuoteMark } from './synthesize-verify.ts';import type { TranscriptPageIndex } from '../transcripts/discover.ts';import { atomOriginPin, atomWriteFailure, type AtomWriteFailure, managedAtomSession, readAtomOrigin, resumeManagedAtoms, publishManagedAtoms, publishManagedAtomGeneration, resumeManagedGenerationMembers,MANAGED_ATOM_DISCOVERY_SQL, managedAtomDiscoverySql, type AtomOrigin } from '../persistence/atom-maintenance.ts';@@ -269,7 +269,9 @@ export interface ExtractAtomsOpts {** Folding rides `normalizeForGrounding` — the v0.47.8.0 quote-repair core in* synthesize-verify.ts (the "ONE folding core" invariant: this module must- * mean the same thing by "normalized substring" as the repair ladder does).+ * mean the same thing by "normalized substring" as the repair ladder does),+ * plus its markup fold (#8371): a quote that drops the source's line markers,+ * quotation marks or a spaced dash still says the source's exact words.* Everything is decided in FOLDED space, including uniqueness: a typographic* twin ('“go now”' vs '"go now"') is only visible there, and returning an* offset that might name the wrong passage is worse than returning none.@@ -287,9 +289,10 @@ export function locateQuote(quote: string,): { start: number; end: number } | null {if (!quote || !content) return null;- const c = normalizeForGrounding(content);- const q = normalizeForGrounding(quote);+ const c = normalizeForGrounding(content, { markup: true });+ const q = normalizeForGrounding(quote, { markup: true });if (!q.norm) return null;+ const quoted = quote.trim();// Enumerate every folded hit, keep those that survive the round-trip// boundary check, THEN judge ambiguity. Order matters: rejecting on raw@@ -302,16 +305,18 @@ export function locateQuote(let seen = 0;while (at !== -1 && seen < MAX_CANDIDATES) {seen++;- const start = c.map[at]!;+ let start = c.map[at]!;// map[] names the FIRST code unit of the original character; advance the// end by the whole code point, not +1 — a bare +1 splits the surrogate// pair when the quote ends with a non-BMP char ('ship it 🚀'), and the// half-pair slice then fails the round-trip re-fold below.const lastOrig = c.map[at + q.norm.length - 1]!;const lastCp = content.codePointAt(lastOrig);- const end = lastOrig + (lastCp !== undefined && lastCp > 0xffff ? 2 : 1);+ let end = lastOrig + (lastCp !== undefined && lastCp > 0xffff ? 2 : 1);+ if (isGroundingQuoteMark(quoted[0]) && isGroundingQuoteMark(content[start - 1])) start--;+ if (isGroundingQuoteMark(quoted.at(-1)) && isGroundingQuoteMark(content[end])) end++;if (- normalizeForGrounding(content.slice(start, end)).norm === q.norm &&+ normalizeForGrounding(content.slice(start, end), { markup: true }).norm === q.norm &&!valid.some(v => v.start === start && v.end === end)) {valid.push({ start, end });diff --git a/src/core/cycle/synthesize-verify.ts b/src/core/cycle/synthesize-verify.tsindex 94d6b23376..01db1396a8 100644--- a/src/core/cycle/synthesize-verify.ts+++ b/src/core/cycle/synthesize-verify.ts@@ -202,8 +202,8 @@ export function emptyQuoteVerifyStats(): QuoteVerifyStats {* source char desynced every later offset and could slice garbage — or* nothing — back into a page as a "verbatim" repair).*/-export function normalizeForGrounding(s: string, opts: { tolerant?: boolean } = {}): { norm: string; map: number[] } {- return foldForGrounding(s, true, opts.tolerant === true) as { norm: string; map: number[] };+export function normalizeForGrounding(s: string, opts: { tolerant?: boolean; markup?: boolean } = {}): { norm: string; map: number[] } {+ return foldForGrounding(s, true, opts.tolerant === true, opts.markup === true) as { norm: string; map: number[] };}/**@@ -216,10 +216,10 @@ export function normalizeForGrounding(s: string, opts: { tolerant?: boolean } =* Parity matters: the rescue gate and the repair ladder must mean the same* thing by "normalized substring of the transcript".*/-function foldForGrounding(s: string, withMap: boolean, tolerant = false): { norm: string; map: number[] } | string {+function foldForGrounding(s: string, withMap: boolean, tolerant = false, markup = false): { norm: string; map: number[] } | string {const out: string[] = [];const map: number[] = [];- const skip = foldSkipSet(s, tolerant);+ const skip = foldSkipSet(s, tolerant, markup);let pendingSpace = false;// Iterate by CODE POINT (for..of), not code unit: a surrogate pair// lowercases as a pair (Deseret 𐐀 → 𐐨) but never half by half, so a@@ -277,16 +277,53 @@ function bracketMask(s: string): Set<number> | null {return skip;}+const DOUBLE_QUOTE_MARKS = new Set(['"', '“', '”', '„', '«', '»']);+const SINGLE_QUOTE_MARKS = new Set(["'", '‘', '’']);+const SPACED_DASHES = new Set(['-', '–', '—']);+export const isGroundingQuoteMark = (ch: string | undefined): boolean => ch !== undefined && (DOUBLE_QUOTE_MARKS.has(ch) || SINGLE_QUOTE_MARKS.has(ch));+const isWordChar = (ch: string | undefined): boolean => ch !== undefined && /[\p{L}\p{N}]/u.test(ch);+const isSpaceOrEdge = (ch: string | undefined): boolean => ch === undefined || /\s/.test(ch);+const isDigit = (ch: string | undefined): boolean => ch !== undefined && /\p{N}/u.test(ch);+const nonSpace = (s: string, from: number, step: 1 | -1): string | undefined => { let k = from; while (k >= 0 && k < s.length && /\s/.test(s[k]!)) k += step; return s[k]; };++/**+ * #8371: code-unit offsets the markup fold also skips, for a quote a model+ * copied without the source's Markdown and punctuation: line-leading '#' runs+ * and '>' markers, quotation marks (an apostrophe inside a word stays), and a+ * dash with whitespace on both sides and no number beside it (a spaced minus or+ * range is meaning). Words are never skipped.+ */+function markupMask(s: string): Set<number> | null {+ const skip = new Set<number>();+ let lineStart = true;+ for (let i = 0; i < s.length; i++) {+ const ch = s[i]!;+ if (ch === '\n') { lineStart = true; continue; }+ if (lineStart) {+ if (/\s/.test(ch)) continue;+ if (ch === '>' && (isSpaceOrEdge(s[i + 1]) || s[i + 1] === '>')) { skip.add(i); continue; }+ let j = i;+ while (s[j] === '#') j++;+ if (j > i && isSpaceOrEdge(s[j])) { for (let k = i; k < j; k++) skip.add(k); i = j - 1; continue; }+ lineStart = false;+ }+ if (DOUBLE_QUOTE_MARKS.has(ch) || (SINGLE_QUOTE_MARKS.has(ch) && !(isWordChar(s[i - 1]) && isWordChar(s[i + 1])))) skip.add(i);+ else if (SPACED_DASHES.has(ch) && isSpaceOrEdge(s[i - 1]) && isSpaceOrEdge(s[i + 1]) && !isDigit(nonSpace(s, i - 1, -1)) && !isDigit(nonSpace(s, i + 1, 1))) skip.add(i);+ }+ return skip.size ? skip : null;+}+/*** Code-unit offsets the fold skips: markdown inline markup (every `*`, every* backtick, and `~` in a `~~` run) in both modes, so `**sync**`, `` `sync` ``* and `sync` read alike; plus `bracketMask` in tolerant mode. `_` is kept:* it is too common in identifiers and paths.*/-function foldSkipSet(s: string, tolerant: boolean): Set<number> | null {+function foldSkipSet(s: string, tolerant: boolean, markup = false): Set<number> | null {const brackets = tolerant ? bracketMask(s) : null;- if (!/[*`]|~~/.test(s)) return brackets;- const skip = brackets ?? new Set<number>();+ const masked = markup ? [...(markupMask(s) ?? [])].reduce((set, i) => set.add(i), brackets ?? new Set<number>()) : brackets;+ if (!/[*`]|~~/.test(s)) return masked;+ const skip = masked ?? new Set<number>();for (let i = 0; i < s.length; i++) {const ch = s[i];if (ch === '*' || ch === '`' || (ch === '~' && (s[i - 1] === '~' || s[i + 1] === '~'))) skip.add(i);diff --git a/test/cycle/extract-atoms-quote-markup-8373.test.ts b/test/cycle/extract-atoms-quote-markup-8373.test.tsnew file mode 100644index 0000000000..76cc7c0294--- /dev/null+++ b/test/cycle/extract-atoms-quote-markup-8373.test.ts@@ -0,0 +1,54 @@+/**+ * #8371 / #8373 — an atom's source quote that differs from the source only by+ * Markdown block markers ('## ', '> '), quotation marks or a spaced dash says+ * the source's exact words. It must locate (provenance kept, offsets into the+ * ORIGINAL text) instead of being stamped quote_unverified as a paraphrase.+ * Changed wording still fails closed.+ */+import { describe, test, expect } from 'bun:test';+import { locateQuote } from '../../src/core/cycle/extract-atoms.ts';++function span(content: string, quote: string): string | null {+ const loc = locateQuote(content, quote);+ return loc ? content.slice(loc.start, loc.end) : null;+}++describe('locateQuote tolerates Markdown and punctuation the model drops (#8371)', () => {+ test('a heading and a blockquote marker', () => {+ const content = '## Decision\n> We will ship on Friday.';+ expect(span(content, 'Decision We will ship on Friday.')).toBe('Decision\n> We will ship on Friday.');+ });++ test('nested blockquote markers and a deeper heading', () => {+ const content = 'intro\n### Plan\n> > Keep the cap at forty hours.';+ expect(span(content, 'Plan Keep the cap at forty hours.')).toBe('Plan\n> > Keep the cap at forty hours.');+ });++ test('inner quotation marks, straight and curly', () => {+ expect(span('She said "done" today.', 'She said done today.')).toBe('She said "done" today.');+ expect(span('She said “done” today.', 'She said done today.')).toBe('She said “done” today.');+ expect(span("He called it 'final' twice.", 'He called it final twice.')).toBe("He called it 'final' twice.");+ });++ test('a spaced dash, plain or em', () => {+ expect(span('Cost: 40 hours - approved.', 'Cost: 40 hours approved.')).toBe('Cost: 40 hours - approved.');+ expect(span('Cost: 40 hours — approved.', 'Cost: 40 hours approved.')).toBe('Cost: 40 hours — approved.');+ });++ test('apostrophes inside words and unspaced dashes still count', () => {+ expect(span("We don't ship on Mondays.", 'We dont ship on Mondays.')).toBeNull();+ expect(span('The budget is 40-50 hours.', 'The budget is 4050 hours.')).toBeNull();+ });++ test('a sign, a range or a comparison next to a number is meaning, not markup', () => {+ expect(span('Net margin: − 12% this quarter.', 'Net margin: 12% this quarter.')).toBeNull();+ expect(span('Net margin: - 12% this quarter.', 'Net margin: 12% this quarter.')).toBeNull();+ expect(span('Shipping in 10 - 20 days.', 'Shipping in 10 20 days.')).toBeNull();+ expect(span('We saw 5% churn.', '>5% churn.')).toBeNull();+ });++ test('changed wording still fails closed', () => {+ expect(span('## Decision\n> We will ship on Friday.', 'Decision We will not ship on Friday.')).toBeNull();+ expect(span('She said "done" today.', 'She said finished today.')).toBeNull();+ });+});