Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(atoms): keep quote provenance when the model drops Markdown or punctuation (#8371)

garrytaneef8bae574d2
diff --git a/src/core/cycle/extract-atoms.ts b/src/core/cycle/extract-atoms.ts
index f5fc349b6e..0acd2f5a02 100644
--- a/src/core/cycle/extract-atoms.ts
+++ b/src/core/cycle/extract-atoms.ts
@@ -86,7 +86,7 @@ import { resolveAlias, resolveTierDefault } from '../model-config.ts';
import { isMissingSchemaObject, warnOncePerProcess } from '../utils.ts';
import { utcDate } from './cycle-date.ts';
import { atomSourceDate } from './atom-source-date.ts';
-import { normalizeForGrounding } from './synthesize-verify.ts';
+import { normalizeForGrounding, isGroundingQuoteMark } from './synthesize-verify.ts';
import type { TranscriptPageIndex } from '../transcripts/discover.ts';
import { atomOriginPin, atomWriteFailure, type AtomWriteFailure, managedAtomSession, readAtomOrigin, resumeManagedAtoms, publishManagedAtoms, publishManagedAtomGeneration, resumeManagedGenerationMembers,
MANAGED_ATOM_DISCOVERY_SQL, managedAtomDiscoverySql, type AtomOrigin } from '../persistence/atom-maintenance.ts';
@@ -269,7 +269,9 @@ export interface ExtractAtomsOpts {
*
* Folding rides `normalizeForGrounding` — the v0.47.8.0 quote-repair core in
* synthesize-verify.ts (the "ONE folding core" invariant: this module must
- * mean the same thing by "normalized substring" as the repair ladder does).
+ * mean the same thing by "normalized substring" as the repair ladder does),
+ * plus its markup fold (#8371): a quote that drops the source's line markers,
+ * quotation marks or a spaced dash still says the source's exact words.
* Everything is decided in FOLDED space, including uniqueness: a typographic
* twin ('“go now”' vs '"go now"') is only visible there, and returning an
* offset that might name the wrong passage is worse than returning none.
@@ -287,9 +289,10 @@ export function locateQuote(
quote: string,
): { start: number; end: number } | null {
if (!quote || !content) return null;
- const c = normalizeForGrounding(content);
- const q = normalizeForGrounding(quote);
+ const c = normalizeForGrounding(content, { markup: true });
+ const q = normalizeForGrounding(quote, { markup: true });
if (!q.norm) return null;
+ const quoted = quote.trim();
// Enumerate every folded hit, keep those that survive the round-trip
// boundary check, THEN judge ambiguity. Order matters: rejecting on raw
@@ -302,16 +305,18 @@ export function locateQuote(
let seen = 0;
while (at !== -1 && seen < MAX_CANDIDATES) {
seen++;
- const start = c.map[at]!;
+ let start = c.map[at]!;
// map[] names the FIRST code unit of the original character; advance the
// end by the whole code point, not +1 — a bare +1 splits the surrogate
// pair when the quote ends with a non-BMP char ('ship it 🚀'), and the
// half-pair slice then fails the round-trip re-fold below.
const lastOrig = c.map[at + q.norm.length - 1]!;
const lastCp = content.codePointAt(lastOrig);
- const end = lastOrig + (lastCp !== undefined && lastCp > 0xffff ? 2 : 1);
+ let end = lastOrig + (lastCp !== undefined && lastCp > 0xffff ? 2 : 1);
+ if (isGroundingQuoteMark(quoted[0]) && isGroundingQuoteMark(content[start - 1])) start--;
+ if (isGroundingQuoteMark(quoted.at(-1)) && isGroundingQuoteMark(content[end])) end++;
if (
- normalizeForGrounding(content.slice(start, end)).norm === q.norm &&
+ normalizeForGrounding(content.slice(start, end), { markup: true }).norm === q.norm &&
!valid.some(v => v.start === start && v.end === end)
) {
valid.push({ start, end });
diff --git a/src/core/cycle/synthesize-verify.ts b/src/core/cycle/synthesize-verify.ts
index 94d6b23376..01db1396a8 100644
--- a/src/core/cycle/synthesize-verify.ts
+++ b/src/core/cycle/synthesize-verify.ts
@@ -202,8 +202,8 @@ export function emptyQuoteVerifyStats(): QuoteVerifyStats {
* source char desynced every later offset and could slice garbage — or
* nothing — back into a page as a "verbatim" repair).
*/
-export function normalizeForGrounding(s: string, opts: { tolerant?: boolean } = {}): { norm: string; map: number[] } {
- return foldForGrounding(s, true, opts.tolerant === true) as { norm: string; map: number[] };
+export function normalizeForGrounding(s: string, opts: { tolerant?: boolean; markup?: boolean } = {}): { norm: string; map: number[] } {
+ return foldForGrounding(s, true, opts.tolerant === true, opts.markup === true) as { norm: string; map: number[] };
}
/**
@@ -216,10 +216,10 @@ export function normalizeForGrounding(s: string, opts: { tolerant?: boolean } =
* Parity matters: the rescue gate and the repair ladder must mean the same
* thing by "normalized substring of the transcript".
*/
-function foldForGrounding(s: string, withMap: boolean, tolerant = false): { norm: string; map: number[] } | string {
+function foldForGrounding(s: string, withMap: boolean, tolerant = false, markup = false): { norm: string; map: number[] } | string {
const out: string[] = [];
const map: number[] = [];
- const skip = foldSkipSet(s, tolerant);
+ const skip = foldSkipSet(s, tolerant, markup);
let pendingSpace = false;
// Iterate by CODE POINT (for..of), not code unit: a surrogate pair
// lowercases as a pair (Deseret 𐐀 → 𐐨) but never half by half, so a
@@ -277,16 +277,53 @@ function bracketMask(s: string): Set<number> | null {
return skip;
}
+const DOUBLE_QUOTE_MARKS = new Set(['"', '“', '”', '„', '«', '»']);
+const SINGLE_QUOTE_MARKS = new Set(["'", '‘', '’']);
+const SPACED_DASHES = new Set(['-', '–', '—']);
+export const isGroundingQuoteMark = (ch: string | undefined): boolean => ch !== undefined && (DOUBLE_QUOTE_MARKS.has(ch) || SINGLE_QUOTE_MARKS.has(ch));
+const isWordChar = (ch: string | undefined): boolean => ch !== undefined && /[\p{L}\p{N}]/u.test(ch);
+const isSpaceOrEdge = (ch: string | undefined): boolean => ch === undefined || /\s/.test(ch);
+const isDigit = (ch: string | undefined): boolean => ch !== undefined && /\p{N}/u.test(ch);
+const nonSpace = (s: string, from: number, step: 1 | -1): string | undefined => { let k = from; while (k >= 0 && k < s.length && /\s/.test(s[k]!)) k += step; return s[k]; };
+
+/**
+ * #8371: code-unit offsets the markup fold also skips, for a quote a model
+ * copied without the source's Markdown and punctuation: line-leading '#' runs
+ * and '>' markers, quotation marks (an apostrophe inside a word stays), and a
+ * dash with whitespace on both sides and no number beside it (a spaced minus or
+ * range is meaning). Words are never skipped.
+ */
+function markupMask(s: string): Set<number> | null {
+ const skip = new Set<number>();
+ let lineStart = true;
+ for (let i = 0; i < s.length; i++) {
+ const ch = s[i]!;
+ if (ch === '\n') { lineStart = true; continue; }
+ if (lineStart) {
+ if (/\s/.test(ch)) continue;
+ if (ch === '>' && (isSpaceOrEdge(s[i + 1]) || s[i + 1] === '>')) { skip.add(i); continue; }
+ let j = i;
+ while (s[j] === '#') j++;
+ if (j > i && isSpaceOrEdge(s[j])) { for (let k = i; k < j; k++) skip.add(k); i = j - 1; continue; }
+ lineStart = false;
+ }
+ if (DOUBLE_QUOTE_MARKS.has(ch) || (SINGLE_QUOTE_MARKS.has(ch) && !(isWordChar(s[i - 1]) && isWordChar(s[i + 1])))) skip.add(i);
+ else if (SPACED_DASHES.has(ch) && isSpaceOrEdge(s[i - 1]) && isSpaceOrEdge(s[i + 1]) && !isDigit(nonSpace(s, i - 1, -1)) && !isDigit(nonSpace(s, i + 1, 1))) skip.add(i);
+ }
+ return skip.size ? skip : null;
+}
+
/**
* Code-unit offsets the fold skips: markdown inline markup (every `*`, every
* backtick, and `~` in a `~~` run) in both modes, so `**sync**`, `` `sync` ``
* and `sync` read alike; plus `bracketMask` in tolerant mode. `_` is kept:
* it is too common in identifiers and paths.
*/
-function foldSkipSet(s: string, tolerant: boolean): Set<number> | null {
+function foldSkipSet(s: string, tolerant: boolean, markup = false): Set<number> | null {
const brackets = tolerant ? bracketMask(s) : null;
- if (!/[*`]|~~/.test(s)) return brackets;
- const skip = brackets ?? new Set<number>();
+ const masked = markup ? [...(markupMask(s) ?? [])].reduce((set, i) => set.add(i), brackets ?? new Set<number>()) : brackets;
+ if (!/[*`]|~~/.test(s)) return masked;
+ const skip = masked ?? new Set<number>();
for (let i = 0; i < s.length; i++) {
const ch = s[i];
if (ch === '*' || ch === '`' || (ch === '~' && (s[i - 1] === '~' || s[i + 1] === '~'))) skip.add(i);
diff --git a/test/cycle/extract-atoms-quote-markup-8373.test.ts b/test/cycle/extract-atoms-quote-markup-8373.test.ts
new file mode 100644
index 0000000000..76cc7c0294
--- /dev/null
+++ b/test/cycle/extract-atoms-quote-markup-8373.test.ts
@@ -0,0 +1,54 @@
+/**
+ * #8371 / #8373 — an atom's source quote that differs from the source only by
+ * Markdown block markers ('## ', '> '), quotation marks or a spaced dash says
+ * the source's exact words. It must locate (provenance kept, offsets into the
+ * ORIGINAL text) instead of being stamped quote_unverified as a paraphrase.
+ * Changed wording still fails closed.
+ */
+import { describe, test, expect } from 'bun:test';
+import { locateQuote } from '../../src/core/cycle/extract-atoms.ts';
+
+function span(content: string, quote: string): string | null {
+ const loc = locateQuote(content, quote);
+ return loc ? content.slice(loc.start, loc.end) : null;
+}
+
+describe('locateQuote tolerates Markdown and punctuation the model drops (#8371)', () => {
+ test('a heading and a blockquote marker', () => {
+ const content = '## Decision\n> We will ship on Friday.';
+ expect(span(content, 'Decision We will ship on Friday.')).toBe('Decision\n> We will ship on Friday.');
+ });
+
+ test('nested blockquote markers and a deeper heading', () => {
+ const content = 'intro\n### Plan\n> > Keep the cap at forty hours.';
+ expect(span(content, 'Plan Keep the cap at forty hours.')).toBe('Plan\n> > Keep the cap at forty hours.');
+ });
+
+ test('inner quotation marks, straight and curly', () => {
+ expect(span('She said "done" today.', 'She said done today.')).toBe('She said "done" today.');
+ expect(span('She said “done” today.', 'She said done today.')).toBe('She said “done” today.');
+ expect(span("He called it 'final' twice.", 'He called it final twice.')).toBe("He called it 'final' twice.");
+ });
+
+ test('a spaced dash, plain or em', () => {
+ expect(span('Cost: 40 hours - approved.', 'Cost: 40 hours approved.')).toBe('Cost: 40 hours - approved.');
+ expect(span('Cost: 40 hours — approved.', 'Cost: 40 hours approved.')).toBe('Cost: 40 hours — approved.');
+ });
+
+ test('apostrophes inside words and unspaced dashes still count', () => {
+ expect(span("We don't ship on Mondays.", 'We dont ship on Mondays.')).toBeNull();
+ expect(span('The budget is 40-50 hours.', 'The budget is 4050 hours.')).toBeNull();
+ });
+
+ test('a sign, a range or a comparison next to a number is meaning, not markup', () => {
+ expect(span('Net margin: − 12% this quarter.', 'Net margin: 12% this quarter.')).toBeNull();
+ expect(span('Net margin: - 12% this quarter.', 'Net margin: 12% this quarter.')).toBeNull();
+ expect(span('Shipping in 10 - 20 days.', 'Shipping in 10 20 days.')).toBeNull();
+ expect(span('We saw 5% churn.', '>5% churn.')).toBeNull();
+ });
+
+ test('changed wording still fails closed', () => {
+ expect(span('## Decision\n> We will ship on Friday.', 'Decision We will not ship on Friday.')).toBeNull();
+ expect(span('She said "done" today.', 'She said finished today.')).toBeNull();
+ });
+});