Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* UTF-16-surrogate-safe text helpers.
*
* Shared between:
* - `src/core/eval-contradictions/judge.ts` — `truncateUtf8` truncates
* contradiction-judge prompt inputs to a per-pair char budget without
* splitting emoji / non-BMP CJK / mathematical alphanumerics.
* - `src/core/cycle/synthesize.ts` — `safeSplitIndex` is the tier-3
* hard-split fallback in the dream-cycle chunker. Preserves the D9
* stable-chunk-identity invariant by refusing to orphan a high surrogate.
*
* Two consumers, two natural shapes:
* - `truncateUtf8(text, maxChars) -> string` returns the sliced text.
* - `safeSplitIndex(text, maxChars) -> number` returns the boundary index
* without allocating the sliced string (cheaper chunker hot path).
*
* Both functions cover the same three surrogate cases. The agent-authored
* `safeSliceEnd` from PRs #1378-#1382 handled only case 1; the AT-low
* surrogate case (3) silently bit when a chunk boundary landed one
* position inside an emoji.
*
* UTF-16 surrogate ranges:
* high surrogate: U+D800..U+DBFF
* low surrogate: U+DC00..U+DFFF
*/
/**
* Make a string well-formed UTF-16: replace any unpaired surrogate half (lone
* high U+D800–U+DBFF / lone low U+DC00–U+DFFF) with U+FFFD, preserving valid
* pairs untouched. A lone surrogate is rejected by Postgres inside a `::jsonb`
* cast and aborts the WHOLE batch (#2011 — `extract --stale` died at ~1,550
* pages because `excerpt()` raw-sliced a window boundary through an emoji); it
* also crashes some JSON transports (the brainstorm cross-prompt path).
*
* Uses the Bun/JSC built-in (ES2024). `isWellFormed()` is the cheap guard that
* avoids the `toWellFormed()` allocation when the string is already clean — the
* common case. Prefer this over hand-rolled surrogate regexes: the two-pass
* lookbehind regex mishandles CONSECUTIVE lone low surrogates
* (`"\uDE80\uDE80"` → `"�\uDE80"`, still malformed), whereas the built-in
* handles every case (`→ "��"`).
*/
export function ensureWellFormed(s: string): string {
return s.isWellFormed() ? s : s.toWellFormed();
}
/**
* `ensureWellFormed` over every string (and object key) inside a JSON-shaped
* value, for payloads written to JSONB columns: one lone surrogate anywhere in
* the value makes Postgres reject the whole write (22P02, #6860).
*/
export function wellFormedJson<T>(value: T): T {
if (typeof value === 'string') return ensureWellFormed(value) as T;
if (Array.isArray(value)) return value.map(wellFormedJson) as T;
if (value && typeof value === 'object') {
return Object.fromEntries(Object.entries(value).map(([key, item]) => [ensureWellFormed(key), wellFormedJson(item)])) as T;
}
return value;
}
/**
* #7240: `wellFormedJson` plus U+0000 → U+FFFD in every string and key, the other input Postgres refuses inside a
* JSONB value (22P05). One such character in a job payload refused the whole insert, so the phase that submitted it
* failed and cancelled every child it had queued.
*/
export function jsonbSafeJson<T>(value: T): T {
if (typeof value === 'string') return ensureWellFormed(value).replaceAll('\u0000', '\uFFFD') as T;
if (Array.isArray(value)) return value.map(jsonbSafeJson) as T;
if (value && typeof value === 'object') {
return Object.fromEntries(Object.entries(value).map(([key, item]) => [jsonbSafeJson(key), jsonbSafeJson(item)])) as T;
}
return value;
}
function isHighSurrogate(code: number): boolean {
return code >= 0xd800 && code <= 0xdbff;
}
function isLowSurrogate(code: number): boolean {
return code >= 0xdc00 && code <= 0xdfff;
}
/**
* UTF-8-safe truncation: cap at maxChars but never split a multi-byte
* character. Returns the text unchanged if already under the limit.
*
* Pattern reused from `src/core/minions/handlers/subagent-audit.ts` which
* faces the same multi-byte concern.
*/
export function truncateUtf8(text: string, maxChars: number): string {
if (!text) return '';
if (text.length <= maxChars) return text;
return text.slice(0, Math.max(0, safeSplitIndex(text, maxChars)));
}
/**
* Return the largest safe slice index ≤ maxChars (never orphans a UTF-16
* surrogate). Used by chunkers that need the index itself, not the
* truncated string.
*
* Three back-up cases (mirrors `truncateUtf8`'s implementation exactly,
* so the two functions cannot drift):
* 1. Pair STRADDLES the cut: high at maxChars-1, low at maxChars.
* Return maxChars-1 (pair starts the next chunk together).
* 2. Stray high at maxChars-1 (no paired low). Return maxChars-1.
* 3. Low at maxChars-1 (we're inside a pair that started at maxChars-2).
* Return maxChars-2 (whole pair moves to next chunk).
*
* Case 3 is intentionally conservative when the pair is COMPLETE in the
* kept half (e.g. text="abcd🚀", maxChars=6 returns 4, not 6). The 2-char
* shortfall is harmless for chunker determinism — same input → same
* chunks every time — and matches truncateUtf8's well-tested behavior.
* Fixing the conservative back-up would require diverging the two
* functions; we deliberately match them.
*/
export function safeSplitIndex(text: string, maxChars: number): number {
if (maxChars <= 0) return 0;
if (maxChars >= text.length) return text.length;
const unitAtEnd = text.charCodeAt(maxChars);
const unitBefore = text.charCodeAt(maxChars - 1);
// Case 1: pair straddles the cut.
if (isHighSurrogate(unitBefore) && isLowSurrogate(unitAtEnd)) {
return maxChars - 1;
}
// Case 2: stray high surrogate.
if (isHighSurrogate(unitBefore)) {
return maxChars - 1;
}
// Case 3: kept half ends at low surrogate; back up two.
if (isLowSurrogate(unitBefore)) {
return Math.max(0, maxChars - 2);
}
return maxChars;
}