Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* eval-contradictions/judge — the LLM contradiction judge wrapper.
*
* One-call, one-pair: send Statement A + Statement B + the user's query to
* the chat gateway and parse the verdict JSON. The prompt is the canonical
* text bumped via PROMPT_VERSION when edits land.
*
* Codex fixes incorporated:
* - Query-conditioned: the judge sees the user's query so it can decide
* "contradiction relevant to what was asked" instead of free-form pair
* disagreement (Codex outside-voice finding).
* - Confidence floor double-enforcement (C1): if the model says
* contradicts: true with confidence < 0.7, the orchestrator downgrades
* to false. Belt-and-suspenders against models that ignore the prompt.
* - judge_errors as first-class: throws are typed and counted in the
* denominator — see judge-errors.ts for the collector shape.
*
* Provider-neutral via the gateway. Hermetically testable via
* gateway.__setChatTransportForTests.
*/
import { chat, type ChatResult } from '../ai/gateway.ts';
import { parseSeverity, defaultSeverityForVerdict } from './severity-classify.ts';
import type { JudgeVerdict, ResolutionKind, Verdict } from './types.ts';
const FENCE_RE = /```(?:json)?\s*\n?([\s\S]*?)```/i;
const FENCE_RE_GLOBAL = /```(?:json)?\s*\n?([\s\S]*?)```/gi;
function repairJsonish(text: string): string {
return text
.replace(FENCE_RE_GLOBAL, (_, inner) => inner)
.replace(/,(\s*[}\]])/g, '$1')
.replace(/(['"])?([\w-]+)\1?\s*:/g, '"$2":')
.trim();
}
function* jsonValueCandidates(text: string): Generator<string> {
for (let start = 0; start < text.length; start++) {
const opener = text[start];
if (opener !== '{' && opener !== '[') continue;
const closer = opener === '{' ? '}' : ']';
const stack: string[] = [closer];
let inString = false;
let escaped = false;
for (let i = start + 1; i < text.length; i++) {
const ch = text[i];
if (inString) {
if (escaped) {
escaped = false;
} else if (ch === '\\') {
escaped = true;
} else if (ch === '"') {
inString = false;
}
continue;
}
if (ch === '"') {
inString = true;
continue;
}
if (ch === '{') {
stack.push('}');
} else if (ch === '[') {
stack.push(']');
} else if (ch === stack[stack.length - 1]) {
stack.pop();
if (stack.length === 0) {
yield text.slice(start, i + 1);
break;
}
} else if (ch === '}' || ch === ']') {
break;
}
}
}
}
function tryParseJSON(text: string): unknown | null {
try {
return JSON.parse(text);
} catch {
return null;
}
}
/**
* Generic 4-strategy LLM JSON parser. Throws when no strategy works rather
* than fabricating an empty object — caller maps to judge_errors.parse_fail.
*
* (We don't reuse parseModelJSON from cross-modal-eval because that one is
* shape-specific to {scores, overall, improvements} and rejects our verdict
* payload. Same 4-strategy spirit, narrower contract.)
*/
export function parseJudgeJSON(text: string): unknown {
if (!text) throw new Error('parseJudgeJSON: empty response');
// Strategy 1: direct parse (strict JSON).
const direct = tryParseJSON(text);
if (direct !== null) return direct;
// Strategy 2: strip ```json fences.
const fenceMatch = text.match(FENCE_RE);
if (fenceMatch && fenceMatch[1]) {
const fenced = tryParseJSON(fenceMatch[1].trim());
if (fenced !== null) return fenced;
}
// Strategy 3: common-repairs pass — trailing commas, single→double quotes.
const cleaned = repairJsonish(text);
const repaired = tryParseJSON(cleaned);
if (repaired !== null) return repaired;
// Strategy 4: find the first balanced JSON object/array inside prose.
for (const candidate of jsonValueCandidates(cleaned)) {
const parsed = tryParseJSON(candidate);
if (parsed !== null) return parsed;
}
throw new Error('parseJudgeJSON: all strategies failed');
}
/** Default per-pair text budget (UTF-8-safe truncation). C4 default. */
export const DEFAULT_MAX_PAIR_CHARS = 1500;
// v0.42.0.0: truncateUtf8 lives in src/core/text-safe.ts (shared with the
// dream-cycle chunker's safeSplitIndex). Imported here for the local
// `buildJudgePrompt` use AND re-exported for back-compat with anything
// importing it from this module.
import { truncateUtf8 } from '../text-safe.ts';
export { truncateUtf8 };
export interface JudgeInput {
/** The user's query for the search that retrieved both members. */
query: string;
/**
* Statement A: slug + text + optional source-tier + holder (if take) +
* optional effective_date (Lane A1). When effective_date is null/undefined
* the prompt shows `(date unknown)` for that side; the judge classifies
* based on chunk text alone, same as the v1 prompt did.
*/
a: {
slug: string;
text: string;
source_tier?: string;
holder?: string | null;
effective_date?: string | null;
};
b: {
slug: string;
text: string;
source_tier?: string;
holder?: string | null;
effective_date?: string | null;
};
/** Provider:model id; routed through gateway.chat. */
model: string;
/** UTF-8-safe truncation limit per pair member. C4 flag. */
maxPairChars?: number;
/** Test hook: pass a stubbed chat for hermetic tests. Production passes undefined → real gateway. */
chatFn?: typeof chat;
/** Abort signal for cancellation. */
abortSignal?: AbortSignal;
}
export interface JudgeOutput {
verdict: JudgeVerdict;
/** Token usage from the gateway. Forwarded to the cost tracker. */
usage: { inputTokens: number; outputTokens: number };
}
/**
* Validated resolution_kind values. Anything outside this set defaults to
* 'manual_review' (the safe, no-action option for contradictions; new verdicts
* route through auto-supersession.ts when resolution_kind is null on input).
*
* v0.34 / Lane A2: added temporal_supersede, flag_for_review, log_timeline_change.
*/
function parseResolutionKind(value: unknown): ResolutionKind | null {
if (
value === 'takes_supersede' ||
value === 'dream_synthesize' ||
value === 'takes_mark_debate' ||
value === 'manual_review' ||
value === 'temporal_supersede' ||
value === 'flag_for_review' ||
value === 'log_timeline_change'
) {
return value;
}
return null;
}
const VALID_VERDICTS: ReadonlySet<Verdict> = new Set([
'no_contradiction',
'contradiction',
'temporal_supersession',
'temporal_regression',
'temporal_evolution',
'negation_artifact',
]);
/** Validate a verdict string from JSON; throws on missing/invalid so caller maps to parse_fail. */
export function parseVerdict(value: unknown): Verdict {
if (typeof value !== 'string' || !VALID_VERDICTS.has(value as Verdict)) {
throw new Error(`judge JSON missing or invalid verdict: ${JSON.stringify(value)}`);
}
return value as Verdict;
}
/**
* Validate the raw parsed JSON against the JudgeVerdict shape. Throws on
* fundamentally-broken shape (missing verdict/confidence) so the caller
* counts it under judge_errors.parse_fail rather than fabricating a verdict.
*
* v0.34 / Lane A2: parses the new `verdict: Verdict` enum field instead of
* the v1 `contradicts: boolean`. PROMPT_VERSION = '2' (bumped in A1) means
* the persistent cache won't return v1-shaped rows for these calls.
*
* C1 enforcement: `verdict === 'contradiction'` with confidence < 0.7 is
* downgraded to `'no_contradiction'` (belt-and-suspenders against models
* ignoring the prompt rule). The 5 non-contradiction verdicts do NOT have a
* confidence floor — they're informational classifications, not error flags.
*/
export function normalizeVerdict(raw: unknown): JudgeVerdict {
if (Array.isArray(raw)) {
if (raw.length !== 1) {
throw new Error('judge JSON array must contain exactly one verdict object');
}
raw = raw[0];
}
if (!raw || typeof raw !== 'object') {
throw new Error('judge JSON missing or not an object');
}
const v = raw as Record<string, unknown>;
// Parse verdict first so we can throw a useful error before checking other
// fields. v1-shaped `contradicts: true/false` responses are accepted as a
// repair for small/local models that understand the task but drift from the
// current JSON field name.
let verdict: Verdict;
if (v.verdict !== undefined) {
verdict = parseVerdict(v.verdict);
} else if (typeof v.contradicts === 'boolean') {
verdict = v.contradicts ? 'contradiction' : 'no_contradiction';
} else if (typeof v.contradiction === 'boolean') {
verdict = v.contradiction ? 'contradiction' : 'no_contradiction';
} else {
verdict = parseVerdict(v.verdict);
}
const rawConfidence =
typeof v.confidence === 'string' && v.confidence.trim() !== ''
? Number(v.confidence)
: v.confidence;
if (typeof rawConfidence !== 'number' || !Number.isFinite(rawConfidence)) {
throw new Error('judge JSON missing or invalid confidence');
}
const clampedConfidence = Math.min(1, Math.max(0, rawConfidence));
const axisRaw = typeof v.axis === 'string' ? v.axis : '';
const resolutionKind = parseResolutionKind(v.resolution_kind);
// C1 double-enforce: only `verdict === 'contradiction'` carries the
// confidence floor. Downgrade to no_contradiction below the threshold.
if (verdict === 'contradiction' && clampedConfidence < 0.7) {
verdict = 'no_contradiction';
}
// Severity: judge can set it; if invalid, fall back to the default for the
// verdict (D7 map: temporal_supersession→info, temporal_regression→high, ...).
// parseSeverity coerces unknown strings to 'low' historically, but now we
// route through defaultSeverityForVerdict instead so each verdict gets a
// meaningful default.
const severity = parseSeverity(v.severity, defaultSeverityForVerdict(verdict));
const isFinding = verdict !== 'no_contradiction';
// Only `contradiction` keeps the v1 fallback to 'manual_review' when the
// judge omits a resolution_kind. The new verdicts pass through whatever the
// judge said (or null) and auto-supersession.ts picks the kind based on
// verdict semantics.
let normalizedResolutionKind: ResolutionKind | null;
if (verdict === 'contradiction') {
normalizedResolutionKind = resolutionKind ?? 'manual_review';
} else if (isFinding) {
normalizedResolutionKind = resolutionKind ?? null;
} else {
normalizedResolutionKind = null;
}
return {
verdict,
severity,
axis: isFinding ? axisRaw : '',
confidence: clampedConfidence,
resolution_kind: normalizedResolutionKind,
};
}
/**
* Build the judge prompt. Query-conditioned (Codex fix) — the model sees
* what the user actually asked so it can decide whether the disagreement is
* relevant to the query.
*
* Holder is shown when present (take pairs): "Garry holds X" vs "Garry
* holds not-X" is a flip; "Alice holds X" vs "Bob holds not-X" is not.
*/
export function buildJudgePrompt(opts: {
query: string;
a: {
slug: string;
text: string;
source_tier?: string;
holder?: string | null;
effective_date?: string | null;
};
b: {
slug: string;
text: string;
source_tier?: string;
holder?: string | null;
effective_date?: string | null;
};
maxPairChars: number;
}): string {
const a = truncateUtf8(opts.a.text, opts.maxPairChars);
const b = truncateUtf8(opts.b.text, opts.maxPairChars);
const aMeta = [opts.a.slug, opts.a.source_tier && `source-tier ${opts.a.source_tier}`, opts.a.holder && `holder ${opts.a.holder}`].filter(Boolean).join(', ');
const bMeta = [opts.b.slug, opts.b.source_tier && `source-tier ${opts.b.source_tier}`, opts.b.holder && `holder ${opts.b.holder}`].filter(Boolean).join(', ');
// Lane A1: emit the page-level effective_date on its own line so the judge
// can reason temporally. `(date unknown)` keeps the v1 fallback behavior
// when the page has no effective_date — judge classifies on text alone.
const aDateTag = opts.a.effective_date ? `(from: ${opts.a.effective_date})` : '(date unknown)';
const bDateTag = opts.b.effective_date ? `(from: ${opts.b.effective_date})` : '(date unknown)';
return [
'You are a contradiction judge for a personal knowledge brain. The user',
'ran a search and got two results back. Decide whether the two statements',
"contradict each other in a way that would mislead someone trying to",
"answer the user's query.",
'',
`User's query: ${opts.query}`,
'',
`Statement A ${aDateTag} (${aMeta}):`,
a,
'',
`Statement B ${bDateTag} (${bMeta}):`,
b,
'',
'Rules:',
'- The (from: YYYY-MM-DD) tag is the page-level effective date. Use it to',
' classify what kind of difference this is, not just whether it exists.',
' (date unknown) means the page has no temporal anchor — judge on text',
' alone for that side.',
'- Time check first. The three temporal verdicts need evidence of two',
' DIFFERENT times: different (from:) dates, different dates written in',
' the text, or explicit change wording ("now", "previously", "grew from").',
' Never infer an order from the values themselves (a larger or smaller',
' number), from the kind of document (memo, deck, call notes), or from',
' which statement is listed first. Without that evidence, two values for',
' the same fact are a contradiction, not a change over time.',
'- A conflict or a change needs a value for the same attribute on BOTH',
' sides. Silence is not a conflict: when one statement does not mention',
' the attribute the other states (it lists other facts), use',
' no_contradiction.',
'- Pick exactly one verdict from the six values below.',
'- Use temporal_supersession when the newer-dated claim updates or replaces',
' the older one (role change, status change). Not an error.',
'- Use temporal_regression when a metric or status went BACKWARDS over time',
' (e.g., MRR dropped from $200K to $150K). This is a signal worth flagging.',
'- Use temporal_evolution for legitimate change over time that is neither',
' supersession nor regression (e.g., evolving narrative, multi-step decision).',
'- Use negation_artifact when one side contains an explicit negation that',
' the surface tokens make look like a positive claim (e.g., "NOT X" parsed',
' as "X"). The data is correct; the apparent conflict is a parsing artifact.',
'- Use contradiction ONLY for genuinely conflicting claims at the same point',
' in time, where the dates do not explain the difference.',
'- Use no_contradiction when the statements are compatible.',
'- When both sides carry the same date, or nothing dates either side,',
' time does not explain the difference: two values for the same fact are',
' a contradiction.',
'- Only claims about the SAME entity can conflict. Two entities whose names',
' merely look alike (e.g. "Northwind" vs "Northgate") are',
' different entities: use no_contradiction for their differing values.',
'- A negative claim about one party and a positive claim about a different',
' party agree ("Fund A is not leading the round" and "Fund B is leading',
' the round"): use no_contradiction.',
'',
'- Subjective opinions held at different times by the SAME holder may be',
' a contradiction (a flip). Opinions held by DIFFERENT holders are not.',
'- Different aspects of the same entity are not contradictions.',
"- Incidental disagreements unrelated to the user's query do not count.",
' Judge only on claims relevant to what the user asked.',
'',
'Reply with JSON ONLY:',
'{',
' "verdict": "no_contradiction" | "contradiction" | "temporal_supersession" | "temporal_regression" | "temporal_evolution" | "negation_artifact",',
' "severity": "info" | "low" | "medium" | "high",',
' "axis": "<one-line: what they disagree about, or empty>",',
' "confidence": 0.0..1.0,',
' "resolution_kind": "takes_supersede" | "dream_synthesize" | "takes_mark_debate" | "manual_review" | "temporal_supersede" | "flag_for_review" | "log_timeline_change" | null',
'}',
'',
'Severity rubric:',
'- info: temporal_supersession and temporal_evolution (not errors; informational).',
'- low: naming/format differences (Alice Smith vs A. Smith); negation artifacts.',
'- medium: factual values that may be stale (revenue, headcount).',
'- high: identity / structural claims (founder/CEO/CFO role); temporal_regression.',
'',
'Reply verdict:contradiction only when confidence >= 0.7. Other verdicts have',
'no confidence floor.',
].join('\n');
}
/** Detect refusal-shaped responses. Caller maps to judge_errors.refusal. */
function isRefusalResponse(result: ChatResult): boolean {
if (result.stopReason === 'refusal') return true;
const txt = result.text?.toLowerCase?.() ?? '';
return (
txt.includes("i can't help") ||
txt.includes('i cannot help') ||
txt.includes('refuse to answer')
);
}
/**
* Main entry. Calls the gateway, parses JSON, normalizes the verdict with
* C1 confidence enforcement. Throws on parse / refusal / transport errors;
* caller wraps in try/catch and records via JudgeErrorCollector.
*/
export async function judgeContradiction(input: JudgeInput): Promise<JudgeOutput> {
const maxPairChars = input.maxPairChars ?? DEFAULT_MAX_PAIR_CHARS;
// input.a/b carry effective_date through PairMember (Lane A1); buildJudgePrompt
// emits it on the Statement line or falls through to `(date unknown)`.
const prompt = buildJudgePrompt({
query: input.query,
a: input.a,
b: input.b,
maxPairChars,
});
const callFn = input.chatFn ?? chat;
const result = await callFn({
model: input.model,
messages: [{ role: 'user', content: prompt }],
maxTokens: 1024,
abortSignal: input.abortSignal,
allowFallback: false,
});
if (isRefusalResponse(result)) {
throw new Error('judge refused to answer');
}
const raw = parseJudgeJSON(result.text);
const verdict = normalizeVerdict(raw);
return {
verdict,
usage: {
inputTokens: result.usage.input_tokens ?? 0,
outputTokens: result.usage.output_tokens ?? 0,
},
};
}