Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.36.1.0 (T4) — grade_takes cycle phase.
*
* Walks unresolved takes that are old enough to have outcome data, retrieves
* evidence from the brain, asks a judge model to verdict each one. Writes
* verdicts to take_grade_cache. Optionally — only when operator has flipped
* the opt-in config flag — auto-applies high-confidence verdicts to the
* canonical takes table via engine.resolveTake.
*
* Auto-resolve posture (D17 — auto-resolve disabled by default):
* On a fresh install, grade_takes runs and writes verdicts to the cache,
* but `applied=false` on every row. Operator reviews the queue, then flips
* `cycle.grade_takes.auto_resolve.enabled: true` once trust is earned.
*
* Conservative threshold (D12):
* When auto_resolve.enabled is true, a verdict auto-applies only when
* confidence >= 0.95 (single-judge path; T5 ensemble path tightens this
* further). Schema enforces monotonic config tightening: tightening
* thresholds is always free, loosening requires --allow-loosen-confidence
* flag because relaxing after data accumulates silently shifts which
* historical resolutions count as auto-applied.
*
* Evidence retrieval (#2811):
* The default evidence retriever runs a real hybrid search on the take's
* claim (expansion off, source-scoped, the take's own page excluded) and
* formats the hits via the pure `formatEvidenceBlock` — each item carries
* its slug + effective_date and is annotated relative to the take's
* since_date (evidence dated BEFORE the claim is context, not outcome).
* Items clamp at ~500 chars; the whole block caps at 4k. Zero hits and
* retrieval failures fall back to explicit notes steering the judge to
* 'unresolvable' rather than fabricating.
*
* Test seam: opts.judge + opts.evidenceRetriever are injected so the
* phase runs hermetically in unit tests.
*/
import { createHash } from 'node:crypto';
import { BaseCyclePhase, effectivePhaseDeadlineMs, type ScopedReadOpts, type BasePhaseOpts } from './base-phase.ts';
import { hybridSearch } from '../search/hybrid.ts';
import { INTERNAL_BREADTH_SEARCH_OPTS } from '../search/internal-breadth.ts';
import type { SearchResult } from '../types.ts';
import { chat as gatewayChat, getChatModel } from '../ai/gateway.ts';
import { createGlobalLlmHaltTracker, haltedClassOf, type GlobalLlmErrorClass } from '../ai/errors.ts';
import { splitProviderModelId } from '../model-id.ts';
import { GBrainError } from '../types.ts';
import type { OperationContext } from '../operations.ts';
import type { BrainEngine, Take, TakeResolution } from '../engine.ts';
import type { PhaseStatus, CyclePhase } from '../cycle.ts';
import { managedPersistenceEnabled } from '../persistence/ownership.ts';
import { parseTakesFence } from '../takes-fence.ts';
import { submitPageMutation } from '../persistence/page-mutations.ts';
import { maintenanceTransaction } from '../persistence/attribution.ts';
import { utcDate } from './cycle-date.ts';
import { isConfigTruthy } from '../config.ts';
/**
* Bump when the judge prompt or the JSON output shape changes. Old verdicts
* stay valid (composite cache key includes prompt_version); new runs re-spend
* LLM tokens.
*/
export const GRADE_TAKES_PROMPT_VERSION = 'hybrid-evidence-v2';
export const GRADE_TAKE_PROMPT = `[hybrid-evidence-v2] You are grading a single forecasting take. The author
made this claim on the given date. Based on the evidence provided, did the
claim turn out to be:
- correct (the world plays out as predicted)
- incorrect (the world clearly contradicts the prediction)
- partial (some aspects right, some wrong; or right direction wrong magnitude)
- unresolvable (insufficient evidence; outcome still pending)
Output ONLY one JSON object with these fields:
- verdict ('correct' | 'incorrect' | 'partial' | 'unresolvable')
- confidence (number in [0,1]) — your self-reported confidence in this verdict.
- reasoning (string, <=400 chars) — one short paragraph explaining what evidence drove the verdict.
If the evidence is sparse or ambiguous, return verdict='unresolvable' with
confidence reflecting the lack of evidence (NOT certainty of unresolvable).
Only evidence about the subject the claim names counts. A different company,
person or product whose name merely resembles it (for example "Acme Health"
for a claim about "Acme") is not evidence about the claim; when only such
evidence speaks to the outcome, return 'unresolvable'.
TAKE:
Claim: {CLAIM}
Kind: {KIND}
Holder: {HOLDER}
Made on: {SINCE_DATE}
Weight: {WEIGHT}
EVIDENCE:
{EVIDENCE_BLOCK}
`;
/** Verdict from a single judge model. */
export interface JudgeVerdict {
verdict: 'correct' | 'incorrect' | 'partial' | 'unresolvable';
confidence: number;
reasoning: string;
}
/** Judge function signature — injected for tests. */
export type JudgeFn = (input: {
take: Take;
evidence: string;
modelHint?: string;
}) => Promise<JudgeVerdict>;
/**
* Multi-judge ensemble verdict aggregation (E2, T5).
*
* Per D17 + D12 conservative posture: an ensemble verdict auto-applies only
* when ALL three model verdicts agree AND the minimum confidence across the
* three is >= the ensemble threshold (default 0.85). Anything less → cache
* with applied=false (review-queue posture).
*
* 'unresolvable' verdicts NEVER count toward consensus (a single
* 'unresolvable' result drops the agreement count). This is intentional —
* one model saying "I can't tell" plus two saying "correct" should NOT
* auto-apply 'correct'.
*/
export interface EnsembleVerdict {
verdict: JudgeVerdict['verdict'];
minConfidence: number;
agreement: number; // 0..3, count of models that returned this verdict
modelVerdicts: Array<{ modelId: string; verdict: JudgeVerdict['verdict']; confidence: number; failed?: boolean }>;
}
/**
* Aggregate per-model verdicts into an EnsembleVerdict. Pure function.
*
* Algorithm:
* 1. Filter out failed model responses (rejected promises in the caller).
* 2. Tally verdict labels.
* 3. Winner = label with the most votes. Ties: 'unresolvable' loses; any
* other label wins via deterministic alphabetical order.
* 4. agreement = count of models that returned the winning label.
* 5. minConfidence = MIN across the models that returned the winning label.
*
* Caller decides whether to auto-apply based on the (agreement === 3 AND
* minConfidence >= threshold) rule.
*/
export function aggregateEnsemble(
results: Array<{ modelId: string; verdict: JudgeVerdict | null }>,
): EnsembleVerdict {
const modelVerdicts: EnsembleVerdict['modelVerdicts'] = results.map(r =>
r.verdict
? { modelId: r.modelId, verdict: r.verdict.verdict, confidence: r.verdict.confidence }
: { modelId: r.modelId, verdict: 'unresolvable', confidence: 0, failed: true },
);
// Tally only the non-failed verdicts.
const tally = new Map<JudgeVerdict['verdict'], number>();
for (const r of results) {
if (!r.verdict) continue;
tally.set(r.verdict.verdict, (tally.get(r.verdict.verdict) ?? 0) + 1);
}
// Pick the winner. Tie-break: prefer non-unresolvable, then alphabetical
// for determinism.
let winner: JudgeVerdict['verdict'] = 'unresolvable';
let bestCount = 0;
for (const [v, n] of tally.entries()) {
if (n > bestCount) {
winner = v;
bestCount = n;
} else if (n === bestCount) {
// Tie. Prefer non-unresolvable.
if (winner === 'unresolvable' && v !== 'unresolvable') {
winner = v;
} else if (v !== 'unresolvable' && winner !== 'unresolvable' && v < winner) {
winner = v;
}
}
}
// minConfidence: min across the models that returned the winning label.
let minConfidence = 1;
let agreementCount = 0;
for (const r of results) {
if (r.verdict && r.verdict.verdict === winner) {
agreementCount += 1;
if (r.verdict.confidence < minConfidence) minConfidence = r.verdict.confidence;
}
}
if (agreementCount === 0) minConfidence = 0;
return { verdict: winner, minConfidence, agreement: agreementCount, modelVerdicts };
}
/** Evidence retriever signature — injected for tests. */
export type EvidenceRetrieverFn = (take: Take, scope: ScopedReadOpts) => Promise<string>;
export interface GradeTakesOpts extends BasePhaseOpts {
/** Override the phase wall-clock deadline (tests). Default: 30 min, clamped to the job deadline (gbrain#4168). */
deadlineMs?: number;
/** Minimum age in months before a take is eligible for grading. Default 6. */
minAgeMonths?: number;
/** Limit takes processed in this cycle. Default 50. */
takeLimit?: number;
/** Inject the judge model call (tests). */
judge?: JudgeFn;
/** Inject the evidence retriever (tests). */
evidenceRetriever?: EvidenceRetrieverFn;
/** Override prompt_version (tests). */
promptVersion?: string;
/** Judge model id; defaults to the configured chat model. */
model?: string;
/**
* Auto-resolve verdicts above the confidence threshold. Unset: the
* `cycle.grade_takes.auto_resolve.enabled` config key (D17 default: false).
* When false, every verdict lands in take_grade_cache with applied=false
* (review-queue posture). When true, verdicts with confidence >= the
* configured threshold get applied via engine.resolveTake.
*/
autoResolve?: boolean;
/**
* Confidence threshold for auto-resolve. D12 default: 0.95. Schema-level
* monotonic-tightening guard (loosening requires --allow-loosen-confidence)
* lives in the takes resolution layer, not here.
*/
autoResolveThreshold?: number;
/** Identifier recorded as resolved_by when auto-applying. Default 'gbrain:grade_takes'. */
resolvedByLabel?: string;
/**
* v0.36.1.0 (T11 / E4) — gstack-learnings coupling on incorrect/partial
* auto-resolutions. Config gate: `cycle.grade_takes.write_gstack_learnings`.
* Default false for external users (gstack may not be installed); Garry's
* brain flips it true to opt in. Failures are non-fatal (warning).
*/
writeGstackLearnings?: boolean;
/**
* E2 ensemble (T5): when true, borderline single-model verdicts
* (0.6 <= confidence < 0.95) fire a 3-model ensemble tiebreaker. Default
* false (single-model only).
*/
useEnsemble?: boolean;
/**
* E2 ensemble judges. When useEnsemble=true and the single-model verdict
* is borderline, all three judges are called in parallel via Promise.allSettled.
* Defaults to [openai:gpt-5.2, anthropic:claude-sonnet-4-6, google:gemini-2.5-flash]
* via defaultJudge with model-string overrides. Tests inject deterministic
* judges.
*/
ensembleJudges?: Array<{ modelId: string; fn: JudgeFn }>;
/**
* E2 ensemble auto-apply threshold. Default 0.85 (D12 conservative): MIN
* confidence across the agreeing models must be >= this AND agreement
* must be 3/3 unanimous.
*/
ensembleThreshold?: number;
/**
* E2 ensemble TRIGGER band [lower, upper). Single-model verdicts whose
* confidence falls in this band invoke the ensemble. Default [0.6, 0.95).
* Below the lower bound: single is clearly unresolvable / review-only.
* Above the upper bound: single is sufficient.
*/
ensembleTriggerBand?: [number, number];
}
export interface GradeTakesResult {
takes_scanned: number;
cache_hits: number;
verdicts_written: number;
auto_applied: number;
too_recent: number;
budget_exhausted: boolean;
/**
* Set when the take loop broke on a whole-run LLM failure (#3044):
* auth/billing on the first hit, rate_limit after RATE_LIMIT_HALT_STREAK
* consecutive hits — from the single-model judge OR a rejected ensemble
* judge. The phase reports 'warn' ('fail' when NO judge call succeeded)
* so the condition can't hide behind a green summary.
*/
aborted_global_error?: GlobalLlmErrorClass;
/** Single-model judge calls that returned (cache hits don't count). */
judge_calls_succeeded: number;
/** Single-model judge calls that threw (global or per-take alike). */
judge_calls_failed: number;
warnings: string[];
/** E2 ensemble (T5): count of takes where the ensemble tiebreaker fired. */
ensemble_invoked: number;
/** E2 ensemble (T5): count of takes where ensemble produced 3/3 unanimous. */
ensemble_unanimous: number;
/** gbrain#4168: true when the phase deadline fired mid-loop (partial result). */
deadline_hit: boolean;
}
/**
* Compute the evidence_signature for the cache. SHA-256 of evidence text +
* judge_model_id keeps the cache invalidation honest: re-running with new
* evidence OR a different judge produces a fresh row.
*/
export function evidenceSignature(evidence: string, judgeModelId: string): string {
return createHash('sha256').update(judgeModelId + '|' + evidence).digest('hex');
}
/**
* Parse the judge model's JSON output. Tolerant of fence wrapping and of
* prose before or after the object; returns null on unrecoverable parse
* failure or on two answers that disagree.
*/
export function parseJudgeOutput(raw: string): JudgeVerdict | null {
if (!raw || raw.trim().length === 0) return null;
let text = raw.trim();
const fenced = text.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/);
if (fenced) text = (fenced[1] ?? '').trim();
const verdicts = topLevelObjects(text).map(toVerdict).filter((v): v is JudgeVerdict => v !== null);
if (verdicts.length === 0) return null;
// W6: a judge sometimes answers twice ("Correction: ..."), and prose after
// the object used to fail the whole parse. Two answers that disagree are no
// verdict (retried next cycle, as #3910 does for any unparseable answer);
// answers that agree keep the lowest confidence, so a repeat can never raise
// what auto-resolve sees.
if (verdicts.some(v => v.verdict !== verdicts[0]!.verdict)) return null;
const last = verdicts[verdicts.length - 1]!;
return { ...last, confidence: Math.min(...verdicts.map(v => v.confidence)) };
}
/** Every top-level `{...}` in the text that parses as JSON, in order. */
function topLevelObjects(text: string): unknown[] {
const out: unknown[] = [];
let depth = 0;
let start = -1;
let inString = false;
let escaped = false;
for (let i = 0; i < text.length; i++) {
const ch = text[i];
if (inString) {
if (escaped) escaped = false;
else if (ch === '\\') escaped = true;
else if (ch === '"') inString = false;
continue;
}
if (ch === '"' && depth > 0) inString = true;
else if (ch === '{') { if (depth++ === 0) start = i; }
else if (ch === '}' && depth > 0 && --depth === 0) {
try { out.push(JSON.parse(text.slice(start, i + 1))); } catch { /* not JSON: skip */ }
}
}
return out;
}
function toVerdict(parsed: unknown): JudgeVerdict | null {
if (typeof parsed !== 'object' || parsed === null) return null;
const r = parsed as Record<string, unknown>;
const validVerdicts = ['correct', 'incorrect', 'partial', 'unresolvable'] as const;
const verdict = validVerdicts.includes(r.verdict as never) ? (r.verdict as JudgeVerdict['verdict']) : null;
if (!verdict) return null;
const confRaw = typeof r.confidence === 'number' ? r.confidence : Number.parseFloat(String(r.confidence ?? ''));
if (!Number.isFinite(confRaw)) return null;
const confidence = Math.max(0, Math.min(1, confRaw));
const reasoning = typeof r.reasoning === 'string' ? r.reasoning.slice(0, 400) : '';
return { verdict, confidence, reasoning };
}
/** #2811 — evidence retrieval knobs. */
/** Auto-resolve applies a single-judge verdict at or above this confidence (D12). */
export const GRADE_TAKES_AUTO_RESOLVE_THRESHOLD = 0.95;
export const EVIDENCE_SEARCH_LIMIT = 8;
export const EVIDENCE_ITEM_CLAMP_CHARS = 500;
export const EVIDENCE_BLOCK_CAP_CHARS = 4000;
/** The narrow slice of a SearchResult the evidence formatter reads. */
export type EvidenceHit = Pick<SearchResult, 'slug' | 'title' | 'chunk_text' | 'effective_date'>;
/**
* #2811 — pure evidence-block formatter (unit-testable, no engine).
*
* Each hit renders as one item carrying its slug + effective_date, annotated
* relative to the take's since_date: evidence dated BEFORE the claim can only
* be context (it cannot prove an outcome), evidence dated on/after can. Items
* clamp at EVIDENCE_ITEM_CLAMP_CHARS; the whole block caps at
* EVIDENCE_BLOCK_CAP_CHARS. Zero hits produce an explicit no-evidence note
* that steers the judge toward 'unresolvable'.
*/
export function formatEvidenceBlock(take: Take, hits: EvidenceHit[]): string {
const sinceDate = take.since_date ?? 'unknown';
if (hits.length === 0) {
return (
`No evidence found in the brain for this claim (hybrid search returned zero results).\n` +
`The claim was made on: ${sinceDate}. With nothing in the brain mentioning the ` +
`subject, grade 'unresolvable' — do not fabricate an outcome.`
);
}
const lines: string[] = [];
for (const h of hits) {
const itemDate = h.effective_date ? String(h.effective_date).slice(0, 10) : null;
let recency = '';
if (itemDate && take.since_date) {
// Lexicographic compare works for ISO prefixes (YYYY-MM vs YYYY-MM-DD:
// compare on the shorter's length so a same-month hit counts as after).
const cmpLen = Math.min(itemDate.length, take.since_date.length);
recency = itemDate.slice(0, cmpLen) >= take.since_date.slice(0, cmpLen)
? ' (dated after the claim)'
: ' (dated BEFORE the claim — context only, not outcome evidence)';
}
const text = (h.chunk_text ?? '').replace(/\s+/g, ' ').trim().slice(0, EVIDENCE_ITEM_CLAMP_CHARS);
if (!text) continue;
lines.push(`- [${h.slug} • ${itemDate ?? 'undated'}]${recency}\n ${text}`);
}
if (lines.length === 0) {
return (
`No usable evidence text found in the brain for this claim.\n` +
`The claim was made on: ${sinceDate}. Grade 'unresolvable' — do not fabricate an outcome.`
);
}
let block = lines.join('\n');
if (block.length > EVIDENCE_BLOCK_CAP_CHARS) {
block = block.slice(0, EVIDENCE_BLOCK_CAP_CHARS) + '\n[evidence truncated]';
}
return block;
}
/**
* Default evidence retriever (#2811) — real hybrid search on the take's
* claim text: expansion off (deterministic, no extra LLM spend), source
* scope threaded (federated array beats scalar per sourceScopeOpts), and
* the take's OWN page excluded (its body restates the claim — feeding it
* back in would let the judge grade a claim against itself). Fail-open:
* a retrieval error degrades to a claim-only note steering the judge to
* 'unresolvable' rather than aborting the phase.
*
* Takes the engine explicitly; process() binds it so the injected-seam
* type (EvidenceRetrieverFn) is unchanged.
*/
export async function defaultEvidenceRetriever(
engine: BrainEngine,
take: Take,
scope: ScopedReadOpts,
): Promise<string> {
try {
const hits = await hybridSearch(engine, take.claim, {
...INTERNAL_BREADTH_SEARCH_OPTS,
limit: EVIDENCE_SEARCH_LIMIT,
expansion: false,
...(scope.sourceIds && scope.sourceIds.length > 0
? { sourceIds: scope.sourceIds }
: scope.sourceId
? { sourceId: scope.sourceId }
: {}),
...(take.page_slug ? { exclude_slugs: [take.page_slug] } : {}),
});
// Belt-and-suspenders on top of exclude_slugs.
const filtered = hits.filter((h) => h.slug !== take.page_slug);
return formatEvidenceBlock(take, filtered);
} catch (err) {
return (
`[evidence retrieval failed: ${(err as Error).message}]\n` +
`Claim: ${take.claim}\n` +
`Made on: ${take.since_date ?? 'unknown'}\n` +
`With no retrievable evidence, grade 'unresolvable' — do not fabricate an outcome.`
);
}
}
/**
* Production judge — calls gateway.chat with the GRADE_TAKE_PROMPT.
* chatFn is injectable so tests can drive the parse path without the gateway
* (mock.module is banned outside *.serial.test.ts); only cycle.ts and tests
* import this module, and both use the default.
*/
export async function defaultJudge(input: {
take: Take;
evidence: string;
modelHint?: string;
}, chatFn: typeof gatewayChat = gatewayChat): Promise<JudgeVerdict> {
const prompt = GRADE_TAKE_PROMPT
.replace('{CLAIM}', input.take.claim)
.replace('{KIND}', input.take.kind)
.replace('{HOLDER}', input.take.holder)
.replace('{SINCE_DATE}', input.take.since_date ?? 'unknown')
.replace('{WEIGHT}', String(input.take.weight))
.replace('{EVIDENCE_BLOCK}', input.evidence);
const result = await chatFn({
messages: [{ role: 'user', content: prompt }],
...(input.modelHint ? { model: input.modelHint } : {}),
maxTokens: 600,
allowFallback: false,
});
const parsed = parseJudgeOutput(result.text);
if (!parsed) {
// Throw, don't mint a verdict: a synthetic 'unresolvable' row would be
// cached under the deterministic evidence signature and skipped forever,
// indistinguishable from a genuine evidence-based verdict (#3910). The
// per-take catch turns this into a warning and retries next cycle — the
// same policy the loop already applies to thrown judge errors.
throw new Error(`judge output parse failed: unparseable judge response (${result.text.length} chars)`);
}
return parsed;
}
/**
* The phase's work window: unresolved active takes, oldest-first. Takes too
* recent to grade are left out in SQL so they cannot fill the window, and
* takes with no verdict for this prompt and judge come first, so graded takes
* (which stay unresolved while auto-resolve is off) leave room for the ones
* behind them. takeIsOldEnough still runs on each take for dates SQL cannot read.
*/
function listGradeWindow(engine: BrainEngine, opts: { minAgeMonths: number; promptVersion: string; judgeModelId: string; limit: number }): Promise<Take[]> {
const cutoff = new Date();
cutoff.setMonth(cutoff.getMonth() - opts.minAgeMonths);
return engine.listTakes({
resolved: false, active: true, sortBy: 'since_date', oldestFirst: true, sinceOnOrBefore: utcDate(cutoff),
ungradedFirst: { promptVersion: opts.promptVersion, judgeModelId: opts.judgeModelId }, limit: opts.limit,
});
}
/**
* Determine whether a take is old enough to grade. Defaults to 6 months.
* Takes without since_date are NOT graded (we'd be hallucinating context).
*/
export function takeIsOldEnough(take: Take, minAgeMonths: number, now: Date = new Date()): boolean {
if (!take.since_date) return false;
const cutoff = new Date(now);
cutoff.setMonth(cutoff.getMonth() - minAgeMonths);
// Tolerant date parsing — since_date can be YYYY-MM-DD or YYYY-MM.
const sinceStr = take.since_date.length === 7 ? take.since_date + '-15' : take.since_date;
const sinceDate = new Date(sinceStr);
if (Number.isNaN(sinceDate.getTime())) return false;
return sinceDate.getTime() <= cutoff.getTime();
}
/**
* Derive the TakeResolution for a verdict. 'unresolvable' DOES NOT auto-apply
* — only correct/incorrect/partial do.
*/
function verdictToResolution(verdict: JudgeVerdict, resolvedByLabel: string): TakeResolution | null {
if (verdict.verdict === 'unresolvable') return null;
return {
quality: verdict.verdict,
resolvedBy: resolvedByLabel,
source: `grade_takes:${GRADE_TAKES_PROMPT_VERSION}`,
};
}
/**
* Hard wall-clock deadline for the grade_takes phase (gbrain#4168). Same
* clean-partial-exit contract as propose_takes: judge calls have long tails,
* and without a phase deadline the worker's job timeout killed the phase
* mid-write instead of letting it bank completed verdicts.
*/
const GRADE_TAKES_PHASE_DEADLINE_MS = 30 * 60 * 1000;
class GradeTakesPhase extends BaseCyclePhase {
readonly name = 'grade_takes' as CyclePhase;
protected readonly budgetUsdKey = 'cycle.grade_takes.budget_usd';
protected readonly budgetUsdDefault = 3.0;
protected override mapErrorCode(err: unknown): string {
if (err instanceof GBrainError) return err.problem;
if (err instanceof Error) {
if (err.message.includes('budget') || err.message.includes('Budget')) return 'CALIBRATION_GRADE_BUDGET_EXHAUSTED';
if (err.message.includes('parse')) return 'CALIBRATION_GRADE_PARSE_FAIL';
}
return 'GRADE_TAKES_UNKNOWN';
}
protected async process(
engine: BrainEngine,
scope: ScopedReadOpts,
_ctx: OperationContext,
opts: GradeTakesOpts,
): Promise<{ summary: string; details: Record<string, unknown>; status?: PhaseStatus }> {
const judge = opts.judge ?? defaultJudge;
// #2811: bind the engine here so the injected-seam type stays
// (take, scope) => Promise<string>.
const evidenceRetriever: EvidenceRetrieverFn =
opts.evidenceRetriever ??
((take: Take, takeScope: ScopedReadOpts) => defaultEvidenceRetriever(engine, take, takeScope));
const promptVersion = opts.promptVersion ?? GRADE_TAKES_PROMPT_VERSION;
const minAgeMonths = opts.minAgeMonths ?? 6;
const takeLimit = opts.takeLimit ?? 50;
const autoResolve = opts.autoResolve // D17 default OFF; an unreadable key stays off
?? isConfigTruthy(await Promise.resolve(engine.getConfig?.('cycle.grade_takes.auto_resolve.enabled')).catch(() => null));
const autoResolveThreshold = opts.autoResolveThreshold ?? GRADE_TAKES_AUTO_RESOLVE_THRESHOLD;
const resolvedByLabel = opts.resolvedByLabel ?? 'gbrain:grade_takes';
// One resolved string drives the judge call, the verdict-cache key, and
// the stored judge_model_id — the convention propose_takes adopted in
// v0.42.62. Previously the default judge call passed NO model hint (it
// rode the gateway's chat_model) while 'claude-sonnet-4-6' was hardcoded
// into judge_model_id, the evidence signature, and budget metering — on
// brains with a different chat_model, telemetry priced and recorded a
// model that never ran.
const judgeModelFull = opts.model ?? getChatModel();
// Bare tail for the stored judge_model_id + evidence signature
// (historical convention). Stock installs are unchanged: getChatModel()
// defaults to 'anthropic:claude-sonnet-4-6', whose tail equals the old
// hardcoded value — zero verdict-cache invalidation. A genuinely
// different chat_model invalidates, which is correct: the judge really
// changed.
const judgeModelId = splitProviderModelId(judgeModelFull).model || judgeModelFull;
const useEnsemble = opts.useEnsemble ?? false;
const ensembleThreshold = opts.ensembleThreshold ?? 0.85;
const ensembleTriggerBand = opts.ensembleTriggerBand ?? [0.6, 0.95];
const result: GradeTakesResult = {
takes_scanned: 0,
cache_hits: 0,
verdicts_written: 0,
auto_applied: 0,
too_recent: 0,
budget_exhausted: false,
judge_calls_succeeded: 0,
judge_calls_failed: 0,
warnings: [],
ensemble_invoked: 0,
ensemble_unanimous: 0,
deadline_hit: false,
};
// gbrain#4168: relative phase deadline clamped to the job's absolute
// deadline minus the reserve — same clean partial-exit contract as
// propose_takes (break, bank verdicts already written, report).
const phaseStartMs = Date.now();
const deadlineMs = effectivePhaseDeadlineMs(
opts.deadlineMs ?? GRADE_TAKES_PHASE_DEADLINE_MS,
opts.deadlineAtMs,
phaseStartMs,
);
if (deadlineMs <= 0) {
// Job budget already inside the reserve — exit before ANY judge call.
result.warnings.push('phase skipped: job deadline already inside the reserve window');
result.deadline_hit = true;
return {
summary: 'grade_takes: skipped — job deadline inside the reserve window',
details: { ...result, prompt_version: promptVersion, auto_resolve: autoResolve, auto_resolve_threshold: autoResolveThreshold },
status: 'warn',
};
}
// #3044 — shared halt policy over single-model judge AND ensemble
// rejections (rate limits counted at most once per take via
// rateLimitedThisTake). A take that completes without one resets the
// streak; auth/billing halt on the first hit.
const llmHalt = createGlobalLlmHaltTracker();
const takes = await listGradeWindow(engine, { minAgeMonths, promptVersion, judgeModelId, limit: takeLimit });
if (opts.reporter) {
opts.reporter.start('grade_takes.takes' as never, takes.length);
}
// #5280: a managed brain refuses the raw takes update; resolutions go
// through the coordinated, markdown-canonical takes_resolve mutation.
const managed = await managedPersistenceEnabled(engine);
const now = new Date();
for (const take of takes) {
// Phase deadline check (gbrain#4168). Break, not throw: verdicts
// already cached stay banked, and the phase reports partial cleanly
// before the worker's kill switch fires.
const elapsedMs = Date.now() - phaseStartMs;
if (elapsedMs > deadlineMs) {
result.warnings.push(
`phase deadline hit at take ${result.takes_scanned}/${takes.length} ` +
`after ${(elapsedMs / 1000).toFixed(0)}s (cap ${(deadlineMs / 1000).toFixed(0)}s); partial completion`,
);
result.deadline_hit = true;
break;
}
result.takes_scanned += 1;
this.tick(opts);
if (!takeIsOldEnough(take, minAgeMonths, now)) {
result.too_recent += 1;
continue;
}
// Retrieve evidence first — the signature depends on it.
const evidence = await evidenceRetriever(take, scope);
const sig = evidenceSignature(evidence, judgeModelId);
// Idempotency: skip when (take_id, prompt_version, judge_model_id, evidence_signature) exists.
const cached = await engine.executeRaw<{ verdict: string; confidence: number; applied: boolean }>(
`SELECT verdict, confidence, applied FROM take_grade_cache
WHERE take_id = $1 AND prompt_version = $2 AND judge_model_id = $3 AND evidence_signature = $4
LIMIT 1`,
[take.id, promptVersion, judgeModelId, sig],
);
if (cached.length > 0) {
result.cache_hits += 1;
continue;
}
// Budget pre-check. #2811: the input estimate is size-derived — real
// evidence blocks vary from a one-line no-evidence note to the 4k cap,
// and the old fixed 1200 underestimated a full block by ~2x.
const budget = this.checkBudget({
modelId: judgeModelId,
estimatedInputTokens: Math.ceil(
(GRADE_TAKE_PROMPT.length + take.claim.length + evidence.length) / 4,
),
maxOutputTokens: 400,
});
if (!budget.allowed) {
result.budget_exhausted = true;
result.warnings.push(
`budget exhausted at take ${result.takes_scanned}/${takes.length} (cumulative $${budget.cumulativeCostUsd.toFixed(4)} / cap $${budget.budgetUsd.toFixed(2)})`,
);
break;
}
// Call the single-model judge. Per-take errors log a warning and
// continue — UNLESS they classify as a whole-run condition (#3044):
// auth/billing halts on the first hit; a bare rate_limit halts only
// after RATE_LIMIT_HALT_STREAK consecutive hits.
let rateLimitedThisTake = false;
let verdict: JudgeVerdict;
try {
verdict = await judge({ take, evidence, modelHint: judgeModelFull });
} catch (err) {
result.judge_calls_failed += 1;
const msg = err instanceof Error ? err.message : String(err);
const detail = `judge failed on take ${take.id}: ${msg}`;
const decision = llmHalt.observe(err);
if (decision !== 'continue') {
result.aborted_global_error = haltedClassOf(decision)!;
result.warnings.push(
`aborting phase at take ${result.takes_scanned}/${takes.length}: ` +
`${llmHalt.note()} (${detail})`,
);
break;
}
result.warnings.push(detail);
continue;
}
result.judge_calls_succeeded += 1;
// T5 — ensemble tiebreaker for borderline single-model verdicts.
let recordedJudgeModelId = judgeModelId;
let recordedVerdict = verdict;
let ensembleApplyEligible = false;
const inBorderlineBand =
verdict.confidence >= ensembleTriggerBand[0] &&
verdict.confidence < ensembleTriggerBand[1] &&
verdict.verdict !== 'unresolvable';
if (useEnsemble && inBorderlineBand && opts.ensembleJudges && opts.ensembleJudges.length > 0) {
result.ensemble_invoked += 1;
const ensembleResults = await Promise.allSettled(
opts.ensembleJudges.map(j => j.fn({ take, evidence, modelHint: j.modelId })),
);
// #3044: Promise.allSettled flattens judge rejections into null
// verdicts below, so a whole-run condition (revoked key, exhausted
// spend limit) inside an ensemble judge would vanish into a
// "(failed)" note in the reasoning string. Classify each rejection
// with the same two-tier policy as the single-model path:
// auth/billing halts immediately; rate_limit (counted at most once
// per take) feeds the consecutive-streak counter.
let ensembleHaltClass: GlobalLlmErrorClass | null = null;
let ensembleHaltDetail = '';
for (let i = 0; i < ensembleResults.length; i++) {
const res = ensembleResults[i];
if (!res || res.status !== 'rejected') continue;
const decision = llmHalt.observe(res.reason, { countRateLimit: !rateLimitedThisTake });
if (llmHalt.lastClass() === null) continue;
if (llmHalt.lastClass() === 'rate_limit') rateLimitedThisTake = true;
const judgeId = opts.ensembleJudges[i]?.modelId ?? 'unknown';
const rmsg = res.reason instanceof Error ? res.reason.message : String(res.reason);
const detail = `ensemble judge ${judgeId} failed on take ${take.id}: ${rmsg}`;
if (decision !== 'continue') {
ensembleHaltClass = haltedClassOf(decision);
ensembleHaltDetail = detail;
break;
}
result.warnings.push(detail);
}
if (ensembleHaltClass) {
result.aborted_global_error = ensembleHaltClass;
result.warnings.push(
`aborting phase at take ${result.takes_scanned}/${takes.length}: ` +
`${llmHalt.note()} (${ensembleHaltDetail})`,
);
break;
}
const collected: Array<{ modelId: string; verdict: JudgeVerdict | null }> = opts.ensembleJudges.map((j, i) => {
const res = ensembleResults[i];
if (res && res.status === 'fulfilled') return { modelId: j.modelId, verdict: res.value };
return { modelId: j.modelId, verdict: null };
});
const ensemble = aggregateEnsemble(collected);
// Record the ensemble verdict in the cache row instead of the single-model
// verdict. The judge_model_id becomes 'ensemble:<modelA>+<modelB>+<modelC>'
// so a future re-run with different ensemble membership doesn't collide.
recordedJudgeModelId = `ensemble:${opts.ensembleJudges.map(j => j.modelId).join('+')}`;
recordedVerdict = {
verdict: ensemble.verdict,
confidence: ensemble.minConfidence,
reasoning: `ensemble agreement ${ensemble.agreement}/3; per-model: ${
ensemble.modelVerdicts.map(m => `${m.modelId}=${m.verdict}@${m.confidence.toFixed(2)}${m.failed ? '(failed)' : ''}`).join(', ')
}`,
};
if (ensemble.agreement === 3) result.ensemble_unanimous += 1;
// Ensemble auto-apply eligibility: 3/3 unanimous AND min confidence
// >= ensembleThreshold AND verdict not 'unresolvable'.
ensembleApplyEligible =
ensemble.agreement === 3 &&
ensemble.minConfidence >= ensembleThreshold &&
ensemble.verdict !== 'unresolvable';
}
// Decide auto-resolve eligibility BEFORE writing to cache so the
// `applied` column reflects the decision. Two paths:
// - Ensemble path: requires 3/3 unanimous + min conf >= ensembleThreshold
// - Single-model path: requires confidence >= autoResolveThreshold
// 'unresolvable' verdict NEVER auto-applies either way.
const resolution = verdictToResolution(recordedVerdict, resolvedByLabel);
let shouldApply = false;
if (autoResolve && resolution !== null) {
if (recordedJudgeModelId.startsWith('ensemble:')) {
shouldApply = ensembleApplyEligible;
} else {
shouldApply = recordedVerdict.confidence >= autoResolveThreshold;
}
}
// Compute a NEW evidence_signature when ensemble fires, since the
// cache composite key includes judge_model_id. (sig was computed
// against the single-model judge_model_id earlier.)
const recordedSig = recordedJudgeModelId === judgeModelId
? sig
: evidenceSignature(evidence, recordedJudgeModelId);
// Write the verdict to the cache. Idempotency conflict means another
// run beat us to it; either way the row exists with consistent state.
await engine.executeRaw(
`INSERT INTO take_grade_cache
(take_id, prompt_version, judge_model_id, evidence_signature, verdict, confidence, applied)
VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (take_id, prompt_version, judge_model_id, evidence_signature) DO NOTHING`,
[take.id, promptVersion, recordedJudgeModelId, recordedSig, recordedVerdict.verdict, recordedVerdict.confidence, shouldApply],
);
result.verdicts_written += 1;
// Apply to canonical takes if eligible.
if (shouldApply && resolution) {
try {
if (managed) {
// Resolve the judged take by page identity, not the cached slug: a
// rename during judging must not redirect the resolution to another page.
const [page] = await engine.executeRaw<{ slug: string; source_id: string }>(
`SELECT p.slug, p.source_id FROM pages p JOIN takes t ON t.page_id=p.id
WHERE p.id=$1 AND p.deleted_at IS NULL AND t.row_num=$2 AND t.claim=$3`, [take.page_id, take.row_num, take.claim]);
if (!page) throw new Error('the judged take moved or changed during grading; it is graded again next run');
const snapshot = await engine.readPageSnapshot(page.slug, { sourceId: page.source_id });
// The resolution publishes against this exact revision, so the judged
// claim must be the row this revision's fence holds.
const fenced = snapshot ? parseTakesFence(snapshot.page.compiled_truth ?? '').takes.find(t => t.rowNum === take.row_num) : undefined;
if (!snapshot || snapshot.page.id !== take.page_id || fenced?.claim !== take.claim) {
throw new Error('the judged page changed during grading; it is graded again next run');
}
await submitPageMutation({ engine, config: { engine: engine.kind } as never, remote: false, sourceId: page.source_id,
dryRun: false, logger: { info() {}, warn() {}, error() {} } }, { operation: 'takes_resolve', params: {
slug: page.slug, source_id: page.source_id, expected_revision: snapshot.revision, row_num: take.row_num, quality: resolution.quality,
evidence: resolution.source, resolved_by: resolution.resolvedBy,
request_id: createHash('sha256').update(`grade_takes:${take.id}:${recordedSig}:${snapshot.revision}`).digest('hex').replace(/^(.{8})(.{4}).(.{3}).(.{3})(.{12}).*/, '$1-$2-4$3-a$4-$5') } });
} else {
await maintenanceTransaction(engine, tx => tx.resolveTake(take.page_id, take.row_num, resolution));
}
result.auto_applied += 1;
// T11 / E4 — gstack-learnings coupling on incorrect / partial
// auto-resolutions. Best-effort: failures log warning + continue.
if (
(recordedVerdict.verdict === 'incorrect' || recordedVerdict.verdict === 'partial') &&
opts.writeGstackLearnings === true
) {
const { writeIncorrectResolution } = await import('../calibration/gstack-coupling.ts');
const coupling = await writeIncorrectResolution({
event: {
takeId: take.id,
pageSlug: take.page_slug,
rowNum: take.row_num,
holder: take.holder,
claim: take.claim,
quality: recordedVerdict.verdict,
weight: take.weight,
confidence: recordedVerdict.confidence,
reasoning: recordedVerdict.reasoning,
},
enabled: true,
});
if (!coupling.written && coupling.reason !== 'config_disabled') {
result.warnings.push(
`gstack coupling skipped (take ${take.id}): ${coupling.reason}${coupling.error ? ` — ${coupling.error}` : ''}`,
);
}
}
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
result.warnings.push(`auto-apply failed on take ${take.id}: ${msg}`);
// Managed: drop the cached verdict so the next run grades the take
// again instead of skipping it as already applied.
if (managed) await engine.executeRaw(`DELETE FROM take_grade_cache
WHERE take_id=$1 AND prompt_version=$2 AND judge_model_id=$3 AND evidence_signature=$4`,
[take.id, promptVersion, recordedJudgeModelId, recordedSig]);
}
}
// #3044: a take that completed without any rate_limit-classified LLM
// failure breaks the consecutive streak. (Cache hits / too-recent
// takes `continue` before the judge call and neither extend nor
// reset it.)
if (!rateLimitedThisTake) llmHalt.reset();
// Tally is silent — the caller surfaces it via the GradeTakesResult.
void recordedVerdict;
}
if (opts.reporter) opts.reporter.finish();
// Status folds warnings in (the extract_facts precedent from #1928): a
// run with swallowed per-take failures must not read as a clean 'ok'.
// Severity split (#3044): a global halt with ZERO successful judge calls
// means the whole LLM lane is down — phase 'fail' (deriveStatus turns
// one failed phase into a 'partial' cycle; the autopilot handler
// deliberately does not throw on partial). A halt after some successes
// is a partial run → 'warn'.
const warningCount = result.warnings.length;
// A deadline-hit run halted mid-list the same way a budget-exhausted one
// does (matches the propose_takes #4168 posture) — folded into `halted`
// so both the details field and the status derivation see it.
const halted =
result.budget_exhausted ||
result.deadline_hit === true ||
result.aborted_global_error !== undefined;
const phaseFailed =
result.aborted_global_error !== undefined && result.judge_calls_succeeded === 0;
const summary =
`grade_takes: scanned ${result.takes_scanned} takes ` +
`(${result.too_recent} too recent, ${result.cache_hits} cached, ` +
`${result.verdicts_written} new verdicts, ${result.auto_applied} auto-applied)` +
(result.deadline_hit ? ' [deadline hit — partial]' : '') +
(result.aborted_global_error
? `; aborted on ${result.aborted_global_error} error after ${result.takes_scanned} take(s)`
: '') +
(warningCount > 0 ? ` (${warningCount} warning(s))` : '');
return {
summary,
details: {
...result,
halted,
prompt_version: promptVersion,
auto_resolve: autoResolve,
auto_resolve_threshold: autoResolveThreshold,
},
status: phaseFailed ? 'fail' : halted || warningCount > 0 ? 'warn' : 'ok',
};
}
}
export async function runPhaseGradeTakes(
ctx: OperationContext,
opts: GradeTakesOpts = {},
) {
return new GradeTakesPhase().run(ctx, opts);
}
export const __testing = {
GradeTakesPhase,
parseJudgeOutput,
evidenceSignature,
takeIsOldEnough,
verdictToResolution,
aggregateEnsemble,
};