Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.40.1.0 Track D / T6+T7 — Nightly quality probe phase + doctor check.
*
* Hermetic: every external effect goes through the NightlyProbeDeps DI
* surface. No PGLite, no real LLM calls, no env mutation outside withEnv.
*/
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
import {
runNightlyQualityProbe,
shouldRunNightly,
type NightlyProbeDeps,
type NightlyProbeResult,
} from '../src/core/cycle/nightly-quality-probe.ts';
import type { CrossModalBatchSummary } from '../src/core/cycle/nightly-probe-adapters.ts';
import type { QualityProbeFailure } from '../src/core/audit-quality-probe.ts';
import { estimateChatCostUsd, recordChatUsage } from '../src/core/ai/chat-usage.ts';
import { DEFAULT_SLOTS } from '../src/core/cross-modal-eval/runner.ts';
import { withEnv } from './helpers/with-env.ts';
// ---------------------------------------------------------------------------
// Hermetic audit dir per test
// ---------------------------------------------------------------------------
let auditTmp: string;
beforeEach(() => {
auditTmp = mkdtempSync(join(tmpdir(), 'qprobe-audit-'));
});
afterEach(() => {
try { rmSync(auditTmp, { recursive: true, force: true }); } catch { /* best */ }
});
// ---------------------------------------------------------------------------
// 1. shouldRunNightly pure function
// ---------------------------------------------------------------------------
describe('shouldRunNightly (pure function, rate-limit logic)', () => {
test('empty history → run', () => {
expect(shouldRunNightly(new Date('2026-05-22T00:00:00Z'), [])).toEqual({ run: true });
});
test('last event > 24h ago → run', () => {
const r = shouldRunNightly(
new Date('2026-05-22T00:00:00Z'),
[{ ts: '2026-05-20T00:00:00Z' }],
);
expect(r).toEqual({ run: true });
});
test('last event within 24h → rate-limited', () => {
const r = shouldRunNightly(
new Date('2026-05-22T00:00:00Z'),
[{ ts: '2026-05-21T12:00:00Z' }],
);
expect(r).toEqual({ run: false, reason: 'rate_limited' });
});
test('one event old, one event recent → rate-limited (any recent fires it)', () => {
const r = shouldRunNightly(
new Date('2026-05-22T00:00:00Z'),
[
{ ts: '2026-05-01T00:00:00Z' },
{ ts: '2026-05-21T20:00:00Z' },
],
);
expect(r).toEqual({ run: false, reason: 'rate_limited' });
});
test('corrupt timestamp → ignored (does not rate-limit)', () => {
const r = shouldRunNightly(
new Date('2026-05-22T00:00:00Z'),
[{ ts: 'not a date' }],
);
expect(r).toEqual({ run: true });
});
test('configurable window respected', () => {
// 1-hour window: 6h ago counts as old.
const r = shouldRunNightly(
new Date('2026-05-22T00:00:00Z'),
[{ ts: '2026-05-21T18:00:00Z' }],
60 * 60 * 1000,
);
expect(r).toEqual({ run: true });
});
});
// ---------------------------------------------------------------------------
// 2. runNightlyQualityProbe via DI stubs
// ---------------------------------------------------------------------------
function makeDeps(overrides: Partial<NightlyProbeDeps> = {}): NightlyProbeDeps {
return {
isEnabled: async () => true,
hasEmbeddingProvider: async () => true,
resolveMaxUsd: async () => 5,
resolveRepoRoot: async () => process.cwd(),
runLongMemEval: async () => { /* stub */ },
runCrossModalBatch: async () => ({
exitCode: 0,
summary: {
pass_count: 5, fail_count: 0, inconclusive_count: 0, error_count: 0,
est_cost_usd: 0.35, verdict: 'pass',
},
}),
now: () => new Date(),
...overrides,
};
}
describe('runNightlyQualityProbe (DI stub harness)', () => {
test('disabled config → outcome: disabled, no audit row', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
const r = await runNightlyQualityProbe(makeDeps({ isEnabled: async () => false }));
expect(r.outcome).toBe('disabled');
expect(r.exit_code).toBe(0);
// No audit row written.
const events = await readEvents();
expect(events.length).toBe(0);
});
});
test('enabled + no embedding key → outcome: no_embedding_key with audit row', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
const r = await runNightlyQualityProbe(makeDeps({ hasEmbeddingProvider: async () => false }));
expect(r.outcome).toBe('no_embedding_key');
const events = await readEvents();
expect(events.length).toBe(1);
expect(events[0].outcome).toBe('no_embedding_key');
});
});
test('enabled + recent run within 24h → outcome: rate_limited, NO audit row', async () => {
// Pre-seed a recent audit event by running the probe once first.
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
// First run succeeds.
await runNightlyQualityProbe(makeDeps());
// Second run, same hour → rate_limited. A skip is a non-event: the
// autopilot loop invokes the probe every cycle (~5-10 min), so
// logging each skip would flood the audit file and flip doctor's
// any-non-pass-is-bad filter to a permanent WARN.
const r2 = await runNightlyQualityProbe(makeDeps());
expect(r2.outcome).toBe('rate_limited');
const events = await readEvents();
expect(events.length).toBe(1);
expect(events[0].outcome).toBe('pass');
});
});
test('enabled + PASS summary → outcome: pass with audit row', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
const r = await runNightlyQualityProbe(makeDeps());
expect(r.outcome).toBe('pass');
expect(r.exit_code).toBe(0);
const events = await readEvents();
expect(events.length).toBe(1);
expect(events[0].outcome).toBe('pass');
expect(events[0].pass_count).toBe(5);
expect(events[0].est_cost_usd).toBe(0.35);
});
});
test('threads live search-mode/reranker snapshot into LongMemEval', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
let seenSnapshot: Record<string, string> | undefined;
const r = await runNightlyQualityProbe(makeDeps({
resolveSearchConfigSnapshot: async () => ({
'search.mode': 'balanced',
'search.reranker.enabled': 'true',
'search.reranker.model': 'llama-server-reranker:qwen3-reranker-4b',
'search.reranker.timeout_ms': '30000',
}),
runLongMemEval: async (args) => {
seenSnapshot = args.searchConfigSnapshot;
},
}));
expect(r.outcome).toBe('pass');
expect(seenSnapshot).toEqual({
'search.mode': 'balanced',
'search.reranker.enabled': 'true',
'search.reranker.model': 'llama-server-reranker:qwen3-reranker-4b',
'search.reranker.timeout_ms': '30000',
});
});
});
test('enabled + FAIL summary → outcome: fail', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
const r = await runNightlyQualityProbe(makeDeps({
runCrossModalBatch: async () => ({
exitCode: 1,
summary: {
pass_count: 7, fail_count: 3, inconclusive_count: 0, error_count: 0,
est_cost_usd: 0.42, verdict: 'fail',
},
}),
}));
expect(r.outcome).toBe('fail');
expect(r.exit_code).toBe(1);
const events = await readEvents();
expect(events[0].outcome).toBe('fail');
expect(events[0].fail_count).toBe(3);
});
});
test('threads the brain-resolved model routes into both adapters (#5872)', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
const routes = {
reader: { model: 'claude-cli:claude-opus-5-5', source: 'tier_config' as const },
extractor: { model: 'claude-cli:claude-sonnet-5', source: 'tier_config' as const },
slots: { B: 'claude-cli:claude-fable-5' },
};
const seen: { lme?: unknown; crossModal?: unknown } = {};
const r = await runNightlyQualityProbe(makeDeps({
resolveModelRoutes: async () => routes,
runLongMemEval: async (args) => { seen.lme = args.modelRoutes; },
runCrossModalBatch: async (args) => {
seen.crossModal = args.modelRoutes;
return {
exitCode: 0,
summary: { pass_count: 5, fail_count: 0, inconclusive_count: 0, error_count: 0, est_cost_usd: 0.35, verdict: 'pass' },
};
},
}));
expect(r.outcome).toBe('pass');
expect(seen.lme).toEqual(routes);
expect(seen.crossModal).toEqual(routes);
});
});
test('resolveModelRoutes throws → error audit row naming the routes, LongMemEval never starts (#5872)', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
let longMemEvalCalls = 0;
const r = await runNightlyQualityProbe(makeDeps({
resolveModelRoutes: async () => { throw new Error('gateway refresh failed'); },
runLongMemEval: async () => { longMemEvalCalls++; },
}));
expect(r.outcome).toBe('error');
expect(longMemEvalCalls).toBe(0);
const events = await readEvents();
expect(events).toHaveLength(1);
expect(events[0].outcome).toBe('error');
expect(events[0].detail).toBe(
'nightly-quality-probe: could not resolve the model routes (reader, extractor, judge slots) ' +
'from the brain: gateway refresh failed',
);
// No route resolved and no model call ran (#5506).
expect(events[0].reader_model).toBeUndefined();
expect(events[0].chat_calls).toBeUndefined();
});
});
test('runLongMemEval throws → outcome: error with audit row', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
const r = await runNightlyQualityProbe(makeDeps({
runLongMemEval: async () => { throw new Error('longmemeval blew up'); },
}));
expect(r.outcome).toBe('error');
expect(r.exit_code).toBe(1);
const events = await readEvents();
expect(events[0].outcome).toBe('error');
expect(events[0].detail).toContain('longmemeval blew up');
});
});
test('missing fixture → outcome: skipped with reason fixture_unavailable, no model call (#5187)', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
let longMemEvalCalls = 0;
const r = await runNightlyQualityProbe(makeDeps({
resolveRepoRoot: async () => '/this/repo/root/does/not/exist',
runLongMemEval: async () => { longMemEvalCalls++; },
}));
expect(r.outcome).toBe('skipped');
expect(r.exit_code).toBe(0);
expect(longMemEvalCalls).toBe(0);
const events = await readEvents();
expect(events[0].outcome).toBe('skipped');
expect(events[0].reason).toBe('fixture_unavailable');
expect(events[0].detail).toContain('the nightly fixture is not readable at /this/repo/root/does/not/exist/test/fixtures/longmemeval-nightly.jsonl');
expect(events[0].detail).toContain('gbrain doctor');
});
});
test('resolveFixturePath wins over resolveRepoRoot (the embedded fixture, #5187)', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
const { NIGHTLY_PROBE_FIXTURES } = await import('../src/core/cycle/nightly-probe-fixtures.ts');
let seen = '';
const r = await runNightlyQualityProbe(makeDeps({
resolveRepoRoot: async () => '/this/repo/root/does/not/exist',
resolveFixturePath: () => NIGHTLY_PROBE_FIXTURES.longMemEval,
runLongMemEval: async (args) => { seen = args.fixturePath; },
}));
expect(r.outcome).toBe('pass');
expect(seen).toBe(NIGHTLY_PROBE_FIXTURES.longMemEval);
expect(readFileSync(seen, 'utf8').trim().split('\n')).toHaveLength(10);
});
});
test('audit event records fixture_sha8 on successful runs', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
const r = await runNightlyQualityProbe(makeDeps());
expect(r.outcome).toBe('pass');
const events = await readEvents();
expect(events[0].fixture_sha8).toMatch(/^[0-9a-f]{8}$/);
});
});
});
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
async function readEvents(): Promise<any[]> {
// Re-import so it uses the override env var picked up at call time.
const { readRecentQualityProbeEvents } = await import('../src/core/audit-quality-probe.ts');
return readRecentQualityProbeEvents(2);
}
// ---------------------------------------------------------------------------
// 3. computeNightlyQualityProbeHealthCheck pure function (doctor.ts coverage)
// ---------------------------------------------------------------------------
describe('computeNightlyQualityProbeHealthCheck — pure doctor branch coverage', () => {
test('disabled + no events → ok with paste-ready enable hint', async () => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const check = computeNightlyQualityProbeHealthCheck(false, []);
expect(check.name).toBe('nightly_quality_probe_health');
expect(check.status).toBe('ok');
expect(check.message).toMatch(/disabled \(opt-in\)/);
expect(check.message).toMatch(/gbrain config set autopilot\.nightly_quality_probe\.enabled true/);
});
test('latest run skipped (fixture unavailable) → warn with skipped count and reason, never ok (#5187)', async () => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const check = computeNightlyQualityProbeHealthCheck(true, [
{ outcome: 'skipped', ts: '2026-10-02T03:00:00Z', detail: 'the nightly fixture is not readable at /x.jsonl' },
]);
expect(check.status).toBe('warn');
expect(check.message).toContain('skipped=1');
expect(check.message).toContain('Latest: skipped at 2026-10-02T03:00:00Z (the nightly fixture is not readable at /x.jsonl)');
});
test('enabled + no events → ok pending', async () => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const check = computeNightlyQualityProbeHealthCheck(true, []);
expect(check.status).toBe('ok');
expect(check.message).toMatch(/enabled but no probe events/);
});
test('enabled + all-PASS events → ok with latest timestamp', async () => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const events = [
{ outcome: 'pass', ts: '2026-05-20T03:00:00Z' },
{ outcome: 'pass', ts: '2026-05-21T03:00:00Z' },
{ outcome: 'pass', ts: '2026-05-22T03:00:00Z' },
];
const check = computeNightlyQualityProbeHealthCheck(true, events);
expect(check.status).toBe('ok');
expect(check.message).toMatch(/3 PASS runs/);
expect(check.message).toContain('2026-05-22T03:00:00Z');
});
test('enabled + ANY fail/error/budget_exceeded → warn with per-outcome counts', async () => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const events = [
{ outcome: 'pass', ts: '2026-05-19T03:00:00Z' },
{ outcome: 'fail', ts: '2026-05-20T03:00:00Z' },
{ outcome: 'error', ts: '2026-05-21T03:00:00Z', detail: 'longmemeval blew up' },
{ outcome: 'budget_exceeded', ts: '2026-05-22T03:00:00Z' },
];
const check = computeNightlyQualityProbeHealthCheck(true, events);
expect(check.status).toBe('warn');
expect(check.message).toMatch(/3 non-PASS runs/);
expect(check.message).toMatch(/pass=1/);
expect(check.message).toMatch(/fail=1/);
expect(check.message).toMatch(/error=1/);
expect(check.message).toMatch(/budget=1/);
// Latest in the list is what surfaces in the message.
expect(check.message).toContain('budget_exceeded');
expect(check.message).toContain('2026-05-22T03:00:00Z');
});
test('latest event with detail → detail surfaces in warn message', async () => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const events = [
{ outcome: 'error', ts: '2026-05-22T03:00:00Z', detail: 'no embedding provider' },
];
const check = computeNightlyQualityProbeHealthCheck(true, events);
expect(check.status).toBe('warn');
expect(check.message).toContain('no embedding provider');
});
test('single non-PASS event uses singular grammar', async () => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const events = [{ outcome: 'fail', ts: '2026-05-22T03:00:00Z' }];
const check = computeNightlyQualityProbeHealthCheck(true, events);
expect(check.status).toBe('warn');
expect(check.message).toMatch(/1 non-PASS run /); // "run " not "runs "
});
test('cross-week ordering: reader sorts chronologically so "Latest" is the newest run', async () => {
// Regression: the reader walks the CURRENT week's file first, then the
// previous week's. Without sorting, the array tail — which this check
// reports as "Latest:" — was the OLDEST in-window event whenever last
// week's file had entries (observed live: counts updated as new runs
// landed while "Latest" stayed pinned days behind).
const { computeQualityProbeAuditFilename, readRecentQualityProbeEvents } =
await import('../src/core/audit-quality-probe.ts');
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const now = new Date('2026-07-23T12:00:00Z');
const thisWeekFile = computeQualityProbeAuditFilename(now);
const prevWeekFile = computeQualityProbeAuditFilename(new Date(now.getTime() - 7 * 86400000));
writeFileSync(join(auditTmp, thisWeekFile), [
JSON.stringify({ outcome: 'fail', ts: '2026-07-22T08:00:00Z' }),
JSON.stringify({ outcome: 'fail', ts: '2026-07-23T08:00:00Z' }),
].join('\n') + '\n');
writeFileSync(join(auditTmp, prevWeekFile), [
JSON.stringify({ outcome: 'fail', ts: '2026-07-17T08:00:00Z' }),
JSON.stringify({ outcome: 'fail', ts: '2026-07-18T08:00:00Z' }),
].join('\n') + '\n');
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
const events = readRecentQualityProbeEvents(7, now);
expect(events.map(e => e.ts)).toEqual([
'2026-07-17T08:00:00Z',
'2026-07-18T08:00:00Z',
'2026-07-22T08:00:00Z',
'2026-07-23T08:00:00Z',
]);
const check = computeNightlyQualityProbeHealthCheck(true, events);
expect(check.message).toContain('Latest: fail at 2026-07-23T08:00:00Z');
});
});
test('single PASS event uses singular grammar', async () => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const events = [{ outcome: 'pass', ts: '2026-05-22T03:00:00Z' }];
const check = computeNightlyQualityProbeHealthCheck(true, events);
expect(check.status).toBe('ok');
expect(check.message).toMatch(/1 PASS run /); // "run " not "runs "
});
});
// ---------------------------------------------------------------------------
// 4. Codex CDX-5 — doctor flags ALL non-PASS outcomes (no_embedding_key,
// rate_limited, inconclusive must trip warn, not get silently reported as PASS)
// ---------------------------------------------------------------------------
describe('codex CDX-5 — doctor health: every non-PASS outcome surfaces', () => {
test('no_embedding_key outcome → warn (was silently PASS before CDX-5 fix)', async () => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const events = [{ outcome: 'no_embedding_key', ts: '2026-05-22T03:00:00Z' }];
const check = computeNightlyQualityProbeHealthCheck(true, events);
expect(check.status).toBe('warn');
expect(check.message).toMatch(/no_embed_key=1/);
});
test('rate_limited outcome → warn', async () => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const events = [{ outcome: 'rate_limited', ts: '2026-05-22T03:00:00Z' }];
const check = computeNightlyQualityProbeHealthCheck(true, events);
expect(check.status).toBe('warn');
expect(check.message).toMatch(/rate_limited=1/);
});
test('inconclusive outcome → warn', async () => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const events = [{ outcome: 'inconclusive', ts: '2026-05-22T03:00:00Z' }];
const check = computeNightlyQualityProbeHealthCheck(true, events);
expect(check.status).toBe('warn');
expect(check.message).toMatch(/inconclusive=1/);
});
test('counts include the new outcome buckets when mixed with pass/fail/error', async () => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const events = [
{ outcome: 'pass', ts: '2026-05-16T03:00:00Z' },
{ outcome: 'fail', ts: '2026-05-17T03:00:00Z' },
{ outcome: 'error', ts: '2026-05-18T03:00:00Z' },
{ outcome: 'inconclusive', ts: '2026-05-19T03:00:00Z' },
{ outcome: 'budget_exceeded', ts: '2026-05-20T03:00:00Z' },
{ outcome: 'no_embedding_key', ts: '2026-05-21T03:00:00Z' },
{ outcome: 'rate_limited', ts: '2026-05-22T03:00:00Z' },
];
const check = computeNightlyQualityProbeHealthCheck(true, events);
expect(check.status).toBe('warn');
expect(check.message).toMatch(/6 non-PASS runs/); // 7 total, 1 pass, 6 bad
expect(check.message).toMatch(/pass=1/);
expect(check.message).toMatch(/fail=1/);
expect(check.message).toMatch(/error=1/);
expect(check.message).toMatch(/inconclusive=1/);
expect(check.message).toMatch(/budget=1/);
expect(check.message).toMatch(/no_embed_key=1/);
expect(check.message).toMatch(/rate_limited=1/);
});
});
// ---------------------------------------------------------------------------
// 5. #5506: the audit row carries the evidence behind its verdict
// ---------------------------------------------------------------------------
const SONNET = 'anthropic:claude-sonnet-4-6';
const OPUS = 'anthropic:claude-opus-4-7';
const COLLAPSED_PANEL = { judge_models: [SONNET, OPUS, SONNET], panel: { distinct_models: 2, distinct_providers: 1 } };
const CLI_PANEL = {
judge_models: ['claude-cli:claude-opus-5-5', 'claude-cli:claude-sonnet-5', 'claude-cli:claude-haiku-4-5-20251001'],
panel: { distinct_models: 3, distinct_providers: 1 },
};
const ROUTES = {
reader: { model: 'claude-cli:claude-opus-5-5', source: 'tier_config' as const },
extractor: { model: 'claude-cli:claude-sonnet-5', source: 'tier_config' as const },
slots: {},
};
function directnessFailure(questionId: string, scores: number[]): QualityProbeFailure {
const mean = Math.round((scores.reduce((a, b) => a + b, 0) / scores.length) * 10) / 10;
return {
question_id: questionId,
verdict: 'fail',
dimensions: [{ dimension: 'directness', mean, scores, fail_reason: 'mean_below_7' }],
};
}
function batchStub(summary: CrossModalBatchSummary, exitCode = 1): NightlyProbeDeps['runCrossModalBatch'] {
return async () => ({ exitCode, summary });
}
describe('runNightlyQualityProbe: panel, failures and digest on the audit row (#5506)', () => {
test('a FAIL summary on a sound panel writes the judge panel, each failing question and a digest detail', async () => {
const failures = [[6, 7, 6], [5, 8, 6], [5, 8, 5], [6, 7, 5], [6, 8, 6], [5, 7, 6]]
.map((scores, i) => directnessFailure(`q${i + 1}`, scores));
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
const r = await runNightlyQualityProbe(makeDeps({
runCrossModalBatch: batchStub({
pass_count: 4, fail_count: 6, inconclusive_count: 0, error_count: 0, est_cost_usd: 2.8,
verdict: 'fail', total: 10, failures, ...CLI_PANEL,
}),
}));
expect(r.outcome).toBe('fail');
const [event] = await readEvents();
expect(event.judge_models).toEqual(CLI_PANEL.judge_models);
expect(event.distinct_judge_models).toBe(3);
expect(event.distinct_judge_providers).toBe(1);
expect(event.failures).toEqual(failures);
expect(event.reason).toBeUndefined();
expect(event.detail).toBe(
'6/10 questions did not pass (directness mean_below_7 x6); judges: 3 distinct models from 1 provider',
);
expect(event.est_cost_usd).toBe(2.8);
});
});
test('a sonnet/opus/sonnet panel turns a FAIL into inconclusive with the remedy (#5506, D12)', async () => {
const failures = [[6, 7, 6], [5, 8, 6]].map((scores, i) => directnessFailure(`q${i + 1}`, scores));
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
const r = await runNightlyQualityProbe(makeDeps({
runCrossModalBatch: batchStub({
pass_count: 8, fail_count: 2, inconclusive_count: 0, error_count: 0, est_cost_usd: 2.8,
verdict: 'fail', total: 10, failures, ...COLLAPSED_PANEL,
}),
}));
expect(r.outcome).toBe('inconclusive');
const [event] = await readEvents();
expect(event.outcome).toBe('inconclusive');
expect(event.reason).toBe('panel_collapsed');
expect(event.judge_models).toEqual([SONNET, OPUS, SONNET]);
expect(event.failures).toEqual(failures);
expect(event.detail).toBe(
`judge panel collapsed: ${SONNET} holds slots A, C, so its votes count more than once. ` +
'Set three different judge models: gbrain config set models.eval.cross_modal.slot_a <model> ' +
'(and slot_b, slot_c; one provider is enough). ' +
'See docs/eval-bench.md#nightly-cross-modal-quality-probe-opt-in-autopilot (batch verdict: fail); ' +
'2/10 questions did not pass (directness mean_below_7 x2); judges: 2 distinct models from 1 provider',
);
});
});
test('a PASS from a collapsed panel is inconclusive, never a pass', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
const r = await runNightlyQualityProbe(makeDeps({
runCrossModalBatch: batchStub({
pass_count: 10, fail_count: 0, inconclusive_count: 0, error_count: 0, est_cost_usd: 1.8,
verdict: 'pass', total: 10, ...COLLAPSED_PANEL,
}, 0),
}));
expect(r.outcome).toBe('inconclusive');
const [event] = await readEvents();
expect(event.outcome).toBe('inconclusive');
expect(event.reason).toBe('panel_collapsed');
expect(event.judge_models).toEqual([SONNET, OPUS, SONNET]);
expect(event.failures).toBeUndefined();
expect(event.detail).toStartWith(`judge panel collapsed: ${SONNET} holds slots A, C`);
});
});
test('only one model judged (two slots silent) is inconclusive', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
const r = await runNightlyQualityProbe(makeDeps({
runCrossModalBatch: batchStub({
pass_count: 10, fail_count: 0, inconclusive_count: 0, error_count: 0, est_cost_usd: 1.8,
verdict: 'pass', total: 10, judge_models: CLI_PANEL.judge_models,
panel: { distinct_models: 1, distinct_providers: 1, slot_scored_questions: [10, 0, 0] },
}, 0),
}));
expect(r.outcome).toBe('inconclusive');
const [event] = await readEvents();
expect(event.reason).toBe('panel_collapsed');
expect(event.detail).toStartWith('judge panel collapsed: 1 distinct model judged, so there is no cross-check.');
});
});
test('a PASS summary on a sound panel records the panel and no failure list or digest', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
await runNightlyQualityProbe(makeDeps({
runCrossModalBatch: batchStub({
pass_count: 10, fail_count: 0, inconclusive_count: 0, error_count: 0, est_cost_usd: 1.8,
verdict: 'pass', total: 10, ...CLI_PANEL,
}, 0),
}));
const [event] = await readEvents();
expect(event.outcome).toBe('pass');
expect(event.judge_models).toEqual(CLI_PANEL.judge_models);
expect(event.failures).toBeUndefined();
expect(event.detail).toBeUndefined();
expect(event.receipt_dir).toBeUndefined();
});
});
test('more than 10 non-passing questions: the row lists the first 10, the digest counts all', async () => {
const failures: QualityProbeFailure[] = [
...Array.from({ length: 11 }, (_, i) => directnessFailure(`q${i + 1}`, [6, 6, 6])),
{ question_id: 'q12', verdict: 'upstream_error', error: 'reader produced no hypothesis' },
];
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
await runNightlyQualityProbe(makeDeps({
runCrossModalBatch: batchStub({
pass_count: 0, fail_count: 11, inconclusive_count: 0, error_count: 0, est_cost_usd: 2.8,
verdict: 'error', total: 12, failures, judge_models: DEFAULT_SLOTS.map(s => s.model),
panel: { distinct_models: 3, distinct_providers: 3 },
}, 2),
}));
const [event] = await readEvents();
expect(event.outcome).toBe('error');
expect(event.failures).toEqual(failures.slice(0, 10));
expect(event.detail).toBe(
'12/12 questions did not pass (directness mean_below_7 x11, upstream_error x1); ' +
'judges: 3 distinct models from 3 providers',
);
});
});
const SIX_FAILURES = [[6, 7, 6], [5, 8, 6], [5, 8, 5], [6, 7, 5], [6, 8, 6], [5, 7, 6]]
.map((scores, i) => directnessFailure(`q${i + 1}`, scores));
const DIGEST_CASES = [
{
// A malformed row has no failure entry but counts in the total.
name: 'malformed rows count as not passing',
summary: {
verdict: 'error', total: 10, malformed_count: 1, pass_count: 3, fail_count: 6, failures: SIX_FAILURES,
...COLLAPSED_PANEL,
},
detail: '7/10 questions did not pass (directness mean_below_7 x6, malformed x1); ' +
'judges: 2 distinct models from 1 provider',
failures: SIX_FAILURES,
judgeScoredQuestions: undefined,
},
{
name: 'a batch of malformed rows only still gets a digest',
summary: { verdict: 'error', total: 10, malformed_count: 2, pass_count: 8, fail_count: 0, ...COLLAPSED_PANEL },
detail: '2/10 questions did not pass (malformed x2); judges: 2 distinct models from 1 provider',
failures: undefined,
judgeScoredQuestions: undefined,
},
{
name: 'a slot that scored no question is named',
summary: {
verdict: 'fail', total: 10, pass_count: 4, fail_count: 6, failures: SIX_FAILURES,
judge_models: DEFAULT_SLOTS.map(s => s.model),
panel: { distinct_models: 2, distinct_providers: 2, slot_scored_questions: [10, 0, 10] },
},
detail: '6/10 questions did not pass (directness mean_below_7 x6); ' +
`judges: 2 distinct models from 2 providers, slot B (${DEFAULT_SLOTS[1]!.model}) scored no question`,
failures: SIX_FAILURES,
judgeScoredQuestions: [10, 0, 10],
},
];
test.each(DIGEST_CASES)('digest: $name', async ({ summary, detail, failures, judgeScoredQuestions }) => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
await runNightlyQualityProbe(makeDeps({
runCrossModalBatch: batchStub({ inconclusive_count: 0, error_count: 0, est_cost_usd: 2.8, ...summary }, 2),
}));
const [event] = await readEvents();
expect(event.detail).toBe(detail);
expect(event.failures).toEqual(failures);
expect(event.judge_scored_questions).toEqual(judgeScoredQuestions);
});
});
test('rows written after the routes resolved name the reader and extractor', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
await runNightlyQualityProbe(makeDeps({ resolveModelRoutes: async () => ROUTES }));
const after = new Date(Date.now() + 25 * 60 * 60 * 1000);
await runNightlyQualityProbe(makeDeps({
now: () => after,
resolveModelRoutes: async () => ROUTES,
runLongMemEval: async () => { throw new Error('longmemeval blew up'); },
}));
const { readRecentQualityProbeEvents } = await import('../src/core/audit-quality-probe.ts');
const events = readRecentQualityProbeEvents(2, after);
expect(events.map(e => [e.outcome, e.reader_model, e.extractor_model])).toEqual([
['pass', 'claude-cli:claude-opus-5-5', 'claude-cli:claude-sonnet-5'],
['error', 'claude-cli:claude-opus-5-5', 'claude-cli:claude-sonnet-5'],
]);
});
});
});
describe('runNightlyQualityProbe: metered chat spend (#5506)', () => {
const usage = { input_tokens: 1000, output_tokens: 500 };
const PRICED = 'anthropic:claude-haiku-4-5';
const UNPRICED = 'acme:unpriced-model-9000';
const pricedCost = () => estimateChatCostUsd(PRICED, usage)!;
test('calls in both stages are metered: priced cost summed, unpriced counted apart', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
await runNightlyQualityProbe(makeDeps({
runLongMemEval: async () => {
recordChatUsage({ model: PRICED, usage });
await Promise.resolve();
recordChatUsage({ model: UNPRICED, usage });
},
runCrossModalBatch: async () => {
await Promise.resolve();
recordChatUsage({ model: PRICED, usage });
return {
exitCode: 0,
summary: { pass_count: 10, fail_count: 0, inconclusive_count: 0, error_count: 0, est_cost_usd: 2.8, verdict: 'pass' },
};
},
}));
const [event] = await readEvents();
expect(event.chat_calls).toBe(3);
expect(event.chat_cost_usd).toBeCloseTo(2 * pricedCost(), 6);
expect(event.unpriced_chat_calls).toBe(1);
expect(event.est_cost_usd).toBe(2.8);
});
});
test('a run whose every call is unpriced records 0 cost and every call as unpriced', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
await runNightlyQualityProbe(makeDeps({
runLongMemEval: async () => {
recordChatUsage({ model: UNPRICED, usage });
recordChatUsage({ model: UNPRICED, usage });
},
}));
const [event] = await readEvents();
expect([event.chat_calls, event.chat_cost_usd, event.unpriced_chat_calls]).toEqual([2, 0, 2]);
});
});
test('a run that fails part-way records the spend metered up to the failure', async () => {
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
const r = await runNightlyQualityProbe(makeDeps({
runLongMemEval: async () => {
recordChatUsage({ model: PRICED, usage });
throw new Error('longmemeval blew up');
},
}));
expect(r.outcome).toBe('error');
const [event] = await readEvents();
expect(event.detail).toContain('longmemeval blew up');
expect(event.chat_calls).toBe(1);
expect(event.chat_cost_usd).toBeCloseTo(pricedCost(), 6);
expect(event.unpriced_chat_calls).toBe(0);
});
});
});
// ---------------------------------------------------------------------------
// 6. #5506: doctor names the latest run's judge panel
// ---------------------------------------------------------------------------
describe('computeNightlyQualityProbeHealthCheck: judge panel (#5506)', () => {
const TS = '2026-10-02T11:14:26Z';
const OK = `1 PASS run in last 7d. Latest: ${TS}.`;
const WARN =
'1 non-PASS run in last 7d (pass=0 fail=1 error=0 inconclusive=0 budget=0 no_embed_key=0 rate_limited=0). ' +
`Latest: fail at ${TS}.`;
const CLI_MODELS = ['claude-cli:claude-opus-5-5', 'claude-cli:claude-sonnet-5', 'claude-cli:claude-haiku-4-5-20251001'];
const DUPLICATE_NOTE =
` ${SONNET} holds more than one slot, so its votes count more than once. ` +
'Next step: set models.eval.cross_modal.slot_a, slot_b and slot_c to three different models ' +
'(gbrain config set models.eval.cross_modal.slot_a <model>; one provider is enough, for example three claude-cli models).';
const NOT_CROSS_MODAL = ' Fewer than 3 providers judged, so the panel is not cross-modal (information only).';
const PANELS = {
collapsed: {
fields: { judge_models: [SONNET, OPUS, SONNET], distinct_judge_models: 2, distinct_judge_providers: 1 },
note: ` Latest judge panel (slot order): ${SONNET}, ${OPUS}, ${SONNET}; 2 distinct models from 1 provider.` + DUPLICATE_NOTE,
},
oneProvider: {
fields: { judge_models: CLI_MODELS, distinct_judge_models: 3, distinct_judge_providers: 1 },
note: ` Latest judge panel (slot order): ${CLI_MODELS.join(', ')}; 3 distinct models from 1 provider.` + NOT_CROSS_MODAL,
},
twoProviders: {
fields: { judge_models: [CLI_MODELS[0]!, SONNET, OPUS], distinct_judge_models: 3, distinct_judge_providers: 2 },
note: ` Latest judge panel (slot order): ${CLI_MODELS[0]}, ${SONNET}, ${OPUS}; 3 distinct models from 2 providers.` +
NOT_CROSS_MODAL,
},
threeProviders: {
fields: { judge_models: DEFAULT_SLOTS.map(s => s.model), distinct_judge_models: 3, distinct_judge_providers: 3 },
note: ` Latest judge panel (slot order): ${DEFAULT_SLOTS.map(s => s.model).join(', ')}; 3 distinct models from 3 providers.`,
},
silentSlot: {
fields: {
judge_models: DEFAULT_SLOTS.map(s => s.model), judge_scored_questions: [10, 0, 10],
distinct_judge_models: 2, distinct_judge_providers: 2,
},
note: ` Latest judge panel (slot order): ${DEFAULT_SLOTS.map(s => s.model).join(', ')}; 2 distinct models from 2 providers.` +
` Slot B (${DEFAULT_SLOTS[1]!.model}) scored no question, so it did not judge.` + NOT_CROSS_MODAL,
},
// The model shared with slot A never scored in slot C, so its votes did not count twice.
silentDuplicate: {
fields: {
judge_models: [SONNET, OPUS, SONNET], judge_scored_questions: [10, 10, 0],
distinct_judge_models: 2, distinct_judge_providers: 1,
},
note: ` Latest judge panel (slot order): ${SONNET}, ${OPUS}, ${SONNET}; 2 distinct models from 1 provider.` +
` Slot C (${SONNET}) scored no question, so it did not judge.` + NOT_CROSS_MODAL,
},
// Scored counts of the wrong length are ignored: every slot counts as having judged.
malformedScoredCounts: {
fields: {
judge_models: [SONNET, OPUS, SONNET], judge_scored_questions: [10, 0],
distinct_judge_models: 2, distinct_judge_providers: 1,
},
note: ` Latest judge panel (slot order): ${SONNET}, ${OPUS}, ${SONNET}; 2 distinct models from 1 provider.` + DUPLICATE_NOTE,
},
};
const CASES = (Object.keys(PANELS) as Array<keyof typeof PANELS>).flatMap(panel => [
{ panel, outcome: 'pass', status: 'ok' as const, message: OK + PANELS[panel].note },
{ panel, outcome: 'fail', status: 'warn' as const, message: WARN + PANELS[panel].note },
]);
test.each(CASES)('$panel panel on a $outcome run', async ({ panel, outcome, status, message }) => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const check = computeNightlyQualityProbeHealthCheck(true, [{ outcome, ts: TS, ...PANELS[panel].fields }]);
expect(check.status).toBe(status);
expect(check.message).toBe(message);
});
const UNCHANGED_CASES = [
{ name: 'no panel fields', fields: {} },
{ name: 'judge_models not a list', fields: { judge_models: 'a,b,a', distinct_judge_models: 2, distinct_judge_providers: 1 } },
{ name: 'a non-string judge model', fields: { judge_models: [SONNET, 7, SONNET], distinct_judge_models: 2, distinct_judge_providers: 1 } },
{ name: 'counts missing', fields: { judge_models: [SONNET, OPUS, SONNET] } },
];
test.each(UNCHANGED_CASES)('$name: the messages stay as before', async ({ fields }) => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
expect(computeNightlyQualityProbeHealthCheck(true, [{ outcome: 'pass', ts: TS, ...fields }]).message).toBe(OK);
expect(computeNightlyQualityProbeHealthCheck(true, [{ outcome: 'fail', ts: TS, ...fields }]).message).toBe(WARN);
});
test('an audit row in the pre-#5506 shape reads back and renders unchanged', async () => {
const { computeQualityProbeAuditFilename, readRecentQualityProbeEvents } =
await import('../src/core/audit-quality-probe.ts');
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const now = new Date('2026-10-02T12:00:00Z');
writeFileSync(join(auditTmp, computeQualityProbeAuditFilename(now)), JSON.stringify({
ts: TS, outcome: 'fail', exit_code: 1, pass_count: 4, fail_count: 6, inconclusive_count: 0,
error_count: 0, est_cost_usd: 2.8, fixture_sha8: 'abcd1234',
}) + '\n');
await withEnv({ GBRAIN_AUDIT_DIR: auditTmp }, async () => {
const events = readRecentQualityProbeEvents(7, now);
expect(events).toHaveLength(1);
expect(computeNightlyQualityProbeHealthCheck(true, events).message).toBe(WARN);
});
});
});
describe('#7145 nightly_quality_probe_health explains a failing judge dimension', () => {
const failure = (id: string, dimension: string, mean: number, scores: number[]) =>
({ question_id: id, verdict: 'fail', dimensions: [{ dimension, mean, scores, fail_reason: 'mean_below_7' }] });
test('one dimension failing while correctness never does: named, observed against the bar, with what it asks and where to look', async () => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const { PROBE_QA_DIMENSIONS } = await import('../src/core/cycle/nightly-probe-adapters.ts');
const events = [
{ outcome: 'fail', ts: '2026-05-20T03:00:00Z', reader_model: 'anthropic:reader-example', failures: [failure('nightly-1', 'directness', 6.3, [6, 7, 6]), failure('nightly-2', 'directness', 5.7, [5, 6, 6])] },
{ outcome: 'fail', ts: '2026-05-21T03:00:00Z', reader_model: 'anthropic:reader-example', receipt_dir: '/synthetic/audit/nightly-probe/run-2', failures: [failure('nightly-1', 'directness', 6.7, [7, 7, 6])] },
];
const check = computeNightlyQualityProbeHealthCheck(true, events, PROBE_QA_DIMENSIONS);
expect(check.status).toBe('warn');
expect(check.message).toContain('Failing judge dimension(s): directness in 2 runs (3 questions, judge mean 5.7-6.7; mean_below_7).');
expect(check.message).toContain('a mean of at least 7 across judges and no judge below 5, on a 1-10 scale');
expect(check.message).toContain('directness asks: Does it answer THIS question without hedging or padding or answering something else?');
expect(check.message).toContain('Correctness never failed');
expect(check.message).toContain('(reader anthropic:reader-example)');
expect(check.message).toContain('/synthetic/audit/nightly-probe/run-2 (lme-output.jsonl, summary.json)');
expect(check.details).toMatchObject({ failing_dimensions: [{ dimension: 'directness', runs: 2, questions: 3, mean_min: 5.7, mean_max: 6.7 }],
bar: { mean_at_least: 7, every_judge_at_least: 5, scale: '1-10' }, correctness_failed: false });
});
test('a correctness failure is not called a style problem', async () => {
const { computeNightlyQualityProbeHealthCheck } = await import('../src/commands/doctor.ts');
const check = computeNightlyQualityProbeHealthCheck(true, [{ outcome: 'fail', ts: '2026-05-21T03:00:00Z', failures: [failure('nightly-3', 'correctness', 3, [3, 3, 3])] }]);
expect(check.message).toContain('correctness in 1 run');
expect(check.message).not.toContain('Correctness never failed');
expect(check.details).toMatchObject({ correctness_failed: true });
});
});