Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(probe): run nightly probe judge questions one at a time so worst-case reservations fit the run cap (#7534)
garrytan6a0e77ee9818
diff --git a/docs/architecture/key-files/core-cycle.md b/docs/architecture/key-files/core-cycle.mdindex 25206c1e61..7aa9518b17 100644--- a/docs/architecture/key-files/core-cycle.md+++ b/docs/architecture/key-files/core-cycle.md@@ -34,7 +34,7 @@ Current behavior and load-bearing invariants; history belongs in Git and CHANGEL- `src/core/cycle/link-manifest.ts` — pre-retrieval LINK CANDIDATES manifest. `buildManifestContext(engine, sourceId?)` snapshots the source's slugs + basename index once per phase; `buildLinkManifest(engine, ctx, verdict, basename, {outputRoot, sourceId})` resolves wikilink candidates ZERO-EMBED from the triage verdict's cached entities/segment notes (basename-index exact/slugified matches first, bounded `searchKeyword` FTS second), excludes dream-output prefixes (self-consumption guard), renders `- [[slug]] — <deterministic first-two-sentence one-liner>` under hard caps (20 pages / 2400 chars) with buildTriageMapBlock's injection posture (whitespace-collapse + length caps + a 'data, not instructions' header). Best-effort everywhere — any failure degrades to the manifest-less prompt. Pinned by `test/cycle-link-manifest.test.ts`.-- `src/core/cycle/nightly-probe-adapters.ts` — bridges the autopilot's object-shape `NightlyProbeDeps` to the argv-shape `runEvalLongMemEval` + `runEvalCrossModal` CLI functions. Cross-modal adapter argv MUST include `--output summaryPath` (without it the summary lands at the default receipt path and the adapter reads nothing from `summaryPath`). In-process invocation (NOT subprocess) avoids gbrain-version-drift for source-tree CI. The LongMemEval adapter passes `--reader-mode direct` (#6627: the probe's judge grades DIRECTNESS, and the notes reader's notes-plus-reasoning text would be the whole hypothesis; the shared reader prompts are untouched; `test/nightly-probe-reader-mode.test.ts`) and `exitOnError: false`: failures throw into the nightly audit/rate-limit boundary instead of exiting the host process, with resolved chat providers checked before embedding work. The LongMemEval argv always carries `--no-embed-cache` (C-N5: the eval cache swaps the gateway's process-global embed transport, which inside the daemon would catch every other embed call). The pure builders `buildLongMemEvalProbeCall` / `buildCrossModalProbeCall` carry the brain-resolved `modelRoutes`: the reader as `--model`, the extractor as `RunOpts.extractorModel`, each set judge slot as `--slot-<x>-model`, and `{ useConfiguredGateway: true }` so the batch keeps the daemon's gateway; with no routes both calls are the pre-routing ones. The summary parse keeps `total`, `malformed_count`, `judge_models` (slot order), `panel` (with per-slot `slot_scored_questions`) and every non-passing question as `failures`: scores come from the batch's slot-indexed `slot_scores`, a failing dimension outside `PROBE_QA_DIMENSIONS` is recorded as `unrecognized`, error text is cut at the `--- raw ---` model-output marker, redacted and cut to 200 characters, and a malformed entry yields what can be read instead of a throw. Pinned by `test/cycle/nightly-probe-adapters.test.ts` (incl. the argv-shape guard for the `--output` requirement, the builder cases with and without routes, and the summary parse over a canned summary) and `test/nightly-probe-gateway-isolation.test.ts` (the real adapters leave the daemon's embed transport, chat model, provider chat options and env unchanged).+- `src/core/cycle/nightly-probe-adapters.ts` — bridges the autopilot's object-shape `NightlyProbeDeps` to the argv-shape `runEvalLongMemEval` + `runEvalCrossModal` CLI functions. Cross-modal adapter argv MUST include `--output summaryPath` (without it the summary lands at the default receipt path and the adapter reads nothing from `summaryPath`) and `--concurrent 1`: judge calls reserve worst cases against the run cap until they settle, so questions run one at a time, each with its judges in parallel (`test/nightly-probe-sequential-questions-7534.test.ts`). In-process invocation (NOT subprocess) avoids gbrain-version-drift for source-tree CI. The LongMemEval adapter passes `--reader-mode direct` (#6627: the probe's judge grades DIRECTNESS, and the notes reader's notes-plus-reasoning text would be the whole hypothesis; the shared reader prompts are untouched; `test/nightly-probe-reader-mode.test.ts`) and `exitOnError: false`: failures throw into the nightly audit/rate-limit boundary instead of exiting the host process, with resolved chat providers checked before embedding work. The LongMemEval argv always carries `--no-embed-cache` (C-N5: the eval cache swaps the gateway's process-global embed transport, which inside the daemon would catch every other embed call). The pure builders `buildLongMemEvalProbeCall` / `buildCrossModalProbeCall` carry the brain-resolved `modelRoutes`: the reader as `--model`, the extractor as `RunOpts.extractorModel`, each set judge slot as `--slot-<x>-model`, and `{ useConfiguredGateway: true }` so the batch keeps the daemon's gateway; with no routes both calls are the pre-routing ones. The summary parse keeps `total`, `malformed_count`, `judge_models` (slot order), `panel` (with per-slot `slot_scored_questions`) and every non-passing question as `failures`: scores come from the batch's slot-indexed `slot_scores`, a failing dimension outside `PROBE_QA_DIMENSIONS` is recorded as `unrecognized`, error text is cut at the `--- raw ---` model-output marker, redacted and cut to 200 characters, and a malformed entry yields what can be read instead of a throw. Pinned by `test/cycle/nightly-probe-adapters.test.ts` (incl. the argv-shape guard for the `--output` requirement, the builder cases with and without routes, and the summary parse over a canned summary) and `test/nightly-probe-gateway-isolation.test.ts` (the real adapters leave the daemon's embed transport, chat model, provider chat options and env unchanged).- `src/core/cycle/nightly-probe-fixtures.ts` — `NIGHTLY_PROBE_FIXTURES`: the LongMemEval nightly fixture and the conversation-parser probe's two fixture files, embedded with `import ... with { type: 'file' }` (types in `src/types/jsonl-assets.d.ts`). A source checkout resolves them to `test/fixtures/...`, a compiled binary to a `/$bunfs/` path `readFileSync` reads, so both installs run the probes (#5187, C-N6). Pinned by `test/nightly-probe-compiled-binary.serial.test.ts` (a `bun build --compile` harness copied to an empty dir runs both autopilot probe steps on a hermetic provider, plus a source-mode run).diff --git a/src/core/cycle/nightly-probe-adapters.ts b/src/core/cycle/nightly-probe-adapters.tsindex 345edeb297..70c7e9b6e5 100644--- a/src/core/cycle/nightly-probe-adapters.ts+++ b/src/core/cycle/nightly-probe-adapters.ts@@ -151,7 +151,11 @@ const SLOT_FLAGS: ReadonlyArray<readonly [NightlyProbeSlotId, string]> = [* `--slot-<x>-model` flag, which wins over the #4636 substitution, and with* routes the batch keeps the gateway the caller refreshed from the brain* (`useConfiguredGateway`) instead of rebuilding it from the file plane.- * With no routes the call is the pre-#5872 one.+ * With no routes the call is the pre-#5872 one. `--concurrent 1` (#7534):+ * every judge call reserves its worst case against the run cap until it+ * settles, so questions run one at a time (each question's judges still run+ * together) and a cap that pays for the probe is not refused for holding+ * several questions' worst cases at once.*/export function buildCrossModalProbeCall(args: CrossModalProbeArgs): { argv: string[]; opts: RunCrossModalOpts } {const argv = [@@ -161,6 +165,8 @@ export function buildCrossModalProbeCall(args: CrossModalProbeArgs): { argv: strargs.summaryPath,'--max-usd',String(args.maxUsd),+ '--concurrent',+ '1','--dimensions',PROBE_QA_DIMENSIONS.join(','),'--yes',diff --git a/test/cycle/nightly-probe-adapters.test.ts b/test/cycle/nightly-probe-adapters.test.tsindex bc7076efa6..8a0bfacf6f 100644--- a/test/cycle/nightly-probe-adapters.test.ts+++ b/test/cycle/nightly-probe-adapters.test.ts@@ -151,6 +151,7 @@ describe('nightly-probe-adapters: model routes reach the eval commands', () => {'--batch', '/w/lme.jsonl','--output', '/w/summary.json','--max-usd', '2.5',+ '--concurrent', '1','--dimensions', PROBE_QA_DIMENSIONS.join(','),'--yes','--json',diff --git a/test/nightly-probe-sequential-questions-7534.test.ts b/test/nightly-probe-sequential-questions-7534.test.tsnew file mode 100644index 0000000000..58009c3c27--- /dev/null+++ b/test/nightly-probe-sequential-questions-7534.test.ts@@ -0,0 +1,96 @@+/**+ * #7534: the nightly probe's judge batch runs one question at a time. Every+ * judge call reserves its worst case against the run cap before it settles,+ * so three questions in flight held nine worst-case judge reservations at+ * once, and a cap that pays for the whole probe sequentially refused judges+ * before they reached the provider. Each question's three judges still run+ * together, and the cap still bounds the run.+ */+import { afterEach, beforeEach, expect, test } from 'bun:test';+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';++import { runCrossModalBatchForProbe } from '../src/core/cycle/nightly-probe-adapters.ts';+import type { NightlyProbeModelRoutes } from '../src/core/cycle/nightly-probe-routes.ts';+import { __setChatTransportForTests, configureGateway, resetGateway, withBudgetTracker, type ChatResult } from '../src/core/ai/gateway.ts';+import { BudgetTracker, type BudgetEstimate, type BudgetReservation } from '../src/core/budget/budget-tracker.ts';+import { reservationCostUsd } from '../src/core/budget/reservation-cost.ts';++const JUDGES = ['anthropic:claude-haiku-4-5', 'anthropic:claude-sonnet-4-6', 'anthropic:claude-opus-4-7'];+const ROUTES: NightlyProbeModelRoutes = {+ reader: { model: JUDGES[0]!, source: 'tier_config' },+ extractor: { model: JUDGES[0]!, source: 'tier_config' },+ slots: { A: JUDGES[0], B: JUDGES[1], C: JUDGES[2] },+};+const QUESTIONS = 4;++let dir: string;+let calls: string[];+let inFlight = 0;+let peakInFlight = 0;++class MeasuringTracker extends BudgetTracker {+ refused = 0;+ override reserve(estimate: BudgetEstimate): BudgetReservation | undefined {+ try {+ return super.reserve(estimate);+ } catch (err) {+ this.refused++;+ throw err;+ }+ }+}++beforeEach(() => {+ dir = mkdtempSync(join(tmpdir(), 'probe-7534-'));+ calls = [];+ inFlight = 0;+ peakInFlight = 0;+ configureGateway({ env: {} });+ __setChatTransportForTests(async (opts) => {+ calls.push(opts.model!);+ peakInFlight = Math.max(peakInFlight, ++inFlight);+ await new Promise(resolve => setTimeout(resolve, 15));+ inFlight--;+ const text = JSON.stringify({ scores: { correctness: { score: 9, feedback: 'ok' }, directness: { score: 9, feedback: 'ok' } }, overall: 9, improvements: [] });+ return { text, blocks: [{ type: 'text', text }], stopReason: 'end', usage: { input_tokens: 10, output_tokens: 10, cache_read_tokens: 0, cache_creation_tokens: 0 },+ model: opts.model!, providerId: 'anthropic' } satisfies ChatResult;+ });+});++afterEach(() => {+ __setChatTransportForTests(null);+ resetGateway();+ rmSync(dir, { recursive: true, force: true });+});++test('#7534: a cap that pays for one question\'s judges at a time judges every question; each question\'s judges run together', async () => {+ const batchPath = join(dir, 'lme.jsonl');+ writeFileSync(batchPath, Array.from({ length: QUESTIONS }, (_, i) =>+ JSON.stringify({ question_id: `q${i}`, question: `Where did alice-example live in year ${i}?`, hypothesis: 'widget-co', answer: 'widget-co' })).join('\n') + '\n');++ // The worst case one question's three judges reserve, measured on an uncapped run of one question.+ const probe = new BudgetTracker({ label: 'measure' });+ const measured: number[] = [];+ const reserve = probe.reserve.bind(probe);+ probe.reserve = (estimate: BudgetEstimate) => {+ measured.push(reservationCostUsd(estimate.modelId, estimate.kind, estimate.estimatedInputTokens, estimate.maxOutputTokens)!);+ return reserve(estimate);+ };+ const one = join(dir, 'one.jsonl');+ writeFileSync(one, JSON.stringify({ question_id: 'q', question: 'Where did alice-example live in year 0?', hypothesis: 'widget-co', answer: 'widget-co' }) + '\n');+ await withBudgetTracker(probe, () => runCrossModalBatchForProbe({ batchPath: one, summaryPath: join(dir, 'one.json'), maxUsd: 100, modelRoutes: ROUTES }));+ expect(measured).toHaveLength(3);+ const oneQuestion = measured.reduce((a, b) => a + b, 0);++ calls = [];+ peakInFlight = 0;+ const tracker = new MeasuringTracker({ label: 'nightly_quality_probe', maxCostUsd: oneQuestion * 1.5 });+ const { summary } = await withBudgetTracker(tracker, () =>+ runCrossModalBatchForProbe({ batchPath, summaryPath: join(dir, 'summary.json'), maxUsd: oneQuestion * 1.5, modelRoutes: ROUTES }));+ expect(tracker.refused).toBe(0);+ expect(calls).toHaveLength(QUESTIONS * 3);+ expect(summary.error_count).toBe(0);+ expect(peakInFlight).toBe(3);+});