Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix: pre-landing review fixes

garrytan23216fce7c94
diff --git a/src/core/ai/gateway.ts b/src/core/ai/gateway.ts
index ddfad53354..0d8e144388 100644
--- a/src/core/ai/gateway.ts
+++ b/src/core/ai/gateway.ts
@@ -2777,6 +2777,11 @@ export function withBudgetTracker<T>(tracker: BudgetTracker, fn: () => Promise<T
return __budgetStore.run(tracker, fn);
}
+/** Run fn outside any budget scope: calls it makes are not billed to the caller's tracker. */
+export function withoutBudgetTracker<T>(fn: () => Promise<T>): Promise<T> {
+ return __budgetStore.exit(fn);
+}
+
export function getCurrentBudgetTracker(): BudgetTracker | null {
return __budgetStore.getStore() ?? null;
}
diff --git a/src/core/skillopt/validate-gate.ts b/src/core/skillopt/validate-gate.ts
index 63824d4fca..93df7548f8 100644
--- a/src/core/skillopt/validate-gate.ts
+++ b/src/core/skillopt/validate-gate.ts
@@ -22,6 +22,7 @@
import { runWithLimit, type SettledItem } from '../worker-pool.ts';
import { createHash } from 'node:crypto';
+import { withoutBudgetTracker } from '../ai/gateway.ts';
import { isSkilloptMustAbort, SKILLOPT_HOOK_INFRA_ERROR, SKILLOPT_RUNTIME_EXCEEDED } from './must-abort.ts';
import { runRollout, type RolloutOpts } from './rollout.ts';
import { scoreTrajectory } from './score.ts';
@@ -78,7 +79,7 @@ interface PerTask {
/** One hook call for the whole batch; a malformed reply is an infra error, never a score. */
async function hookBatch(opts: ValidateGateOpts, hook: RolloutBatchHook, runsPerTask: number): Promise<PerTask[]> {
if (opts.deadlineMs !== undefined && Date.now() > opts.deadlineMs) throw new Error(SKILLOPT_RUNTIME_EXCEEDED);
- const rows = await hook({
+ const rows = await withoutBudgetTracker(() => hook({
phase: opts.phase ?? 'sel_gate',
skillText: opts.candidateSkillText,
skillSha256: createHash('sha256').update(opts.candidateSkillText).digest('hex'),
@@ -86,7 +87,7 @@ async function hookBatch(opts: ValidateGateOpts, hook: RolloutBatchHook, runsPer
runsPerTask,
epoch: opts.epoch ?? 0,
step: opts.step ?? 0,
- }, opts.abortSignal);
+ }, opts.abortSignal));
const needsTrajectory = opts.phase === 'train_forward';
const badRun = (r: ScoredRollout | undefined) => typeof r?.score !== 'number' || !(r.score >= 0 && r.score <= 1)
|| (needsTrajectory && (typeof r.trajectory !== 'object' || r.trajectory === null));
diff --git a/test/fixtures/goldens/exports/runtime.json b/test/fixtures/goldens/exports/runtime.json
index 736052faed..043b876b15 100644
--- a/test/fixtures/goldens/exports/runtime.json
+++ b/test/fixtures/goldens/exports/runtime.json
@@ -532,7 +532,8 @@
"toolLoop",
"transcribe",
"validateModelId",
- "withBudgetTracker"
+ "withBudgetTracker",
+ "withoutBudgetTracker"
],
"./ai/invocation-guard": [
"aiInvocationObserverFailures",
diff --git a/test/fixtures/goldens/exports/types.json b/test/fixtures/goldens/exports/types.json
index a523e32b53..4da9a0219f 100644
--- a/test/fixtures/goldens/exports/types.json
+++ b/test/fixtures/goldens/exports/types.json
@@ -2496,6 +2496,13 @@
"signature": [
"<T>(tracker: BudgetTracker, fn: () => Promise<T>) => Promise<T>"
]
+ },
+ {
+ "kind": "function",
+ "name": "withoutBudgetTracker",
+ "signature": [
+ "<T>(fn: () => Promise<T>) => Promise<T>"
+ ]
}
],
"./ai/invocation-guard": [
diff --git a/test/skillopt/benchmark-hooks.test.ts b/test/skillopt/benchmark-hooks.test.ts
index 4e2b2dff36..e5e017b40c 100644
--- a/test/skillopt/benchmark-hooks.test.ts
+++ b/test/skillopt/benchmark-hooks.test.ts
@@ -13,6 +13,8 @@ import { describeJudges } from '../../src/core/skillopt/reflect.ts';
import { READ_ONLY_BRAIN_TOOLS, rolloutToolsForSkill } from '../../src/core/skillopt/rollout.ts';
import type { BenchmarkTask, RolloutBatchHook, RolloutBatchRequest, ScoredRollout } from '../../src/core/skillopt/types.ts';
import * as skillopt from '../../src/core/skillopt/index.ts';
+import { getCurrentBudgetTracker, withBudgetTracker } from '../../src/core/ai/gateway.ts';
+import { BudgetTracker } from '../../src/core/budget/budget-tracker.ts';
const engine = {} as BrainEngine;
const task = (id: string, arg = 'x'): BenchmarkTask => ({ task_id: id, task: `do ${id}`, judge: { kind: 'rule', checks: [{ op: 'contains', arg }] } } as BenchmarkTask);
@@ -54,6 +56,19 @@ describe('rollout hook (B1.3)', () => {
expect(isSkilloptMustAbort(err)).toBe(true);
});
+ test('the hook runs outside the run budget, so in-process rollouts are never billed to the optimizer cap', async () => {
+ const seen: Array<BudgetTracker | null> = [];
+ const hook: RolloutBatchHook = async (req) => {
+ seen.push(getCurrentBudgetTracker());
+ await Promise.resolve();
+ seen.push(getCurrentBudgetTracker());
+ return req.tasks.map(() => [row(1), row(1), row(1)]);
+ };
+ const tracker = new BudgetTracker({ maxCostUsd: 1, label: 'skillopt:test' });
+ await withBudgetTracker(tracker, () => scoreSkillPerTask({ engine, skillText: 'S', tasks: tasks('s', 2), targetModel: 'm', rolloutHook: hook }));
+ expect(seen).toEqual([null, null]);
+ });
+
test('a train_forward row without a trajectory is an infra error', async () => {
const hook: RolloutBatchHook = async (req) => req.tasks.map(() => [{ score: 1 } as ScoredRollout]);
const err = await scoreSkillPerTask({ engine, skillText: 'S', tasks: tasks('t', 2), targetModel: 'm', runsPerTask: 1, phase: 'train_forward', rolloutHook: hook })
diff --git a/test/skillopt/run-outcome.test.ts b/test/skillopt/run-outcome.test.ts
index 8a7709b83c..973500fb30 100644
--- a/test/skillopt/run-outcome.test.ts
+++ b/test/skillopt/run-outcome.test.ts
@@ -25,6 +25,7 @@ import {
import {
advanceCursor,
assertResumeCompatible,
+ checkpointOptimizerMode,
rewindCursor,
emptyTally,
resumeCursor,
@@ -280,6 +281,16 @@ describe('resume spec refusal', () => {
expect(() => assertResumeCompatible(stored, { ...SPEC, optimizer_model: 'openai:gpt-5', reflect_max_tokens: 64000, epochs: 4 })).not.toThrow();
});
+ test('optimizer mode: a legacy or reflect checkpoint resumes as the loop and refuses one-shot', () => {
+ expect(checkpointOptimizerMode(stored)).toBe('reflect');
+ expect(checkpointOptimizerMode(checkpoint())).toBe('reflect');
+ expect(() => assertResumeCompatible(stored, { ...SPEC, optimizer_mode: 'reflect' })).not.toThrow();
+ expect(() => assertResumeCompatible(stored, { ...SPEC, optimizer_mode: 'one-shot-rewrite' })).toThrow(/optimizer_mode changed/);
+ const oneShot = checkpoint({ run_spec: { ...SPEC, optimizer_mode: 'one-shot-rewrite' } });
+ expect(checkpointOptimizerMode(oneShot)).toBe('one-shot-rewrite');
+ expect(() => assertResumeCompatible(oneShot, { ...SPEC, optimizer_mode: 'reflect' })).toThrow(/optimizer_mode changed/);
+ });
+
test('legacy checkpoint without a spec still refuses a changed benchmark', () => {
expect(() => assertResumeCompatible(checkpoint(), { ...SPEC, benchmark_sha8: 'bbbb2222' })).toThrow(/benchmark_sha8 changed/);
expect(() => assertResumeCompatible(checkpoint(), SPEC)).not.toThrow();
@@ -325,6 +336,12 @@ describe('resume command', () => {
expect(buildResumeCommand('demo', 'run-1', SPEC, 'reflect_context_too_small')).toContain('--reflect-max-tokens 32000');
});
+ test('a legacy or reflect spec resumes with --loop; a one-shot spec without it', () => {
+ expect(buildResumeCommand('demo', 'run-1', SPEC)).toContain(' --loop');
+ expect(buildResumeCommand('demo', 'run-1', { ...SPEC, optimizer_mode: 'reflect' })).toContain(' --loop');
+ expect(buildResumeCommand('demo', 'run-1', { ...SPEC, optimizer_mode: 'one-shot-rewrite' })).not.toContain('--loop');
+ });
+
test('carries the run mode, cost cap and runtime cap', () => {
const capped = buildResumeCommand('demo', 'run-1', SPEC, undefined, { mode: 'rewrite', maxCostUsd: 20, maxRuntimeMin: 45 });
expect(capped).toContain('--rewrite --max-cost-usd 20 --max-runtime-min 45');