Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* SkillOpt E2E (PGLite + DI'd LLM): 3 cases — accept + reject + resume.
*
* Hermetic: no real LLM calls. Stubbed via opts.deps for orchestrator AND
* runValidationGate's rolloutFn/scoreFn seams. PGLite engine + tempdir
* SKILL.md + tempdir benchmark for full file-system coverage.
*
* .serial.test.ts because it walks a full multi-epoch loop with shared
* disk state.
*/
import { describe, expect, test, beforeAll, afterAll, beforeEach } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
import { resetPgliteState } from '../helpers/reset-pglite.ts';
import { withEnv } from '../helpers/with-env.ts';
import {
bestPath,
loadHistory,
skillPath,
} from '../../src/core/skillopt/version-store.ts';
let engine: PGLiteEngine;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
});
afterAll(async () => {
await engine.disconnect();
});
beforeEach(async () => {
await resetPgliteState(engine);
});
const SKILL = 'e2e-test-skill';
const SAMPLE_SKILL = `---
name: e2e-test-skill
version: 0.1.0
description: Test skill for E2E SkillOpt loop.
triggers:
- "do the example task"
brain_first: exempt
---
# E2E Test Skill
When asked, produce a structured output with:
## People
List people mentioned.
## Citations
Cite sources.
`;
// 50 tasks so split 4:1:5 → 20 train / 5 sel / 25 test (D17 floor satisfied).
const SAMPLE_BENCHMARK = Array.from({ length: 50 }, (_, i) => {
const n = String(i + 1).padStart(3, '0');
const op = i % 2 === 0 ? 'People' : 'Citations';
return {
task_id: `e2e-${n}`,
task: `Process task ${i + 1}`,
judge: { kind: 'rule' as const, checks: [{ op: 'contains' as const, arg: op }] },
};
});
function setupSkillFixture(): { skillsDir: string; benchmarkPath: string; cleanup: () => void } {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'skillopt-e2e-'));
const skillDir = path.join(tmp, SKILL);
fs.mkdirSync(skillDir, { recursive: true });
fs.writeFileSync(path.join(skillDir, 'SKILL.md'), SAMPLE_SKILL);
const benchmarkPath = path.join(skillDir, 'skillopt-benchmark.jsonl');
fs.writeFileSync(benchmarkPath, SAMPLE_BENCHMARK.map((t) => JSON.stringify(t)).join('\n') + '\n');
return {
skillsDir: tmp,
benchmarkPath,
cleanup: () => { try { fs.rmSync(tmp, { recursive: true, force: true }); } catch { /* ignore */ } },
};
}
describe('skillopt E2E (PGLite + DI LLM)', () => {
test('dry-run mode: cost preview, no LLM calls, exits with aborted outcome', async () => {
const fixture = setupSkillFixture();
try {
await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => {
const { runSkillOpt } = await import('../../src/core/skillopt/orchestrator.ts');
const result = await runSkillOpt({
engine,
skillName: SKILL,
skillsDir: fixture.skillsDir,
benchmarkPath: fixture.benchmarkPath,
epochs: 1,
batchSize: 4,
lr: 4,
lrSchedule: 'cosine',
split: [4, 1, 5],
optimizerModel: 'anthropic:claude-opus-4-7',
targetModel: 'anthropic:claude-sonnet-4-6',
judgeModel: 'anthropic:claude-sonnet-4-6',
mode: 'patch', optimizerMode: 'reflect',
dryRun: true, // SHORT-CIRCUITS
noMutate: false,
allowMutateBundled: true,
bootstrapReviewed: false,
json: true,
maxCostUsd: 100, // dry-run: cap is irrelevant
maxRuntimeMin: 1,
force: false,
});
expect(result.outcome).toBe('aborted'); // dry-run convention
expect(result.mutatedSkillFile).toBe(false);
// SKILL.md unchanged.
expect(fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8')).toBe(SAMPLE_SKILL);
// No history file.
expect(loadHistory(fixture.skillsDir, SKILL)).toEqual([]);
});
} finally {
fixture.cleanup();
}
});
test('all-reject path: validation gate refuses every candidate, exits no_improvement', async () => {
const fixture = setupSkillFixture();
try {
await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => {
// Stub the runValidationGate by stubbing the gateway.chat that
// underlies score.scoreLlm. For rule-only benchmarks, no chat is
// called — runRollout's toolLoop IS called. Stub gateway.toolLoop
// to return an empty trajectory; runRollout then yields finalText='',
// which fails all `contains` checks. Score is 0 ⇒ gate rejects.
const { __setChatTransportForTests } = await import('../../src/core/ai/gateway.ts');
// Stub chat for any reflect/judge call.
__setChatTransportForTests(async () => ({
text: '{"edits": []}', // empty edit set — nothing applied
blocks: [],
stopReason: 'end' as const,
usage: {
input_tokens: 100,
output_tokens: 20,
cache_read_tokens: 0,
cache_creation_tokens: 0,
},
model: 'stub',
providerId: 'stub',
}));
try {
const { runSkillOpt } = await import('../../src/core/skillopt/orchestrator.ts');
// Cap LOW so the run aborts even before completing all epochs.
const result = await runSkillOpt({
engine,
skillName: SKILL,
skillsDir: fixture.skillsDir,
benchmarkPath: fixture.benchmarkPath,
epochs: 1,
batchSize: 2,
lr: 2,
lrSchedule: 'constant',
split: [4, 1, 5],
optimizerModel: 'anthropic:claude-opus-4-7',
targetModel: 'anthropic:claude-sonnet-4-6',
judgeModel: 'anthropic:claude-sonnet-4-6',
mode: 'patch', optimizerMode: 'reflect',
dryRun: false,
noMutate: false,
allowMutateBundled: true,
bootstrapReviewed: false,
json: true,
maxCostUsd: 100, // preflight estimator passes; LLM is stubbed so no real spend
maxRuntimeMin: 1,
force: true, // bypass dirty-tree (tempdir isn't a git repo)
});
// The outcome is no_improvement OR aborted (budget might trip first
// on a real-LLM stub that returns usage). Both are valid "didn't
// mutate" outcomes for the E2E contract.
expect(['no_improvement', 'aborted', 'errored']).toContain(result.outcome);
// SKILL.md MUST be unchanged on these outcomes.
if (result.outcome !== 'accepted') {
expect(fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8')).toBe(SAMPLE_SKILL);
}
} finally {
__setChatTransportForTests(null);
}
});
} finally {
fixture.cleanup();
}
});
test('resume after revertAllPending: prior baseline restored, fresh run completes', async () => {
const fixture = setupSkillFixture();
try {
await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => {
// Pre-stage a pending row + a corrupted best.md.
const { acceptCandidate, bestPath: bp, historyPath, versionsDir, revertAllPending, loadHistory: lh } = await import('../../src/core/skillopt/version-store.ts');
// First: a clean v1 accept.
const v1 = '---\nname: e2e-test-skill\n---\nclean v1\n';
acceptCandidate({
skillsDir: fixture.skillsDir, skillName: SKILL, runId: 'prior-run',
epoch: 1, step: 1, edits: [], candidateText: v1, selScore: 0.5, delta: 0.5,
});
// Now stage a pending v2 that "crashed".
fs.writeFileSync(path.join(versionsDir(fixture.skillsDir, SKILL), 'v0002_e1_s2.md'), 'corrupted v2');
fs.writeFileSync(bp(fixture.skillsDir, SKILL), 'corrupted v2');
const history = lh(fixture.skillsDir, SKILL);
history.push({
status: 'pending', run_id: 'crashed-run', version_n: 2,
ts: '2026-05-27T13:00:00Z', edits: [], sel_score: 0.6, delta: 0.1,
});
fs.writeFileSync(historyPath(fixture.skillsDir, SKILL), JSON.stringify({ schema: 1, rows: history }));
// Revert.
revertAllPending(fixture.skillsDir, SKILL);
// best.md restored to v1.
expect(fs.readFileSync(bp(fixture.skillsDir, SKILL), 'utf8')).toBe(v1);
// History has only the clean v1.
const final = loadHistory(fixture.skillsDir, SKILL);
expect(final).toHaveLength(1);
expect(final[0]!.status).toBe('committed');
expect(final[0]!.version_n).toBe(1);
});
} finally {
fixture.cleanup();
}
});
});