Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { describe, expect, test } from 'bun:test';
import { mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { spawnSync } from 'node:child_process';
import { load } from 'js-yaml';
import { verifyNightlyE2E } from '../../scripts/verify-nightly-e2e.ts';
import { classifyShards, main as classifyMain, DOCS as CLASSIFY_DOCS, type ShardJob } from '../../scripts/classify-full-e2e-shards.ts';
const repo = join(import.meta.dir, '../..');
const fullProfile = "github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.full_corpus)";
const expected = Array.from({ length: 4 }, (_, i) => [`test/e2e/fixture-${i}-a.test.ts`, `test/e2e/fixture-${i}-b.test.ts`]);
function fixture(fn: (root: string) => void, partitions = expected) {
const root = mkdtempSync(join(tmpdir(), 'gbrain-nightly-e2e-'));
try {
for (let index = 0; index < partitions.length; index++) {
const dir = join(root, `coverage-full-e2e-${index + 1}`);
mkdirSync(dir);
writeFileSync(join(dir, 'lane-manifest.json'), JSON.stringify({ lane: `e2e-${index + 1}`, sha: 'fixture-sha', lcovCount: 2, complete: true }));
writeFileSync(join(dir, 'executed-files.txt'), partitions[index].join('\n') + '\n');
}
fn(root);
} finally { rmSync(root, { recursive: true, force: true }); }
}
describe('nightly E2E execution receipts', () => {
test('accepts exactly four complete disjoint partitions of the same commit', () => fixture(root => {
expect(() => verifyNightlyE2E(root, expected, 'fixture-sha')).not.toThrow();
}));
test.each(['missing artifact', 'missing manifest', 'missing receipt', 'wrong SHA', 'incomplete', 'duplicate lane', 'wrong lane', 'missing file', 'duplicate file', 'extra file', 'overlapping partitions', 'malformed manifest'])('%s fails closed', kind => fixture(root => {
const dir = join(root, 'coverage-full-e2e-4');
const manifest = join(dir, 'lane-manifest.json');
const receipt = join(dir, 'executed-files.txt');
const data = JSON.parse(readFileSync(manifest, 'utf8'));
if (kind === 'missing artifact') rmSync(dir, { recursive: true });
if (kind === 'missing manifest') rmSync(manifest);
if (kind === 'missing receipt') rmSync(receipt);
if (kind === 'wrong SHA') writeFileSync(manifest, JSON.stringify({ ...data, sha: 'stale-commit' }));
if (kind === 'incomplete') writeFileSync(manifest, JSON.stringify({ ...data, complete: false }));
if (kind === 'duplicate lane') writeFileSync(manifest, JSON.stringify({ ...data, lane: 'e2e-1' }));
if (kind === 'wrong lane') writeFileSync(manifest, JSON.stringify({ ...data, lane: 'e2e-5' }));
if (kind === 'missing file') writeFileSync(receipt, expected[3][0] + '\n');
if (kind === 'duplicate file') writeFileSync(receipt, [expected[3][0], expected[3][0]].join('\n'));
if (kind === 'extra file') writeFileSync(receipt, [...expected[3], 'test/e2e/extra.test.ts'].join('\n'));
if (kind === 'overlapping partitions') writeFileSync(receipt, expected[0].join('\n'));
if (kind === 'malformed manifest') writeFileSync(manifest, '{');
expect(() => verifyNightlyE2E(root, expected, 'fixture-sha')).toThrow();
}));
test('an absent artifact directory cannot claim complete execution', () => fixture(root => {
expect(() => verifyNightlyE2E(join(root, 'absent'), expected, 'fixture-sha')).toThrow();
}));
});
describe('nightly E2E scheduling', () => {
test('full-profile shell flags use environment data instead of expression interpolation', () => {
const workflow = load(readFileSync(join(repo, '.github/workflows/e2e.yml'), 'utf8')) as any;
const steps = [
workflow.jobs['prepare-e2e'].steps.find((step: any) => step.id === 'select'),
workflow.jobs['e2e-status'].steps.find((step: any) => step.name === 'Aggregate result'),
];
for (const step of steps) {
expect(step.env?.FULL_CORPUS).toBe('${{ ' + fullProfile + ' }}');
expect(step.run).toContain('if [ "$FULL_CORPUS" = "true" ]; then');
expect(step.run).not.toContain('${{ ' + fullProfile + ' }}');
}
});
test('the four actual runner partitions cover the complete default discovery exactly once', () => {
const discover = (shard: string) => {
const result = spawnSync('bash', ['scripts/run-e2e.sh', '--dry-run-list'], {
cwd: repo, encoding: 'utf8', env: { ...process.env, SHARD: shard, COVERAGE_DIR: '' },
});
expect(result.status, result.stderr).toBe(0);
return result.stdout.trim().split('\n').filter(Boolean);
};
const liveKeyOnly = readFileSync(join(repo, 'scripts/e2e-live-key-only.txt'), 'utf8')
.split('\n').map(line => line.trim()).filter(line => line && !line.startsWith('#'));
const corpus = [...readdirSync(join(repo, 'test/e2e')).filter(file => file.endsWith('.test.ts')).map(file => `test/e2e/${file}`).filter(file => !liveKeyOnly.includes(file)), 'test/phantom-redirect-engine-parity.test.ts'].sort();
expect(discover('').sort()).toEqual(corpus);
const byName = spawnSync('bash', ['scripts/run-e2e.sh', '--dry-run-list', ...liveKeyOnly], { cwd: repo, encoding: 'utf8', env: { ...process.env, SHARD: '', COVERAGE_DIR: '' } });
expect(byName.stdout.trim().split('\n')).toEqual(liveKeyOnly);
const partitions = [1, 2, 3, 4].map(n => discover(`${n}/4`));
expect(partitions.every(files => files.length > 0)).toBe(true);
expect(partitions.flat().sort()).toEqual(corpus);
expect(new Set(partitions.flat()).size).toBe(corpus.length);
fixture(root => {
const verify = () => spawnSync(process.execPath, ['scripts/verify-nightly-e2e.ts', root, '4', 'fixture-sha'], {
cwd: repo, encoding: 'utf8', env: { ...process.env, SHARD: '3/4', COVERAGE_DIR: join(root, 'unused') },
});
const complete = verify();
expect(complete.status, complete.stderr).toBe(0);
expect(complete.stdout).toContain(`all ${corpus.length} files accounted for across 4 isolated workers`);
rmSync(join(root, 'coverage-full-e2e-4/executed-files.txt'));
expect(verify().status).toBe(1);
}, partitions);
});
test('nightly jobs preserve job-local Postgres, independent artifacts, always-reporting and required execution evidence', () => {
const workflow = load(readFileSync(join(repo, '.github/workflows/e2e.yml'), 'utf8')) as any;
const job = workflow.jobs['coverage-full-e2e'];
expect(job.if).toBe(fullProfile);
expect(job.strategy).toEqual({ 'fail-fast': false, matrix: { shard: [1, 2, 3, 4] } });
expect(job.services.postgres.image).toBe('pgvector/pgvector:pg16');
const run = job.steps.find((step: any) => step.run === 'bash scripts/run-e2e.sh');
expect(run.env.SHARD).toBe('${{ matrix.shard }}/4');
expect(run.env.DATABASE_URL).toContain('/gbrain_test');
expect(run.env.COVERAGE_DIR).toBe('${{ runner.temp }}/coverage');
const bootstrap = job.steps.find((step: any) => step.name === 'Bootstrap isolated E2E schema');
expect(job.steps.indexOf(bootstrap)).toBeLessThan(job.steps.indexOf(run));
expect(bootstrap.env).toEqual({ DATABASE_URL: run.env.DATABASE_URL, GBRAIN_CI_DISABLE_TEST_ENV_FILE: '1', GBRAIN_MODEL_DISCOVERY: 'off' });
fixture(root => {
const home = join(root, 'home');
mkdirSync(join(home, '.gbrain'), { recursive: true });
const sentinel = join(home, '.gbrain/config.json');
writeFileSync(sentinel, '{"doNotTouch":true}\n');
const result = spawnSync('bash', ['-e', '-c', bootstrap.run], {
cwd: repo, encoding: 'utf8', timeout: 10000,
env: { ...process.env, ...bootstrap.env, RUNNER_TEMP: root, HOME: home, GBRAIN_HOME: home,
DATABASE_URL: 'postgresql://localhost:1/production', GBRAIN_DATABASE_URL: 'postgresql://localhost:1/also_production',
GBRAIN_E2E_ALLOW_DB: 'production', OPENAI_API_KEY: 'synthetic-bootstrap-canary' },
});
expect(result.status).not.toBe(0);
expect(result.stderr).toContain('does not look like a test database');
expect(result.stderr).not.toContain('synthetic-bootstrap-canary');
expect(readFileSync(sentinel, 'utf8')).toBe('{"doNotTouch":true}\n');
expect(readdirSync(root).some(name => name.startsWith('e2e-bootstrap.'))).toBe(false);
});
for (const [stepName, artifact] of [['Upload coverage', 'coverage-full-e2e-'], ['Upload execution receipt', 'e2e-full-execution-']]) {
const step = job.steps.find((step: any) => step.name === stepName);
expect(step.if).toBe('always()');
expect(step.with.name).toBe(artifact + '${{ matrix.shard }}');
}
const report = workflow.jobs['coverage-full-report'];
expect(report.needs).toContain('coverage-full-e2e');
expect(report.if).toBe(`always() && (${fullProfile})`);
const mergeStep = report.steps.find((step: any) => step.name === 'Merge full corpus');
const merge = mergeStep.run;
expect(merge).toContain('scripts/verify-nightly-e2e.ts "$RUNNER_TEMP/coverage-artifacts" 4 "$GITHUB_SHA"');
expect(merge).toContain(',e2e-1,e2e-2,e2e-3,e2e-4');
const classify = report.steps.find((step: any) => step.name === 'Classify full E2E shards');
expect(report.steps.indexOf(classify)).toBeLessThan(report.steps.indexOf(mergeStep));
expect(classify.id).toBe('shards');
expect(classify.run).toBe('bun scripts/classify-full-e2e-shards.ts');
expect(classify.env).toEqual({ FULL_E2E_RESULT: '${{ needs.coverage-full-e2e.result }}', GH_TOKEN: '${{ github.token }}' });
expect(report.permissions).toEqual({ contents: 'read', actions: 'read', checks: 'read' });
// Status functions such as cancelled() are refused outside `if:` (HTTP 422
// at dispatch), and inside an always() job started after a cancel they
// read false (run 37371914954), so no step may lean on one.
for (const step of report.steps) expect(JSON.stringify(step)).not.toMatch(/\bcancelled\(\)/);
for (const name of ['Merge full corpus', 'Coverage summary → step summary', 'Baseline gate (fullCorpus, like-for-like)']) {
expect(report.steps.find((step: any) => step.name === name).if).toBe("steps.shards.outputs.state == 'complete'");
}
expect(CLASSIFY_DOCS).toBe('docs/operations/verify-and-nightly-e2e.md#full-corpus-report-states');
expect(readFileSync(join(repo, 'docs/operations/verify-and-nightly-e2e.md'), 'utf8')).toMatch(/^## Full-corpus report states$/m);
const status = workflow.jobs['e2e-status'];
expect(status.if).toBe("${{ needs.prepare-e2e.outputs.reuse != 'true' && always() }}");
expect(status.needs).toContain('coverage-full-e2e');
expect(status.needs).not.toContain('coverage-full-report');
const validate = status.steps.find((step: any) => step.name === 'Verify complete nightly execution');
expect(validate.if).toBe(fullProfile);
expect(validate.run).toBe('bun scripts/verify-nightly-e2e.ts "$RUNNER_TEMP/e2e-execution" 4 "$GITHUB_SHA"');
});
test('manual full corpus is opt-in and uses the complete scheduled profile without cancelling ordinary runs', () => {
const workflow = load(readFileSync(join(repo, '.github/workflows/e2e.yml'), 'utf8')) as any;
expect(workflow.on.workflow_dispatch.inputs.full_corpus).toMatchObject({ type: 'boolean', default: false });
for (const name of ['coverage-full-unit', 'coverage-full-serial', 'coverage-full-slow', 'coverage-full-e2e']) expect(workflow.jobs[name].if).toBe(fullProfile);
const [, firstCheckout, retryCheckout, ...rest] = workflow.jobs['e2e-status'].steps;
expect(firstCheckout.if).toBe(fullProfile);
expect(retryCheckout.if).toBe(`steps.${firstCheckout.id}.outcome == 'failure'`);
for (const step of rest) expect(step.if).toBe(fullProfile);
const selection = workflow.jobs['prepare-e2e'].steps.find((step: any) => step.id === 'select');
const select = selection.run;
expect(selection.env.FULL_CORPUS).toBe('${{ ' + fullProfile + ' }}');
const group = workflow.concurrency.group;
expect(group).toBe("${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.sha) || github.ref }}${{ github.event_name == 'workflow_dispatch' && inputs.full_corpus && '-full-corpus' || '' }}${{ github.event_name == 'schedule' && '-nightly' || '' }}");
for (const [event, enabled, expected] of [
['pull_request', false, false], ['pull_request', true, false], ['push', true, false],
['workflow_dispatch', false, false], ['workflow_dispatch', true, true], ['schedule', false, true],
] as const) {
const condition = fullProfile.replaceAll('github.event_name', JSON.stringify(event)).replaceAll('inputs.full_corpus', String(enabled));
expect(new Function(`return (${condition})`)()).toBe(expected);
fixture(root => {
const bin = join(root, 'bin');
mkdirSync(bin);
writeFileSync(join(bin, 'git'), '#!/bin/sh\nexit 0\n', { mode: 0o755 });
writeFileSync(join(bin, 'bun'), `#!/bin/sh
case "$*" in
"scripts/select-e2e.ts --changed-files "*) echo test/e2e/selected.test.ts ;;
"scripts/e2e-matrix.ts prepare") cat > "$RUNNER_TEMP/received-selection"; echo '{}' ;;
*) exit 2 ;;
esac
`, { mode: 0o755 });
const run = spawnSync('bash', ['-e', '-c', select], {
encoding: 'utf8', env: { ...process.env, FULL_CORPUS: String(expected), PATH: `${bin}:${process.env.PATH}`, RUNNER_TEMP: root, GITHUB_OUTPUT: join(root, 'outputs') },
});
expect(run.status, run.stderr).toBe(0);
expect(readFileSync(join(root, 'received-selection'), 'utf8')).toBe(expected ? '' : 'test/e2e/selected.test.ts\n');
});
}
});
});
describe('full E2E shard classification (#6040)', () => {
const run = { repo: 'acme-example/brain', runId: '42', attempt: '1', serverUrl: 'https://example.test' };
// Annotation texts recorded from real runs: 37273083933 (cancelled by a
// newer push, concurrency) and 37371914954 (cancelled by a user).
const CONCURRENCY = 'Canceling since a higher priority waiting request for E2E Tests-refs/heads/master exists';
const USER = 'The run was canceled by @capy-ai[bot].';
const TIMEOUT = 'The job running on runner ubicloud-runner-1 has exceeded the maximum execution time of 60 minutes.';
const OP = 'The operation was canceled.';
const shard = (n: number, conclusion: string | null, annotations: string[] = []): ShardJob => ({ name: `coverage-full-e2e (${n})`, conclusion, annotations });
const all = (conclusion: string, annotations: string[]) => [1, 2, 3, 4].map(n => shard(n, conclusion, annotations));
test.each([
['all shards succeeded', 'success', all('success', []), 'complete'],
['cancelled by a newer run in the concurrency group', 'cancelled', all('cancelled', [CONCURRENCY, OP]), 'cancelled'],
['cancelled by a user', 'cancelled', all('cancelled', [USER, OP]), 'cancelled'],
['some shards finished before the run was cancelled', 'cancelled', [shard(1, 'success'), shard(2, 'cancelled', [USER, OP]), shard(3, 'success'), shard(4, 'cancelled', [USER])], 'cancelled'],
['a shard hit timeout-minutes in a run nobody cancelled', 'cancelled', [shard(1, 'success'), shard(2, 'cancelled', [TIMEOUT, OP]), shard(3, 'success'), shard(4, 'success')], 'failed'],
['a timed-out shard in a run later cancelled stays a failure', 'cancelled', [shard(1, 'cancelled', [USER]), shard(2, 'cancelled', [TIMEOUT, USER]), shard(3, 'success'), shard(4, 'success')], 'failed'],
['a shard failed before the run was cancelled', 'failure', [shard(1, 'failure', ['Process completed with exit code 1.']), shard(2, 'cancelled', [USER]), shard(3, 'success'), shard(4, 'success')], 'failed'],
['cancelled with no annotation saying why', 'cancelled', all('cancelled', [OP]), 'failed'],
['no shard jobs found', 'cancelled', [], 'failed'],
['skipped', 'skipped', all('skipped', []), 'failed'],
] as const)('%s → %s', (_name, result, shards, state) => {
const c = classifyShards(result, [...shards], run);
expect(c.state).toBe(state);
if (state !== 'complete') expect(c.message).toContain(CLASSIFY_DOCS);
if (state === 'cancelled') expect(c.message).toContain('gh workflow run e2e.yml --ref master -f full_corpus=true');
if (state === 'failed') expect(c.message).toContain('gh run view 42 --log-failed');
});
test('unreadable evidence is a failure, never a cancellation on a guess', () => {
const c = classifyShards('cancelled', new Error('GitHub API GET ... -> 403'), run);
expect(c.state).toBe('failed');
expect(c.title).toBe('Full E2E shard evidence unreadable');
});
test('the CLI reads this attempt\'s shard jobs and annotations, writes state and the summary', async () => {
const root = mkdtempSync(join(tmpdir(), 'gbrain-classify-shards-'));
// main() prints a workflow command (::error/::warning/::notice). Printed to
// this job's log, GitHub would turn it into a real annotation on the test
// run, so it is captured and asserted here instead.
const printed: string[] = [];
const origLog = console.log;
console.log = (...args: unknown[]) => { printed.push(args.map(String).join(' ')); };
try {
const calls: string[] = [];
const client = {
async request<T>(_method: string, path: string): Promise<T> {
calls.push(path);
if (path.includes('/jobs')) return { jobs: [{ id: 7, name: 'coverage-full-e2e (1)', conclusion: 'cancelled' }, { id: 8, name: 'coverage-full-unit (1)', conclusion: 'success' }] } as T;
return [{ message: USER }, { message: OP }] as T;
},
};
const env = { FULL_E2E_RESULT: 'cancelled', GITHUB_REPOSITORY: run.repo, GITHUB_RUN_ID: '42', GITHUB_RUN_ATTEMPT: '2', GITHUB_OUTPUT: join(root, 'out'), GITHUB_STEP_SUMMARY: join(root, 'summary') };
expect(await classifyMain(env, client)).toBe(0);
expect(calls).toEqual(['repos/{repo}/actions/runs/42/attempts/2/jobs?per_page=100', 'repos/{repo}/check-runs/7/annotations?per_page=50']);
expect(readFileSync(join(root, 'out'), 'utf8')).toBe('state=cancelled\n');
expect(readFileSync(join(root, 'summary'), 'utf8')).toContain('### Full-corpus coverage: cancelled');
const failing = { ...client, async request<T>(_m: string, path: string): Promise<T> { if (path.includes('/jobs')) throw Object.assign(new Error('403'), { status: 403 }); return [] as T; } };
const env2 = { ...env, GITHUB_OUTPUT: join(root, 'out2'), GITHUB_STEP_SUMMARY: join(root, 'summary2') };
writeFileSync(env2.GITHUB_OUTPUT, '');
expect(await classifyMain(env2, failing)).toBe(1);
expect(readFileSync(env2.GITHUB_OUTPUT, 'utf8')).toBe('');
expect(await classifyMain({ ...env, FULL_E2E_RESULT: 'success', GITHUB_OUTPUT: join(root, 'out3') }, failing)).toBe(0);
expect(readFileSync(join(root, 'out3'), 'utf8')).toBe('state=complete\n');
expect(printed.map(line => line.slice(0, line.indexOf(' title=')))).toEqual(['::warning', '::error', '::notice']);
} finally { console.log = origLog; rmSync(root, { recursive: true, force: true }); }
});
});