The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
/**
* Tests for scripts/lib/eval-harness/replay.js and effect-fence.js
* Run with: node tests/lib/eval-harness/replay.test.js
*/
'use strict';
const assert = require('assert');
const fs = require('fs');
const path = require('path');
const { spawnSync } = require('child_process');
const replay = require('../../../scripts/lib/eval-harness/replay');
const { test, tempDir, cleanup, finish } = require('./helpers');
console.log('\n=== eval-harness replay ===\n');
const tools = {
read_inventory: { effect_class: 'SE0', determinism: 'deterministic', impl: (args) => ({ sku: args.sku, count: 7 }) },
write_note: { effect_class: 'SE1', determinism: 'deterministic', impl: () => ({ ok: true }) },
publish: { effect_class: 'SE3', determinism: 'nondeterministic', impl: () => ({ ok: true }) },
charge_card: { effect_class: 'SE4', determinism: 'nondeterministic', impl: () => { throw new Error('never'); } },
};
test('record mode stores content-addressed fixtures with arg and response hashes', () => {
const dir = tempDir('record');
try {
const store = new replay.FixtureStore(dir);
const recorder = replay.createReplayer(tools, { mode: 'record', store });
const response = recorder.call('read_inventory', { sku: 'x' });
assert.strictEqual(response.count, 7);
assert.ok(store.has('read_inventory', { sku: 'x' }));
const record = store.get('read_inventory', { sku: 'x' });
assert.strictEqual(record.tool, 'read_inventory');
assert.strictEqual(recorder.calls[0].status, 'recorded');
} finally {
cleanup(dir);
}
});
test('replay mode never calls the implementation and fails closed on a missing fixture', () => {
const dir = tempDir('replay');
try {
const store = new replay.FixtureStore(dir);
let liveCalls = 0;
const spyTools = { ...tools, read_inventory: { ...tools.read_inventory, impl: () => { liveCalls += 1; return { count: 7 }; } } };
replay.createReplayer(spyTools, { mode: 'record', store }).call('read_inventory', { sku: 'x' });
assert.strictEqual(liveCalls, 1);
const replayer = replay.createReplayer(spyTools, { mode: 'replay', store });
assert.strictEqual(replayer.call('read_inventory', { sku: 'x' }).count, 7);
assert.throws(() => replayer.call('read_inventory', { sku: 'missing' }), (error) => error.code === 'tool.fixture_missing');
assert.strictEqual(liveCalls, 1);
} finally {
cleanup(dir);
}
});
test('a hash-mismatched or corrupt fixture fails closed', () => {
const dir = tempDir('mismatch');
try {
const store = new replay.FixtureStore(dir);
const record = store.put('read_inventory', { sku: 'x' }, { count: 1 });
const filePath = store.pathFor(record.key);
const tampered = JSON.parse(fs.readFileSync(filePath, 'utf8'));
tampered.response.count = 999;
fs.writeFileSync(filePath, JSON.stringify(tampered));
assert.throws(() => store.get('read_inventory', { sku: 'x' }), (error) => error.code === 'tool.fixture_mismatch');
fs.writeFileSync(filePath, '{not json');
assert.throws(() => store.get('read_inventory', { sku: 'x' }), (error) => error.code === 'tool.fixture_corrupt');
} finally {
cleanup(dir);
}
});
test('SE3 and above are refused in replay, and anything above maxEffectClass is refused in record', () => {
const dir = tempDir('effects');
try {
const store = new replay.FixtureStore(dir);
const replayer = replay.createReplayer(tools, { mode: 'replay', store, maxEffectClass: 'SE4' });
assert.throws(() => replayer.call('publish', {}), (error) => error.code === 'tool.effect_forbidden');
assert.throws(() => replayer.call('charge_card', {}), (error) => error.code === 'tool.effect_forbidden');
const recorder = replay.createReplayer(tools, { mode: 'record', store, maxEffectClass: 'SE0' });
assert.throws(() => recorder.call('write_note', {}), (error) => error.code === 'tool.effect_forbidden');
assert.throws(() => recorder.call('nope', {}), (error) => error.code === 'tool.unknown');
} finally {
cleanup(dir);
}
});
test('tools must declare effect_class and determinism', () => {
const dir = tempDir('declare');
try {
const store = new replay.FixtureStore(dir);
assert.throws(() => replay.createReplayer({ bad: { impl: () => 1 } }, { mode: 'replay', store }), /effect_class/);
assert.throws(() => replay.createReplayer({ bad: { effect_class: 'SE0', impl: () => 1 } }, { mode: 'replay', store }), /determinism/);
} finally {
cleanup(dir);
}
});
test('retired effect preload refuses before any supplied code runs', () => {
const dir = tempDir('fence');
try {
const canary = path.join(dir, 'executed');
const result = spawnSync(process.execPath, ['--require', replay.EFFECT_FENCE_PRELOAD, '-e',
`require('fs').writeFileSync(${JSON.stringify(canary)}, 'executed');`], {
cwd: dir, encoding: 'utf8', timeout: 2000,
env: { ECC_EFFECT_FENCE_ROOT: dir },
});
assert.notStrictEqual(result.status, 0);
assert.match(result.stderr, /gate.isolation_required/);
assert.ok(!fs.existsSync(canary));
} finally {
cleanup(dir);
}
});
test('own-key arguments cannot alias another replay fixture or fall back to a legacy key', () => {
const dir = tempDir('own-key');
try {
const store = new replay.FixtureStore(dir);
const args = JSON.parse('{"__proto__":{"marker":"fixture"}}');
const legacy = store.put('read_inventory', {}, { count: 7 });
const before = fs.readFileSync(store.pathFor(legacy.key));
assert.notStrictEqual(store.key('read_inventory', args), legacy.key);
let calls = 0;
const replayer = replay.createReplayer({ read_inventory: { effect_class: 'SE0', determinism: 'deterministic', impl() { calls += 1; throw new Error('must not call'); } } }, { mode: 'replay', store });
assert.throws(() => replayer.call('read_inventory', args), error => error.code === 'tool.fixture_missing');
assert.strictEqual(calls, 0);
assert.deepStrictEqual(fs.readFileSync(store.pathFor(legacy.key)), before);
assert.deepStrictEqual(fs.readdirSync(dir), [legacy.key + '.json']);
store.put('read_inventory', args, { count: 9 });
assert.strictEqual(replayer.call('read_inventory', args).count, 9);
assert.strictEqual(replayer.call('read_inventory', {}).count, 7);
assert.strictEqual(calls, 0);
} finally { cleanup(dir); }
});
test('nested own-key response survives persistence and tampering fails closed', () => {
const dir = tempDir('own-response');
try {
const store = new replay.FixtureStore(dir);
const response = JSON.parse('{"items":[{"__proto__":{"count":7}}]}');
const record = store.put('read_inventory', {}, response);
assert.deepStrictEqual(store.get('read_inventory', {}).response, response);
const file = store.pathFor(record.key);
const tampered = JSON.parse(fs.readFileSync(file, 'utf8'));
tampered.response.items[0].__proto__.count = 9;
fs.writeFileSync(file, JSON.stringify(tampered));
const before = fs.readFileSync(file);
assert.throws(() => store.get('read_inventory', {}), error => error.code === 'tool.fixture_mismatch');
assert.deepStrictEqual(fs.readFileSync(file), before);
} finally { cleanup(dir); }
});
test('ordinary fixture bytes and key remain identical to the pinned base', () => {
const dir = tempDir('base-fixture');
try {
const store = new replay.FixtureStore(dir);
const record = store.put('read_inventory', { sku: 'x' }, { count: 7 });
assert.strictEqual(record.key, "38531922cfce2687a257eebffffa3fa9ef03b132f862fff9e371cab0bf2391ef");
assert.strictEqual(fs.readFileSync(store.pathFor(record.key), 'utf8'), "{\"args_hash\":\"90765859d73de6e117260f0b4cefcb88f09d8e79e71ca576868a28d662f98851\",\"key\":\"38531922cfce2687a257eebffffa3fa9ef03b132f862fff9e371cab0bf2391ef\",\"response\":{\"count\":7},\"response_hash\":\"b0beaf5a3dbe82ae841ac88bdc3b1174d7e4dec57454b6539e556e58eaadc600\",\"tool\":\"read_inventory\"}\n");
} finally { cleanup(dir); }
});
finish('replay');