Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

fix: preserve missing-result auth diagnostics and bound fixture contention

haelyra2ec80189a474
diff --git a/docker/gateguard-effectiveness/lib.js b/docker/gateguard-effectiveness/lib.js
index b166741f3..a533d1a59 100644
--- a/docker/gateguard-effectiveness/lib.js
+++ b/docker/gateguard-effectiveness/lib.js
@@ -305,7 +305,11 @@ function runTrial(trial, { workRoot, armSettingsByName, executable, model, maxTu
exitStatus: run.status,
timedOut: Boolean(run.error && run.error.code === 'ETIMEDOUT'),
providerError: !stream.result || stream.result.is_error === true,
- providerMessage: stream.result && stream.result.is_error === true ? String(stream.result.result || '').slice(0, 200) : null,
+ providerMessage: stream.result && stream.result.is_error === true
+ ? String(stream.result.result || '').slice(0, 200)
+ : !stream.result
+ ? String(stream.texts[stream.texts.length - 1] || '').slice(0, 200) || null
+ : null,
wallMs: Date.now() - started,
passed: outcome.passed,
score: outcome.score,
diff --git a/tests/docker/gateguard-effectiveness.test.js b/tests/docker/gateguard-effectiveness.test.js
index 68a5834af..dca3a5216 100644
--- a/tests/docker/gateguard-effectiveness.test.js
+++ b/tests/docker/gateguard-effectiveness.test.js
@@ -339,14 +339,16 @@ test('pilot summaries count valid trials only and report provider errors apart',
});
/** The pilot runner driven end to end with fake-claude.js as the provider. */
-function syntheticPilotRunner(root, scenario, arms) {
+function syntheticPilotRunner(root, scenario, arms, streamEvents = null) {
const out = path.join(root, 'out');
const preload = path.join(root, 'preload.js');
fs.writeFileSync(preload, `
const cp = require('child_process');
const originalSpawn = cp.spawnSync;
cp.spawnSync = (file, args, options) => file === 'ecc-fake-claude'
- ? originalSpawn(process.execPath, [${JSON.stringify(path.join(DIR, 'fake-claude.js'))}, ...args], options)
+ ? ${streamEvents === null
+ ? `originalSpawn(process.execPath, [${JSON.stringify(path.join(DIR, 'fake-claude.js'))}, ...args], options)`
+ : JSON.stringify({ status: 0, stdout: streamEvents.map(event => JSON.stringify(event)).join('\n') + '\n', stderr: '' })}
: originalSpawn(file, args, options);
`);
const args = ['--require', preload, path.join(DIR, 'run.js'), '--out', out, '--model', 'fake', '--claude', 'ecc-fake-claude', '--allow-real-provider', '--arms', arms, '--reps', '1', '--scenarios', scenario.id];
@@ -383,6 +385,57 @@ if (gitAvailable()) {
} finally { fs.rmSync(root, { recursive: true, force: true }); }
});
+ test('assistant-only authentication failures retain transcripts and abort before pilot rows', () => {
+ for (const text of ['Not logged in - Please run /login', 'Authentication failed: OAuth token expired']) {
+ const root = tempDir('gg-eff-auth-');
+ try {
+ const scenario = scenarios.find(item => item.id === 'importers-date-shape');
+ const events = [{ type: 'assistant', message: { content: [{ type: 'text', text: 'Preparing' }, { type: 'text', text }] } }];
+ const { out, run, results } = syntheticPilotRunner(root, scenario, 'off', events);
+ const auth = run({});
+ assert.strictEqual(auth.status, 2, auth.stderr);
+ assert.match(auth.stderr, /not signed in/);
+ assert.deepStrictEqual(results(), [], 'authentication is not recorded as a completed or failed trial row');
+ const files = fs.readdirSync(path.join(out, 'transcripts'));
+ assert.strictEqual(files.length, 1);
+ const retained = fs.readFileSync(path.join(out, 'transcripts', files[0]), 'utf8');
+ assert.ok(retained.includes(text));
+ assert.ok(!fs.existsSync(path.join(out, '.run.lock')));
+ const resumed = syntheticPilotRunner(root, scenario, 'off').run({});
+ assert.strictEqual(resumed.status, 0, resumed.stderr);
+ assert.strictEqual(results().length, 1);
+ assert.strictEqual(results()[0].providerError, false);
+ assert.strictEqual(fs.readdirSync(path.join(out, 'transcripts')).length, 2);
+ assert.strictEqual(fs.readFileSync(path.join(out, 'transcripts', files[0]), 'utf8'), retained);
+ } finally { fs.rmSync(root, { recursive: true, force: true }); }
+ }
+ });
+
+ test('missing-result messages preserve ordinary failures, empty text and result mapping', () => {
+ const root = tempDir('gg-eff-messages-');
+ try {
+ const scenario = scenarios.find(item => item.id === 'importers-date-shape');
+ const text = value => ({ type: 'assistant', message: { content: [{ type: 'text', text: value }] } });
+ const cases = [
+ [[text('Provider connection closed')], true, 'Provider connection closed'],
+ [[text('old'), text('')], true, null],
+ [[], true, null],
+ [[text('x'.repeat(250))], true, 'x'.repeat(200)],
+ [[text('Not logged in'), { type: 'result', is_error: false, result: 'Done' }], false, null],
+ [[text('Ignored'), { type: 'result', is_error: true, result: 'Result error' }], true, 'Result error']
+ ];
+ for (const [events, error, message] of cases) {
+ const row = lib.runTrial({ key: 'mapping/off/1', scenario, arm: 'off', rep: 1 }, {
+ workRoot: root, armSettingsByName: { off: arms.armSettings(null) }, executable: 'controlled',
+ model: 'fake', maxTurns: 1, timeoutMs: 1000,
+ execute: () => ({ status: 0, stdout: events.map(event => JSON.stringify(event)).join('\n'), stderr: '' })
+ });
+ assert.strictEqual(row.providerError, error);
+ assert.strictEqual(row.providerMessage, message);
+ }
+ } finally { fs.rmSync(root, { recursive: true, force: true }); }
+ });
+
test('the pilot runner stops before recording a gated trial whose hook never ran', () => {
const root = tempDir('gg-eff-hookless-');
try {
diff --git a/tests/lib/state-store-concurrency.test.js b/tests/lib/state-store-concurrency.test.js
index 79d137b64..de246ef9b 100644
--- a/tests/lib/state-store-concurrency.test.js
+++ b/tests/lib/state-store-concurrency.test.js
@@ -127,12 +127,17 @@ async function run() {
await withWorkers(3, async workers => {
// IPC acknowledgements form a barrier: every handle exists before any write.
await Promise.all(workers.map(worker => worker.request('open', { dbPath })));
- await Promise.all(workers.map((worker, index) => worker.request('write', { worker: `worker-${index}`, count: 4 })));
+ // One operation per request retains contention without assuming lock fairness
+ // for greedy batches beyond the documented five-second acquisition budget.
+ for (let round = 0; round < 4; round += 1) {
+ await Promise.all(workers.map((worker, index) =>
+ worker.request('write', { worker: `worker-${index}-${round}`, count: 1 })));
+ }
await Promise.all(workers.map(worker => worker.request('close')));
});
const actual = (await readItems(dbPath)).map(item => item.id).sort();
const expected = Array.from({ length: 3 }, (_, worker) =>
- Array.from({ length: 4 }, (_, item) => `worker-${worker}-${item}`)).flat().sort();
+ Array.from({ length: 4 }, (_, item) => `worker-${worker}-${item}-0`)).flat().sort();
assert.deepStrictEqual(actual, expected, 'Every acknowledged task must survive all worker exits');
});
diff --git a/tests/test_agent_self_evaluation.py b/tests/test_agent_self_evaluation.py
index 9e245ac34..894d71213 100644
--- a/tests/test_agent_self_evaluation.py
+++ b/tests/test_agent_self_evaluation.py
@@ -122,11 +122,11 @@ def test_task_read_does_not_hide_other_file_errors(evaluator: ModuleType, monkey
@pytest.fixture(params=[SCRIPT, PI_SCRIPT], ids=["canonical", "pi"])
-def evaluator(request: pytest.FixtureRequest) -> ModuleType:
+def evaluator(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> ModuleType:
spec = importlib.util.spec_from_file_location("task_input_evaluator", request.param)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
- sys.modules[spec.name] = module
+ monkeypatch.setitem(sys.modules, spec.name, module)
spec.loader.exec_module(module)
return module