The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix: preserve missing-result auth diagnostics and bound fixture contention
haelyra2ec80189a474
diff --git a/docker/gateguard-effectiveness/lib.js b/docker/gateguard-effectiveness/lib.jsindex b166741f3..a533d1a59 100644--- a/docker/gateguard-effectiveness/lib.js+++ b/docker/gateguard-effectiveness/lib.js@@ -305,7 +305,11 @@ function runTrial(trial, { workRoot, armSettingsByName, executable, model, maxTuexitStatus: run.status,timedOut: Boolean(run.error && run.error.code === 'ETIMEDOUT'),providerError: !stream.result || stream.result.is_error === true,- providerMessage: stream.result && stream.result.is_error === true ? String(stream.result.result || '').slice(0, 200) : null,+ providerMessage: stream.result && stream.result.is_error === true+ ? String(stream.result.result || '').slice(0, 200)+ : !stream.result+ ? String(stream.texts[stream.texts.length - 1] || '').slice(0, 200) || null+ : null,wallMs: Date.now() - started,passed: outcome.passed,score: outcome.score,diff --git a/tests/docker/gateguard-effectiveness.test.js b/tests/docker/gateguard-effectiveness.test.jsindex 68a5834af..dca3a5216 100644--- a/tests/docker/gateguard-effectiveness.test.js+++ b/tests/docker/gateguard-effectiveness.test.js@@ -339,14 +339,16 @@ test('pilot summaries count valid trials only and report provider errors apart',});/** The pilot runner driven end to end with fake-claude.js as the provider. */-function syntheticPilotRunner(root, scenario, arms) {+function syntheticPilotRunner(root, scenario, arms, streamEvents = null) {const out = path.join(root, 'out');const preload = path.join(root, 'preload.js');fs.writeFileSync(preload, `const cp = require('child_process');const originalSpawn = cp.spawnSync;cp.spawnSync = (file, args, options) => file === 'ecc-fake-claude'- ? originalSpawn(process.execPath, [${JSON.stringify(path.join(DIR, 'fake-claude.js'))}, ...args], options)+ ? ${streamEvents === null+ ? `originalSpawn(process.execPath, [${JSON.stringify(path.join(DIR, 'fake-claude.js'))}, ...args], options)`+ : JSON.stringify({ status: 0, stdout: streamEvents.map(event => JSON.stringify(event)).join('\n') + '\n', stderr: '' })}: originalSpawn(file, args, options);`);const args = ['--require', preload, path.join(DIR, 'run.js'), '--out', out, '--model', 'fake', '--claude', 'ecc-fake-claude', '--allow-real-provider', '--arms', arms, '--reps', '1', '--scenarios', scenario.id];@@ -383,6 +385,57 @@ if (gitAvailable()) {} finally { fs.rmSync(root, { recursive: true, force: true }); }});+ test('assistant-only authentication failures retain transcripts and abort before pilot rows', () => {+ for (const text of ['Not logged in - Please run /login', 'Authentication failed: OAuth token expired']) {+ const root = tempDir('gg-eff-auth-');+ try {+ const scenario = scenarios.find(item => item.id === 'importers-date-shape');+ const events = [{ type: 'assistant', message: { content: [{ type: 'text', text: 'Preparing' }, { type: 'text', text }] } }];+ const { out, run, results } = syntheticPilotRunner(root, scenario, 'off', events);+ const auth = run({});+ assert.strictEqual(auth.status, 2, auth.stderr);+ assert.match(auth.stderr, /not signed in/);+ assert.deepStrictEqual(results(), [], 'authentication is not recorded as a completed or failed trial row');+ const files = fs.readdirSync(path.join(out, 'transcripts'));+ assert.strictEqual(files.length, 1);+ const retained = fs.readFileSync(path.join(out, 'transcripts', files[0]), 'utf8');+ assert.ok(retained.includes(text));+ assert.ok(!fs.existsSync(path.join(out, '.run.lock')));+ const resumed = syntheticPilotRunner(root, scenario, 'off').run({});+ assert.strictEqual(resumed.status, 0, resumed.stderr);+ assert.strictEqual(results().length, 1);+ assert.strictEqual(results()[0].providerError, false);+ assert.strictEqual(fs.readdirSync(path.join(out, 'transcripts')).length, 2);+ assert.strictEqual(fs.readFileSync(path.join(out, 'transcripts', files[0]), 'utf8'), retained);+ } finally { fs.rmSync(root, { recursive: true, force: true }); }+ }+ });++ test('missing-result messages preserve ordinary failures, empty text and result mapping', () => {+ const root = tempDir('gg-eff-messages-');+ try {+ const scenario = scenarios.find(item => item.id === 'importers-date-shape');+ const text = value => ({ type: 'assistant', message: { content: [{ type: 'text', text: value }] } });+ const cases = [+ [[text('Provider connection closed')], true, 'Provider connection closed'],+ [[text('old'), text('')], true, null],+ [[], true, null],+ [[text('x'.repeat(250))], true, 'x'.repeat(200)],+ [[text('Not logged in'), { type: 'result', is_error: false, result: 'Done' }], false, null],+ [[text('Ignored'), { type: 'result', is_error: true, result: 'Result error' }], true, 'Result error']+ ];+ for (const [events, error, message] of cases) {+ const row = lib.runTrial({ key: 'mapping/off/1', scenario, arm: 'off', rep: 1 }, {+ workRoot: root, armSettingsByName: { off: arms.armSettings(null) }, executable: 'controlled',+ model: 'fake', maxTurns: 1, timeoutMs: 1000,+ execute: () => ({ status: 0, stdout: events.map(event => JSON.stringify(event)).join('\n'), stderr: '' })+ });+ assert.strictEqual(row.providerError, error);+ assert.strictEqual(row.providerMessage, message);+ }+ } finally { fs.rmSync(root, { recursive: true, force: true }); }+ });+test('the pilot runner stops before recording a gated trial whose hook never ran', () => {const root = tempDir('gg-eff-hookless-');try {diff --git a/tests/lib/state-store-concurrency.test.js b/tests/lib/state-store-concurrency.test.jsindex 79d137b64..de246ef9b 100644--- a/tests/lib/state-store-concurrency.test.js+++ b/tests/lib/state-store-concurrency.test.js@@ -127,12 +127,17 @@ async function run() {await withWorkers(3, async workers => {// IPC acknowledgements form a barrier: every handle exists before any write.await Promise.all(workers.map(worker => worker.request('open', { dbPath })));- await Promise.all(workers.map((worker, index) => worker.request('write', { worker: `worker-${index}`, count: 4 })));+ // One operation per request retains contention without assuming lock fairness+ // for greedy batches beyond the documented five-second acquisition budget.+ for (let round = 0; round < 4; round += 1) {+ await Promise.all(workers.map((worker, index) =>+ worker.request('write', { worker: `worker-${index}-${round}`, count: 1 })));+ }await Promise.all(workers.map(worker => worker.request('close')));});const actual = (await readItems(dbPath)).map(item => item.id).sort();const expected = Array.from({ length: 3 }, (_, worker) =>- Array.from({ length: 4 }, (_, item) => `worker-${worker}-${item}`)).flat().sort();+ Array.from({ length: 4 }, (_, item) => `worker-${worker}-${item}-0`)).flat().sort();assert.deepStrictEqual(actual, expected, 'Every acknowledged task must survive all worker exits');});diff --git a/tests/test_agent_self_evaluation.py b/tests/test_agent_self_evaluation.pyindex 9e245ac34..894d71213 100644--- a/tests/test_agent_self_evaluation.py+++ b/tests/test_agent_self_evaluation.py@@ -122,11 +122,11 @@ def test_task_read_does_not_hide_other_file_errors(evaluator: ModuleType, monkey@pytest.fixture(params=[SCRIPT, PI_SCRIPT], ids=["canonical", "pi"])-def evaluator(request: pytest.FixtureRequest) -> ModuleType:+def evaluator(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> ModuleType:spec = importlib.util.spec_from_file_location("task_input_evaluator", request.param)assert spec is not None and spec.loader is not Nonemodule = importlib.util.module_from_spec(spec)- sys.modules[spec.name] = module+ monkeypatch.setitem(sys.modules, spec.name, module)spec.loader.exec_module(module)return module