Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
// Copyright 2026 Ix Infrastructure Inc.
import assert from 'node:assert/strict';
import { execFileSync, spawnSync } from 'node:child_process';
import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
import { delimiter, join, normalize, resolve, sep } from 'node:path';
import { tmpdir } from 'node:os';
import { fileURLToPath } from 'node:url';
import { after, test } from 'node:test';
import { checkManifestSource, importVerified, loadManifest, parseExport, sha256, validateManifest } from '../manifest.mjs';
import { schedule, validateConfig, validateProbe } from '../config.mjs';
import { budgetTokens, commandCounts, isIxMcpTool, ixExitScope, parseMetrics, METRICS } from '../metrics.mjs';
import { markdown, summarize } from '../stats.mjs';
import { dryConfig, resolveConfigPaths, runBenchmark } from '../run.mjs';
import { execute } from '../process.mjs';
import { checkResolution, evaluationError } from '../evaluation.mjs';
import { armEnvironment, shellQuote } from '../isolation.mjs';
import { assertMapped, checkLoginShells, checkTaskIsolation, codexSkillListed, integrationFiles } from '../provision.mjs';
import { CLAUDE_IX_MCP_RULE, HOOK_SCRIPT, MCP_SERVER, PLUGIN_NAME, isHookProcess, nativeArgs } from '../integration.mjs';
const root = fileURLToPath(new URL('../', import.meta.url));
// Worktree-local scratch also keeps the CI smoke independent of machine config.
const scratch = mkdtempSync(join(root, 'results-test-'));
// Run work directories default to os.tmpdir(); failed runs keep theirs, so point it at a private, removed root.
const workRoot = mkdtempSync(join(tmpdir(), 'agent-eval-tests-'));
const inheritedTmp = process.env.TMPDIR; process.env.TMPDIR = workRoot;
after(() => {
if (inheritedTmp === undefined) delete process.env.TMPDIR; else process.env.TMPDIR = inheritedTmp;
rmSync(scratch, { recursive: true, force: true }); rmSync(workRoot, { recursive: true, force: true });
});
const smoke = join(root, 'smoke.json');
const rows = Array.from({ length: 60 }, (_, i) => ({ instance_id: `fixture__repo-${i}`, repo: `fixture/repo-${i % 6}`, base_commit: sha256(`fixture-${i}`).slice(0, 40), problem_statement: `Synthetic export fixture ${i}`, patch: 'HIDDEN GOLD', test_patch: 'HIDDEN TESTS' }));
const integrationTest = (name, fn) => test(name, { skip: process.platform === 'win32' || process.env.AGENT_EVAL_PORTABLE_ONLY === '1' }, fn);
const ndjson = (events) => events.map((e) => JSON.stringify(e)).join('\n');
test('manifest importer pins 50 tasks, covers 5 repos, and is stable across input ordering', () => {
const text = JSON.stringify(rows), manifest = importVerified(text);
assert.equal(manifest.tasks.length, 50);
assert.ok(new Set(manifest.tasks.map((t) => t.repo)).size >= 5);
assert.equal(manifest.source_sha256, sha256(text));
assert.deepEqual(manifest.tasks, importVerified(JSON.stringify([...rows].reverse())).tasks);
assert.deepEqual(manifest.tasks, importVerified(rows.map((r) => JSON.stringify(r)).join('\n')).tasks);
assert.ok(!JSON.stringify(manifest).includes('HIDDEN'));
const path = join(scratch, 'manifest.json'); writeFileSync(path, JSON.stringify(manifest));
assert.deepEqual(loadManifest(path), manifest);
assert.equal(importVerified(text, { count: 3, minRepos: 3, seed: 7 }).tasks.length, 3);
});
test('manifest rejects missing fields, traversal, duplicates and insufficient coverage', () => {
for (const field of ['instance_id', 'repo', 'base_commit', 'problem_statement']) {
const row = { ...rows[0] }; delete row[field];
assert.throws(() => importVerified(JSON.stringify([row]), { count: 1, minRepos: 1 }));
}
assert.throws(() => importVerified(JSON.stringify([rows[0], rows[0]]), { count: 1, minRepos: 1 }), /duplicate/);
assert.throws(() => importVerified(JSON.stringify(rows), { count: 61 }), /Insufficient/);
assert.throws(() => importVerified('{}'), /array/);
assert.throws(() => importVerified(JSON.stringify(rows), { count: 1, minRepos: 2 }), /coverage/);
assert.throws(() => validateManifest({ ...loadManifest(smoke), tasks: [{ ...rows[0], repo: '../repo' }] }), /Invalid repo/);
assert.throws(() => validateManifest({ ...loadManifest(smoke), tasks: [{ ...rows[0], instance_id: '..' }] }), /instance_id/);
});
test('shared arm configuration and randomized schedule are reproducible and balanced', () => {
const config = dryConfig(); validateConfig(config, true);
assert.throws(() => validateConfig(config), /family/);
const invalid = structuredClone(config); invalid.budget.seconds = 0;
assert.throws(() => validateConfig(invalid, true), /positive/);
const jobs = schedule(rows.slice(0, 3));
assert.equal(jobs.length, 12);
assert.deepEqual(jobs, schedule(rows.slice(0, 3)));
assert.notDeepEqual(jobs, schedule(rows.slice(0, 3), 2, 9));
for (const task of rows.slice(0, 3)) for (const repeat of [0, 1]) assert.deepEqual(jobs.filter((j) => j.task === task && j.repeat === repeat).map((j) => j.arm).sort(), ['baseline', 'ix']);
const paths = resolveConfigPaths({ agent: { command: ['node', 'bench/agent-eval/agent.mjs'] }, evaluator: { python: 'local/python', harness_path: 'local/harness' } }, scratch);
assert.equal(paths.agent.command[1], resolve(scratch, 'bench/agent-eval/agent.mjs'));
assert.equal(paths.evaluator.python, resolve(scratch, 'local/python'));
});
test('the loopback endpoint check applies to every provision.mjs spelling and to any config that routes the ix launcher', () => {
const example = JSON.parse(readFileSync(join(root, 'config.example.json'), 'utf8'));
example.environment.plugin_revision = 'a'.repeat(40); example.evaluator.revision = 'b'.repeat(40);
validateConfig(example);
const variant = (environment) => { const c = structuredClone(example); Object.assign(c.environment, environment); return c; };
for (const script of ['bench/agent-eval/provision.mjs', 'provision.mjs', './provision.mjs', resolve(root, 'provision.mjs')]) {
assert.throws(() => validateConfig(variant({ endpoint: 'https://ix.example.com', setup: ['node', script, 'setup'] })), /loopback/, script);
}
assert.throws(() => validateConfig(variant({ endpoint: undefined })), /loopback/);
// A custom setup still gets the launcher (and IX_ENDPOINT) from ix_cli.
const custom = { setup: ['./my-setup.sh'], source_path: undefined, ix_version: undefined };
assert.throws(() => validateConfig(variant({ ...custom, endpoint: 'https://ix.example.com' })), /loopback/);
assert.throws(() => validateConfig(variant({ ...custom, endpoint: undefined })), /loopback/);
validateConfig(variant({ ...custom, ix_cli: undefined, endpoint: undefined }));
});
test('probe rejects unmapped ix, stale maps, plugin drift and baseline contamination', () => {
const request = { arm: 'ix', repo_path: '/fixture/repo', task: rows[0], agent: { version: 'v1' }, environment: { plugin_revision: 'pin' } };
const probe = { version: 1, repo_path: request.repo_path, base_commit: rows[0].base_commit, agent_version: 'v1', ix_available: true, mapped: true, plugin_installed: true, skills_installed: true, hooks_enabled: true, mapped_commit: rows[0].base_commit, plugin_revision: 'pin', evidence: [{ path: 'plugin', sha256: 'hash' }] };
validateProbe(probe, request);
for (const key of ['ix_available', 'mapped', 'plugin_installed', 'skills_installed', 'hooks_enabled']) assert.throws(() => validateProbe({ ...probe, [key]: false }, request), /environment/);
assert.throws(() => validateProbe({ ...probe, mapped_commit: 'stale' }, request), /Stale/);
assert.throws(() => validateProbe({ ...probe, plugin_revision: 'wrong' }, request), /revision/);
assert.throws(() => validateProbe(probe, { ...request, arm: 'baseline' }), /environment/);
});
test('normalized telemetry deduplicates tool ids and counts direct shell searches and ix errors', () => {
assert.deepEqual(commandCounts('echo rg; rg needle | grep x && /bin/find .; env A=b ix locate Foo'), { search_calls: 3, rg_calls: 1, grep_calls: 1, find_calls: 1, ix_calls: 1 });
const metrics = parseMetrics(ndjson([{ type: 'turn', id: 'a' }, { type: 'tool', id: 't', command: 'rg foo; ix locate Foo' }, { type: 'tool', id: 't', command: 'rg foo' }, { type: 'ix_error', code: 'STALE_MAP' }, { type: 'usage', input_tokens: 100, output_tokens: 10, cached_tokens: 20 }, { type: 'done' }]), 'normalized');
assert.equal(metrics.tool_calls, 1); assert.equal(metrics.search_calls, 1); assert.equal(metrics.ix_calls, 1);
assert.deepEqual(metrics.ix_errors_by_code, { STALE_MAP: 1 });
assert.throws(() => parseMetrics('{bad}', 'normalized'));
assert.throws(() => parseMetrics(ndjson([{ type: 'done' }]), 'normalized'), /Incomplete/);
assert.throws(() => parseMetrics(ndjson([{ type: 'turn', id: 'a' }, { type: 'usage', input_tokens: 10, output_tokens: 1 }, { type: 'done' }]), 'normalized'), /cached_tokens/);
});
test('Claude parses terminal usage including cache creation, deduplicated message ids and ix tool failures', () => {
const assistant = { type: 'assistant', message: { id: 'a', content: [{ type: 'tool_use', id: 't', name: 'Bash', input: { command: 'ix locate Foo' } }, { type: 'tool_use', id: 'm', name: 'mcp__ix-memory__locate', input: {} }] } };
const metrics = parseMetrics(ndjson([assistant, assistant, { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 't', is_error: true, content: '{"error":{"code":"UNMAPPED"}}' }] } }, { type: 'result', usage: { input_tokens: 10, output_tokens: 20, cache_read_input_tokens: 30, cache_creation_input_tokens: 40 } }]), 'claude');
assert.equal(metrics.input_tokens, 80); assert.equal(metrics.cached_tokens, 30);
assert.equal(metrics.turns, 1); assert.equal(metrics.tool_calls, 2); assert.equal(metrics.ix_calls, 2);
assert.deepEqual(metrics.ix_errors_by_code, { UNMAPPED: 1 });
});
test('only a command line that runs hook.mjs counts as an Ix hook process', () => {
for (const args of [`${process.execPath} ${HOOK_SCRIPT} claude`, `node ${HOOK_SCRIPT} codex {"type":"agent-turn-complete"}`, `node ${HOOK_SCRIPT}`]) assert.ok(isHookProcess(args), args);
for (const args of ['node build.mjs', `node ${HOOK_SCRIPT}x claude`, `node /elsewhere/hook.mjs claude`, `node ${HOOK_SCRIPT.slice(0, -1)}`]) assert.ok(!isHookProcess(args), args);
});
test('Claude turns count only the main agent: subagent (sidechain) messages do not reach max_turns', () => {
const usage = { type: 'result', usage: { input_tokens: 1, output_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 } };
const task = { type: 'tool_use', id: 'task', name: 'Task', input: { prompt: 'explore' } };
const sub = (id, extra = {}) => ({ type: 'assistant', parent_tool_use_id: 'task', ...extra, message: { id, content: [{ type: 'tool_use', id: `g-${id}`, name: 'Grep', input: {} }] } });
const events = [{ type: 'assistant', parent_tool_use_id: null, message: { id: 'main-1', content: [task] } },
sub('sub-1'), sub('sub-2'), sub('sub-3', { isSidechain: true }), { type: 'assistant', isSidechain: true, message: { id: 'side-4', content: [] } },
{ type: 'user', parent_tool_use_id: null, message: { content: [{ type: 'tool_result', tool_use_id: 'task', content: 'found' }] } },
{ type: 'assistant', message: { id: 'main-2', content: [{ type: 'text', text: 'done' }] } }, usage];
const metrics = parseMetrics(ndjson(events), 'claude');
assert.equal(metrics.turns, 2, 'only the two main-agent messages are turns');
// The subagent's work is still the agent's work: its tool calls and searches count.
assert.equal(metrics.tool_calls, 4); assert.equal(metrics.native_search_calls, 3);
});
test('Claude Ix MCP calls count under both direct and plugin-scoped tool names', () => {
for (const name of ['mcp__ix-memory__locate', 'mcp__ix__search', 'mcp__plugin_ix-agent-eval_ix-memory__locate', 'mcp__plugin_ix_ix__explain']) assert.equal(isIxMcpTool(name), true, name);
for (const name of ['Bash', 'mcp__github__search', 'mcp__plugin_ix-agent-eval_github__search', 'mcp__plugin_tools_ixl__x', 'mcp__ixl__x']) assert.equal(isIxMcpTool(name), false, name);
const tool = { type: 'tool_use', id: 'p', name: 'mcp__plugin_ix-agent-eval_ix-memory__locate', input: {} };
const metrics = parseMetrics(ndjson([{ type: 'assistant', message: { id: 'a', content: [tool] } }, { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'p', is_error: true, content: [{ type: 'text', text: '{"error":{"code":"UNMAPPED"}}' }] }] } }, { type: 'result', usage: { input_tokens: 1, output_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 } }]), 'claude');
assert.equal(metrics.ix_calls, 1); assert.deepEqual(metrics.ix_errors_by_code, { UNMAPPED: 1 });
});
test('token ceiling counts uncached input and output, not cache reads', () => {
// A long Claude session re-reads its cached context every turn; that must not exhaust the ceiling.
assert.equal(budgetTokens({ input_tokens: 250_000, cached_tokens: 240_000, output_tokens: 5_000 }), 15_000);
assert.equal(budgetTokens({ input_tokens: 10, cached_tokens: 0, output_tokens: 2 }), 12);
});
test('example config grants edits and shell; only the ix arm is allowed the Ix MCP server', () => {
const { agent, budget } = JSON.parse(readFileSync(join(root, 'config.example.json'), 'utf8'));
// Claude --print denies anything that would prompt, so both arms need explicit tool permissions.
const argv = (arm) => nativeArgs({ arm, run_path: '/run', agent, budget });
const allowed = (args) => args.filter((arg, i) => args[i - 1] === '--allowedTools');
assert.deepEqual(allowed(argv('ix')), [`Bash,${CLAUDE_IX_MCP_RULE}`]);
assert.deepEqual(allowed(argv('baseline')), ['Bash']);
assert.ok(argv('ix').includes('acceptEdits'));
// The rule must match the provisioned plugin/server names and the telemetry matcher.
assert.equal(CLAUDE_IX_MCP_RULE, `mcp__plugin_${PLUGIN_NAME}_${MCP_SERVER}`);
assert.ok(isIxMcpTool(`${CLAUDE_IX_MCP_RULE}__locate`));
// Keys are platform paths (backslashes on Windows), so look them up with path.join, not POSIX literals.
const runPath = resolve('/run'), files = integrationFiles({ agent, run_path: runPath, home: resolve('/home'), environment: { seconds: 1 } });
const file = (...parts) => { const text = files[join(runPath, ...parts)]; assert.equal(typeof text, 'string', `missing ${join(...parts)}`); return JSON.parse(text); };
assert.deepEqual(Object.keys(file('plugin', '.mcp.json').mcpServers), [MCP_SERVER]);
assert.equal(file('plugin', '.claude-plugin', 'plugin.json').name, PLUGIN_NAME);
assert.ok(Object.keys(files).every((key) => key === normalize(key) && key.startsWith(join(runPath, 'plugin'))));
const ixArgs = argv('ix');
assert.equal(normalize(ixArgs[ixArgs.indexOf('--plugin-dir') + 1]), join('/run', 'plugin'));
assert.throws(() => integrationFiles({ agent: { family: 'codex' }, run_path: '/run', home: '/home', environment: { seconds: 1 } }), /endpoint/);
// Shared args and tool rules cannot grant or override integrations.
for (const args of [['--allowedTools', 'Bash'], ['--allowed-tools=Bash'], ['--mcp-config', '{}']]) assert.throws(() => nativeArgs({ arm: 'ix', run_path: '/run', agent: { ...agent, args }, budget }), /override/);
assert.throws(() => nativeArgs({ arm: 'baseline', run_path: '/run', agent: { ...agent, allowed_tools: [CLAUDE_IX_MCP_RULE] }, budget }), /MCP/);
assert.throws(() => nativeArgs({ arm: 'ix', run_path: '/run', agent: { ...agent, family: 'codex', allowed_tools: ['Bash'] }, budget }), /Claude Code only/);
assert.equal(nativeArgs({ arm: 'ix', run_path: '/run', agent: { ...agent, family: 'codex', allowed_tools: [] }, budget }).includes('--allowedTools'), false);
});
test('Codex skill discovery reads the model-visible skill list from codex debug prompt-input', () => {
// Shape recorded from codex-cli 0.162.1 `debug prompt-input` with a skill in $CODEX_HOME/skills (no model call).
const prompt = (skills) => JSON.stringify([{ type: 'message', role: 'developer', content: [{ type: 'input_text', text: `<skills_instructions>\n## Skills\n${skills}\n</skills_instructions>` }] }, { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'Mention - ix: in prose' }] }]);
const installer = '- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list. (file: r1/skill-installer/SKILL.md)';
assert.equal(codexSkillListed(prompt(`${installer}\n- ix: Answer structural questions about a codebase (file: r0/ix/SKILL.md)`), 'ix'), true);
assert.equal(codexSkillListed(prompt(`${installer}\n- ixproj: Fixture (file: r2/ixproj/SKILL.md)`), 'ix'), false);
assert.equal(codexSkillListed(prompt(installer), 'ix'), false);
assert.throws(() => codexSkillListed('not json', 'ix'));
});
test('Claude Ix permission denials are reported separately from Ix errors', () => {
const tool = { type: 'tool_use', id: 'p', name: `${CLAUDE_IX_MCP_RULE}__locate`, input: {} };
const denied = { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'p', is_error: true, content: `Claude requested permissions to use ${CLAUDE_IX_MCP_RULE}__locate, but you haven't granted it yet.` }] } };
const result = (extra) => ({ type: 'result', usage: { input_tokens: 1, output_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }, ...extra });
const assistant = { type: 'assistant', message: { id: 'a', content: [tool] } };
let metrics = parseMetrics(ndjson([assistant, denied, result({ permission_denials: [{ tool_name: tool.name, tool_use_id: 'p', tool_input: {} }] })]), 'claude');
assert.equal(metrics.ix_permission_denials, 1); assert.equal(metrics.ix_errors, 0);
metrics = parseMetrics(ndjson([assistant, result({ permission_denials: [{ tool_name: 'Bash', tool_use_id: 'b', tool_input: { command: 'ix locate Foo' } }, { tool_name: 'WebFetch', tool_use_id: 'w', tool_input: {} }] })]), 'claude');
assert.equal(metrics.ix_permission_denials, 1);
metrics = parseMetrics(ndjson([assistant, result({ permission_denials: [] })]), 'claude');
assert.equal(metrics.ix_permission_denials, 0);
});
test('Codex sums completed turn usage and captures command/MCP errors by code', () => {
// One `codex exec` prompt is normally one turn; `turns` counts exec turns, not model requests.
const single = parseMetrics(ndjson([{ type: 'thread.started' }, { type: 'turn.started' }, { type: 'item.completed', item: { id: 'a', type: 'command_execution', command: 'rg foo', exit_code: 0 } }, { type: 'item.completed', item: { id: 'b', type: 'file_change' } }, { type: 'turn.completed', usage: { input_tokens: 10, output_tokens: 2, cached_input_tokens: 4 } }]), 'codex');
assert.equal(single.turns, 1); assert.equal(single.tool_calls, 2); assert.equal(single.input_tokens, 10);
const usage = { type: 'turn.completed', usage: { input_tokens: 100, output_tokens: 20, cached_input_tokens: 50 } };
const metrics = parseMetrics(ndjson([{ type: 'turn.started' }, { type: 'item.completed', item: { id: 't', type: 'command_execution', command: 'rg foo; ix locate Foo', exit_code: 1, aggregated_output: '{"error":{"code":"NOT_FOUND"}}' } }, usage, { type: 'turn.started' }, { type: 'item.completed', item: { id: 'm', type: 'mcp_tool_call', server: 'ix-memory', error: { code: 'DOWN' } } }, usage]), 'codex');
assert.equal(metrics.input_tokens, 200); assert.equal(metrics.cached_tokens, 100); assert.equal(metrics.turns, 2);
assert.equal(metrics.search_calls, 1); assert.equal(metrics.ix_calls, 2);
assert.deepEqual(metrics.ix_errors_by_code, { NOT_FOUND: 1, DOWN: 1 });
assert.throws(() => parseMetrics(ndjson([{ type: 'turn.started' }, { type: 'turn.failed' }]), 'codex'), /failed/);
assert.throws(() => parseMetrics(ndjson([{ type: 'turn.started' }, usage, { type: 'turn.started' }]), 'codex'), /Incomplete/);
});
test('Ix failures hidden behind a zero exit status still count as failures, not successes', () => {
const usage = { input_tokens: 1, output_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 };
const claude = (command, content) => parseMetrics(ndjson([{ type: 'assistant', message: { id: 'a', content: [{ type: 'tool_use', id: 't', name: 'Bash', input: { command } }] } },
{ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 't', is_error: false, content }] } }, { type: 'result', usage }]), 'claude');
for (const text of ['sh: 1: ix: not found', 'bash: line 1: ix: command not found', 'zsh: command not found: ix', "env: 'ix': No such file or directory", 'bash: /usr/local/bin/ix: No such file or directory']) {
const metrics = claude('ix search foo 2>&1 | head -5', text);
assert.equal(metrics.ix_succeeded, 0, text); assert.deepEqual(metrics.ix_errors_by_code, { COMMAND_NOT_FOUND: 1 }, text);
}
// Ix arm: `--format llm` errors piped through head are still Ix errors; array content is read in full.
const piped = claude('ix search foo --format llm 2>&1 | head', [{ type: 'text', text: 'notice' }, { type: 'text', text: 'error code=unmapped message="run ix map"' }]);
assert.equal(piped.ix_succeeded, 0); assert.deepEqual(piped.ix_errors_by_code, { unmapped: 1 });
const ok = claude('ix search foo | head', 'Foo src/foo.ts:1');
// A pipeline's status is head's, so it cannot prove ix ran: not an error, not a confirmed success either.
assert.equal(ok.ix_succeeded, 0); assert.equal(ok.ix_errors, 0); assert.equal(ok.ix_calls, 1);
const codex = (command, aggregated_output) => parseMetrics(ndjson([{ type: 'turn.started' }, { type: 'item.completed', item: { id: 'c', type: 'command_execution', command, exit_code: 0, aggregated_output } },
{ type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 1, cached_input_tokens: 0 } }]), 'codex');
const fallback = codex('bash -lc "ix search foo || rg foo"', 'bash: line 1: ix: command not found\nsrc/foo.ts:1:foo');
assert.equal(fallback.ix_succeeded, 0); assert.deepEqual(fallback.ix_errors_by_code, { COMMAND_NOT_FOUND: 1 });
assert.equal(codex('bash -lc "ix search foo || rg foo"', 'src/foo.ts:1:foo').ix_succeeded, 0);
assert.equal(codex('bash -lc "cd src && ix search foo"', 'Foo src/foo.ts:1').ix_succeeded, 1);
});
test('only an ix call that certainly ran counts as a success; only a lone ix call owns its exit status', () => {
const usage = { input_tokens: 1, output_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 };
const claude = (command, is_error, content) => parseMetrics(ndjson([{ type: 'assistant', message: { id: 'a', content: [{ type: 'tool_use', id: 't', name: 'Bash', input: { command } }] } },
{ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 't', is_error, content }] } }, { type: 'result', usage }]), 'claude');
// Baseline probes and fallbacks that exit 0 whether or not ix exists are not isolation leaks.
for (const command of ['ix --version 2>/dev/null || echo "no ix"', 'command -v ix || ix search foo', 'rg foo src || ix text foo', 'ls; ix status 2>/dev/null; echo done', 'ix status &', 'ix search foo | head']) {
const metrics = claude(command, false, 'no ix');
assert.equal(metrics.ix_succeeded, 0, command); assert.equal(metrics.ix_errors, 0, command);
}
// A lone ix call, or an && chain containing one, that exits 0 proves ix ran.
for (const command of ['ix search foo', 'ix search foo 2>&1', 'cd src &&\n ix search foo', 'env A=b ix map && ix search foo > out.txt', '(ix search foo)']) {
assert.equal(claude(command, false, 'Foo src/foo.ts:1').ix_succeeded, 1, command);
}
// A failing compound command is not an Ix error unless its output shows one.
for (const command of ['ix map && npm test', 'ix map; npm test', 'cd src && ix search foo']) {
const metrics = claude(command, true, '1 failing test');
assert.equal(metrics.ix_errors, 0, command); assert.equal(metrics.ix_succeeded, 0, command);
}
assert.deepEqual(claude('ix map && npm test', true, 'error code=STALE_MAP message="x"').ix_errors_by_code, { STALE_MAP: 1 });
assert.deepEqual(claude('ix search foo 2>&1', true, 'boom').ix_errors_by_code, { unknown: 1 });
const codex = (command, exit_code) => parseMetrics(ndjson([{ type: 'turn.started' }, { type: 'item.completed', item: { id: 'c', type: 'command_execution', command, exit_code, aggregated_output: 'failed' } },
{ type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 1, cached_input_tokens: 0 } }]), 'codex');
assert.equal(codex("/bin/bash -lc 'ix map && npm test'", 1).ix_errors, 0);
assert.equal(codex("/bin/bash -lc 'ix search foo'", 1).ix_errors, 1);
assert.deepEqual(ixExitScope('ix a; echo $\'x && y\''), { sole: false, confirmed: false });
assert.deepEqual(ixExitScope('bash -lc "ix a"'), { sole: true, confirmed: true });
assert.deepEqual(ixExitScope('rg x && bash -lc "ix a || true"'), { sole: false, confirmed: false });
});
test('Claude built-in Grep and Glob calls count as searches; permission denials are counted', () => {
const usage = { input_tokens: 1, output_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 };
const content = [{ type: 'tool_use', id: 'g', name: 'Grep', input: { pattern: 'foo' } }, { type: 'tool_use', id: 'l', name: 'Glob', input: { pattern: '**/*.ts' } }, { type: 'tool_use', id: 'r', name: 'Bash', input: { command: 'rg foo' } }, { type: 'tool_use', id: 'e', name: 'Read', input: { file_path: 'a' } }];
const metrics = parseMetrics(ndjson([{ type: 'assistant', message: { id: 'a', content } }, { type: 'result', usage, permission_denials: [{ tool_name: 'WebFetch', tool_use_id: 'w', tool_input: {} }] }]), 'claude');
assert.deepEqual([metrics.search_calls, metrics.native_search_calls, metrics.rg_calls, metrics.grep_calls, metrics.find_calls, metrics.tool_calls], [3, 2, 1, 0, 0, 4]);
assert.equal(metrics.permission_denials, 1); assert.equal(metrics.ix_permission_denials, 0);
assert.ok(METRICS.includes('native_search_calls'));
});
test('shell telemetry handles ANSI-C quotes and never aborts on an unbalanced quote', () => {
assert.equal(commandCounts("echo $'it\\'s'; ix locate A").ix_calls, 1);
assert.equal(commandCounts("echo $'a; ix b'").ix_calls, 0);
for (const command of ["echo 'unterminated; ix x", 'echo "open', "rg x; echo $'tail"]) assert.doesNotThrow(() => commandCounts(command), command);
assert.equal(commandCounts("rg x; echo 'open; ix y").rg_calls, 1);
const metrics = parseMetrics(ndjson([{ type: 'assistant', message: { id: 'a', content: [{ type: 'tool_use', id: 't', name: 'Bash', input: { command: "grep -r 'oops" } }] } },
{ type: 'result', usage: { input_tokens: 1, output_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 } }]), 'claude');
assert.equal(metrics.tool_calls, 1); assert.equal(metrics.grep_calls, 1);
});
function fixtureRuns() {
return [0, 1].flatMap((task) => [0, 1].flatMap((repeat) => ['baseline', 'ix'].map((arm) => ({ instance_id: `fixture-${task}`, arm, repeat, ...Object.fromEntries(METRICS.map((m) => [m, m === 'resolved' ? 1 : arm === 'ix' ? task * 2 : 0])) }))));
}
test('paired task bootstrap has fixture bounds and rejects duplicate/incomplete repeats', () => {
const runs = fixtureRuns(), report = summarize(runs, { samples: 1000, seed: 17 });
assert.equal(report.metrics.input_tokens.mean_difference.estimate, 1);
assert.deepEqual(report.metrics.input_tokens.mean_difference.ci90, [0, 2]);
assert.deepEqual(report.metrics.resolved.mean_difference.ci90, [0, 0]);
assert.deepEqual(report, summarize(runs, { samples: 1000, seed: 17 }));
assert.equal(report.bootstrap.unit, 'task');
assert.match(markdown(report), /90% CI/);
assert.throws(() => summarize([...runs, runs[0]]), /Duplicate/);
assert.throws(() => summarize(runs.slice(1)), /Unpaired/);
assert.throws(() => summarize([{ ...runs[0], input_tokens: NaN }]), /Invalid metric/);
});
integrationTest('three-task stub dry run executes all stages for both arms and persists reports', async () => {
const out = join(scratch, 'smoke');
const report = await runBenchmark({ manifestPath: smoke, config: dryConfig(), out, dryRun: true });
assert.equal(report.tasks, 3); assert.equal(report.repeats, 2);
const runs = readFileSync(join(out, 'runs.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
assert.equal(runs.length, 12);
assert.ok(runs.every((r) => r.resolved === 1));
assert.ok(runs.filter((r) => r.arm === 'baseline').every((r) => r.ix_calls === 0 && r.search_calls === 1));
assert.ok(runs.filter((r) => r.arm === 'ix').every((r) => r.ix_calls === 1));
assert.deepEqual(report.metrics.input_tokens.mean_difference.ci90, [-10, -10]);
assert.match(readFileSync(join(out, 'summary.md'), 'utf8'), /Synthetic dry run/);
assert.ok(existsSync(join(out, 'schedule.json')));
await assert.rejects(runBenchmark({ manifestPath: smoke, config: dryConfig(), out, dryRun: true }), /already exists/);
});
integrationTest('CLI dry-run works from the repository root without Docker or a paid agent', () => {
const out = join(scratch, 'cli');
const stdout = execFileSync(process.execPath, [join(root, 'run.mjs'), '--dry-run', '--out', out], { encoding: 'utf8' });
assert.match(stdout, /Completed/); assert.ok(existsSync(join(out, 'summary.md')));
});
integrationTest('budget exhaustion retains outcomes and completes the comparison report', async () => {
const config = dryConfig(); config.budget.max_tokens = 1;
const out = join(scratch, 'budget');
const report = await runBenchmark({ manifestPath: smoke, config, out, dryRun: true });
const runs = readFileSync(join(out, 'runs.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
assert.equal(runs.length, 12);
assert.ok(runs.every((run) => run.termination === 'token_budget' && run.resolved === 0 && run.official_resolved === 1));
assert.ok(runs.every((run) => Number.isFinite(run.hook_seconds)));
assert.equal(report.metrics.resolved.baseline.mean, 0);
assert.ok(existsSync(join(out, 'summary.md')));
});
integrationTest('invalid environments still stop loudly without a comparison report', async () => {
const badProbe = join(scratch, 'bad-probe.mjs');
writeFileSync(badProbe, "console.log(JSON.stringify({version: 1, mapped: false}))\n");
const badConfig = dryConfig(); badConfig.environment.probe = [process.execPath, badProbe];
await assert.rejects(runBenchmark({ manifestPath: smoke, config: badConfig, out: join(scratch, 'probe'), dryRun: true }), /identity/);
await assert.rejects(runBenchmark({ manifestPath: smoke, config: dryConfig(), out: join(scratch, 'wrong-mode') }), /require --dry-run/);
});
test('subprocess errors and deadlines fail rather than producing partial metrics', async () => {
await assert.rejects(execute([process.execPath, '-e', 'process.exit(7)'], { seconds: 2 }), /exited 7/);
await assert.rejects(execute([process.execPath, '-e', 'setInterval(() => {}, 1000)'], { seconds: .1 }), /deadline/);
});
test('empty solutions need explicit official summary evidence, never a guessed failure', () => {
const receipt = { report_kind: 'empty-patch-summary', resolved: false };
checkResolution({ empty_patch_ids: ['fixture'] }, receipt, 'fixture', '');
assert.throws(() => checkResolution({ empty_patch_ids: [] }, receipt, 'fixture', ''), /evidence/);
assert.throws(() => checkResolution({ empty_patch_ids: ['fixture'] }, receipt, 'fixture', 'nonempty'), /evidence/);
assert.throws(() => checkResolution({}, { resolved: false }, 'fixture', ''), /differs/);
});
test('patch-apply and test-timeout evaluation errors score unresolved; other errors stay fatal', () => {
const receipt = { report_kind: 'error-summary', resolved: false, evaluation_error: 'patch_apply_failed' };
checkResolution({ error_ids: ['fixture'] }, receipt, 'fixture', 'diff');
assert.throws(() => checkResolution({ error_ids: [] }, receipt, 'fixture', 'diff'), /evidence/);
assert.throws(() => checkResolution({ error_ids: ['fixture'] }, receipt, 'fixture', ''), /evidence/);
assert.throws(() => checkResolution({ error_ids: ['fixture'] }, { ...receipt, resolved: true }, 'fixture', 'diff'), /evidence/);
assert.throws(() => checkResolution({ error_ids: ['fixture'] }, { ...receipt, evaluation_error: 'docker' }, 'fixture', 'diff'), /evidence/);
assert.equal(evaluationError('x\n>>>>> Patch Apply Failed:\nerror: corrupt patch'), 'patch_apply_failed');
assert.equal(evaluationError('EvaluationError: Evaluation error in fixture: Test timed out after 900 seconds.'), 'test_timeout');
assert.throws(() => evaluationError('BuildImageError: docker daemon unavailable'), /inspect the harness logs/);
});
integrationTest('SWE-bench adapter turns a logged patch-apply error into an unresolved receipt', () => {
const harness = join(scratch, 'swe-harness'), python = join(scratch, 'fake-python.cjs');
mkdirSync(harness, { recursive: true });
// Isolated from developer signing/hooks config, like the other fixtures.
const git = (...args) => execFileSync('git', ['-C', harness, ...args], { encoding: 'utf8', env: { ...process.env, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: join(scratch, 'no-global-config') } });
git('init', '-q'); writeFileSync(join(harness, 'README'), 'fixture');
git('add', '.'); git('-c', 'user.name=t', '-c', 'user.email=t@t', '-c', 'commit.gpgsign=false', 'commit', '-qm', 'fixture');
const revision = git('rev-parse', 'HEAD').trim();
// Mimics the official harness: no task report, the instance listed in error_ids, the cause in run_instance.log.
writeFileSync(python, `#!${process.execPath}
const { mkdirSync, writeFileSync } = require('node:fs');
const id = process.argv[process.argv.indexOf('--instance_ids') + 1], dir = 'logs/run_evaluation/agent-eval/agent-eval/' + id;
mkdirSync(dir, { recursive: true }); writeFileSync(dir + '/run_instance.log', process.env.FAKE_INSTANCE_LOG);
writeFileSync('agent-eval.agent-eval.json', JSON.stringify({ error_ids: [id], empty_patch_ids: [], resolved_ids: [] }));
`.replace(/^\s+/gm, ''), { mode: 0o755 });
const evaluate = (name, log) => {
const run_path = join(scratch, `swe-${name}`); mkdirSync(run_path, { recursive: true });
writeFileSync(join(run_path, 'prediction.patch'), 'diff --git a/x b/x\n');
const request = { synthetic: false, dataset_export: 'fixture.json', run_path, task: { instance_id: 'fixture__repo-1' }, evaluator: { kind: 'swe-bench', revision, harness_path: harness, python, seconds: 30 } };
writeFileSync(join(run_path, 'request.json'), JSON.stringify(request));
return spawnSync(process.execPath, [join(root, 'swebench.mjs'), join(run_path, 'request.json')], { encoding: 'utf8', env: { ...process.env, FAKE_INSTANCE_LOG: log } });
};
const applied = evaluate('apply', '>>>>> Patch Apply Failed:\npatch: **** malformed patch');
assert.equal(applied.status, 0, applied.stderr);
const receipt = JSON.parse(applied.stdout);
assert.equal(receipt.resolved, false); assert.equal(receipt.report_kind, 'error-summary'); assert.equal(receipt.evaluation_error, 'patch_apply_failed');
const docker = evaluate('docker', 'BuildImageError: docker daemon unavailable');
assert.notEqual(docker.status, 0); assert.match(docker.stderr, /inspect the harness logs/);
});
integrationTest('real-run plumbing checks dataset pins and official evidence and includes new solution files', async () => {
// All data/evaluation here are synthetic fixtures; this does not exercise Docker.
const source = join(scratch, 'source'), cache = join(scratch, 'cache');
mkdirSync(source); mkdirSync(cache);
const git = (...args) => execFileSync('git', args, { cwd: source, encoding: 'utf8', env: { ...process.env, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: join(scratch, 'no-global-config') } });
git('init', '-q'); writeFileSync(join(source, 'source.txt'), 'before\n'); git('add', '.');
git('-c', 'user.name=Fixture', '-c', '[email protected]', '-c', 'commit.gpgsign=false', 'commit', '-qm', 'fixture');
const base = git('rev-parse', 'HEAD').trim();
git('clone', '--bare', source, join(cache, 'fixture__repo.git'));
const exportPath = join(scratch, 'real-fixture-export.json');
const text = JSON.stringify([{ ...rows[0], repo: 'fixture/repo', base_commit: base }]);
writeFileSync(exportPath, text);
const manifestPath = join(scratch, 'real-fixture-manifest.json');
writeFileSync(manifestPath, JSON.stringify(importVerified(text, { count: 1, minRepos: 1 })));
const adapter = join(scratch, 'fixture-adapter.mjs');
const lateWriter = `const [parent, file] = process.argv.slice(1); const wait = () => { try { process.kill(Number(parent), 0); setTimeout(wait, 20); } catch { setTimeout(() => require('node:fs').writeFileSync(file, 'late\\n'), 300); } }; wait();`;
writeFileSync(adapter, `
import {mkdirSync, readFileSync, writeFileSync} from 'node:fs';
import {join} from 'node:path';
import {createHash} from 'node:crypto';
const [stage,file] = process.argv.slice(2), r = JSON.parse(readFileSync(file, 'utf8'));
const hash = text => createHash('sha256').update(text).digest('hex');
const ix = r.arm === 'ix', evidence = join(r.home, 'fixture-evidence.json'), OUTS = ${JSON.stringify(['real-fixture', 'bad-official', 'bad-hash'].map((name) => join(scratch, name) + sep))};
if(stage === 'setup') {
writeFileSync(evidence, '{}'); mkdirSync(join(r.repo_path, '.claude'));
writeFileSync(join(r.repo_path, '.claude', 'settings.json'), '{}');
} else if(stage === 'probe') console.log(JSON.stringify({version:1, repo_path:r.repo_path, base_commit:r.task.base_commit, agent_version:r.agent.version, ix_available:ix, mapped:ix, plugin_installed:ix, skills_installed:ix, hooks_enabled:ix, mapped_commit:r.task.base_commit, plugin_revision:r.environment.plugin_revision, evidence:ix ? [{path:evidence, sha256:hash('{}')}] : []}));
else if(stage === 'agent') {
// The solver must not see the gold-patch export, the evaluator config or the output tree with earlier runs.
const visible = readFileSync(file, 'utf8') + readFileSync(process.env.AGENT_EVAL_REQUEST, 'utf8') + JSON.stringify(process.env);
if (visible.includes(${JSON.stringify(exportPath)}) || OUTS.some((out) => visible.includes(out)) || 'dataset_export' in r || 'evaluator' in r) process.exit(9);
writeFileSync(join(r.repo_path, 'source.txt'), 'after\\n'); writeFileSync(join(r.repo_path, 'added.txt'), 'new solution\\n');
const {execFileSync} = await import('node:child_process');
execFileSync('git', ['add', 'source.txt', 'added.txt']);
execFileSync('git', ['-c','user.name=Fixture','-c','[email protected]','-c','commit.gpgsign=false','commit','-qm','agent fix']);
for(const e of [{type:'turn',id:'a'}, {type:'tool',id:'t',command:'rg fixture'}, {type:'usage',input_tokens:10,output_tokens:10,cached_tokens:0}, {type:'done'}]) console.log(JSON.stringify(e));
// A background process the agent leaves behind: it edits the checkout once the agent has exited.
const {spawn} = await import('node:child_process');
spawn(process.execPath, ['-e', ${JSON.stringify(lateWriter)}, String(process.pid), join(r.repo_path, 'late.txt')], {stdio: 'ignore'}).unref();
} else if(stage === 'evaluate') {
if (r.dataset_export !== ${JSON.stringify(exportPath)} || !OUTS.some((out) => r.run_path.startsWith(out))) process.exit(9);
const report_path = join(r.run_path,'fixture-official-report.json');
const report = JSON.stringify({[r.task.instance_id]:{resolved:true}}); writeFileSync(report_path,report);
console.log(JSON.stringify({instance_id:r.task.instance_id,resolved:!r.evaluator.mismatch,evaluator:'swe-bench',revision:r.evaluator.revision,report_path,report_sha256:hash(report)}));
}
`);
const config = dryConfig();
config.agent = { ...config.agent, family: 'claude', command: [process.execPath, adapter, 'agent'] };
config.environment = { ...config.environment, plugin_revision: base, setup: [process.execPath, adapter, 'setup'], probe: [process.execPath, adapter, 'probe'] };
config.evaluator = { ...config.evaluator, kind: 'swe-bench', revision: base, command: [process.execPath, adapter, 'evaluate'] };
config.dataset_export = exportPath; config.repo_cache = cache;
const out = join(scratch, 'real-fixture');
const report = await runBenchmark({ manifestPath, config, out });
assert.equal(report.metrics.resolved.baseline.mean, 1);
const jobs = JSON.parse(readFileSync(join(out, 'schedule.json'), 'utf8'));
const first = jobs[0], runPath = join(out, `0000-${first.instance_id}-${first.repeat}-${first.arm}`);
const patch = readFileSync(join(runPath, 'prediction.patch'), 'utf8');
assert.match(patch, /source.txt/); assert.match(patch, /added.txt/); assert.doesNotMatch(patch, /settings.json/);
// Nothing the agent left running may change the checkout after it exits, in either arm: the recorded patch is the agent's.
for (const [index, job] of jobs.entries()) {
const recorded = readFileSync(join(out, `${String(index).padStart(4, '0')}-${job.instance_id}-${job.repeat}-${job.arm}`, 'prediction.patch'), 'utf8');
assert.match(recorded, /added.txt/); assert.doesNotMatch(recorded, /late.txt/, `${job.arm} patch includes a post-exit write`);
}
assert.deepEqual(new Set(jobs.map((job) => job.arm)), new Set(['baseline', 'ix']));
// Each solver worked outside the output tree, in a work directory removed once its run was recorded.
const agentRequest = JSON.parse(readFileSync(join(runPath, 'agent-request.json'), 'utf8'));
assert.ok(!agentRequest.run_path.startsWith(out) && !existsSync(agentRequest.run_path));
await assert.rejects(runBenchmark({ manifestPath, config: { ...config, work_root: scratch }, out: join(scratch, 'inside-work-root') }), /inside the agent work root/);
config.evaluator.mismatch = true;
await assert.rejects(runBenchmark({ manifestPath, config, out: join(scratch, 'bad-official') }), /differs from official/);
writeFileSync(exportPath, text + '\n');
await assert.rejects(runBenchmark({ manifestPath, config, out: join(scratch, 'bad-hash') }), /hash mismatch/);
});
test('quoted command examples are not counted as executed Ix calls', () => {
for (const cmd of ["echo 'example; ix locate Foo '", 'echo "example && ix locate Foo"', "printf '%s\\n' 'rg a | grep b'", 'echo escaped\\; ixample']) {
assert.equal(commandCounts(cmd).ix_calls, 0);
assert.equal(commandCounts(cmd).search_calls, 0);
}
assert.equal(commandCounts('"ix" locate Foo; echo "ix locate Bar"').ix_calls, 1);
});
test('example real configuration ships its setup and probe adapters', () => {
const config = JSON.parse(readFileSync(join(root, 'config.example.json'), 'utf8'));
for (const stage of ['setup', 'probe']) assert.ok(existsSync(resolve(root, '../..', config.environment[stage][1])), `missing ${stage}`);
});
test('quote-aware command counting handles shell wrappers, escapes and comments', () => {
assert.equal(commandCounts("/bin/bash -lc 'rg needle; ix locate Foo'").ix_calls, 1);
assert.equal(commandCounts("/bin/bash -lc 'echo \"example; ix locate Foo\"'").ix_calls, 0);
assert.equal(commandCounts('echo escaped\\;ix locate Foo').ix_calls, 0);
assert.equal(commandCounts('# ix locate Foo\nrg needle').rg_calls, 1);
assert.equal(commandCounts('echo "a|grep x"; "grep" needle').grep_calls, 1);
});
test('heredoc bodies are data, not ix or search commands', () => {
// A typical agent repro script: Python names like `ix` and lines starting with grep are file content.
const repro = "cat > reproduce.py << 'EOF'\nimport numpy as np\nix = np.arange(3)\nprint(ix)\ngrep = \"don't\"\ngrep foo bar\nEOF\npython reproduce.py && rg needle";
assert.deepEqual(commandCounts(repro), { search_calls: 1, rg_calls: 1, grep_calls: 0, find_calls: 0, ix_calls: 0 });
assert.equal(commandCounts('cat <<-EOF > x\n\tix locate A\n\tEOF\nix locate B').ix_calls, 1);
assert.equal(commandCounts('cat <<A <<B\nix 1\nA\nix 2\nB\nix locate C').ix_calls, 1);
assert.equal(commandCounts('python - <<EOF\nix\n').ix_calls, 0);
assert.equal(commandCounts('cat <<EOF; ix locate same-line\nix body\nEOF').ix_calls, 1);
assert.equal(commandCounts('echo a <<< "ix locate x"; ix locate y').ix_calls, 1);
assert.equal(commandCounts('bash -lc "cat > f <<EOF\nix bad\nEOF\nix good"').ix_calls, 1);
const tool = { type: 'tool_use', id: 'h', name: 'Bash', input: { command: repro } };
const metrics = parseMetrics(ndjson([{ type: 'assistant', message: { id: 'a', content: [tool] } }, { type: 'result', usage: { input_tokens: 1, output_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 } }]), 'claude');
assert.equal(metrics.ix_calls, 0); assert.equal(metrics.grep_calls, 0);
});
// Emits a Claude stream-json transcript; the mode argument picks the arm-specific Ix behaviour.
function claudeTranscriptAgent() {
const script = join(scratch, 'claude-transcript-agent.mjs');
writeFileSync(script, `
import { readFileSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
const [mode, file] = process.argv.slice(2), r = JSON.parse(readFileSync(file, 'utf8'));
writeFileSync(join(r.repo_path, 'solution.txt'), 'synthetic solution\\n');
const bash = { type: 'tool_use', id: 'b', name: 'Bash', input: { command: 'ix locate Foo' } };
const mcp = { type: 'tool_use', id: 'm', name: ${JSON.stringify(CLAUDE_IX_MCP_RULE + '__locate')}, input: {} };
const events = [], result = { type: 'result', usage: { input_tokens: 10, output_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 } };
if (mode === 'denied' && r.arm === 'ix') {
events.push({ type: 'assistant', message: { id: 'a', content: [mcp] } });
result.permission_denials = [{ tool_name: mcp.name, tool_use_id: 'm', tool_input: {} }];
} else if (mode.startsWith('baseline') && r.arm === 'baseline') {
events.push({ type: 'assistant', message: { id: 'a', content: [bash] } });
const failed = mode === 'baseline-attempt', masked = mode === 'baseline-masked';
if (masked) bash.input.command = 'ix locate Foo 2>&1 | head -5';
events.push({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'b', is_error: failed, content: failed ? 'bash: ix: command not found' : masked ? 'sh: 1: ix: not found' : 'Foo src/foo.ts:1' }] } });
} else events.push({ type: 'assistant', message: { id: 'a', content: [{ type: 'text', text: 'done' }] } });
for (const event of [...events, result]) console.log(JSON.stringify(event));
`);
return (mode) => { const config = dryConfig(); config.agent = { ...config.agent, telemetry: 'claude', command: [process.execPath, script, mode] }; return config; };
}
integrationTest('an Ix permission denial fails the run; a failed baseline ix attempt does not, a working one does', async () => {
const config = claudeTranscriptAgent();
await assert.rejects(runBenchmark({ manifestPath: smoke, config: config('denied'), out: join(scratch, 'ix-denied'), dryRun: true }), /denied Ix tools/);
const out = join(scratch, 'baseline-attempt');
await runBenchmark({ manifestPath: smoke, config: config('baseline-attempt'), out, dryRun: true });
const runs = readFileSync(join(out, 'runs.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
assert.ok(runs.filter((run) => run.arm === 'baseline').every((run) => run.ix_calls === 1 && run.ix_errors === 1 && run.ix_succeeded === 0));
// A pipeline exiting 0 still shows the shell's not-found message; that is an attempt, not a leak.
const masked = join(scratch, 'baseline-masked');
await runBenchmark({ manifestPath: smoke, config: config('baseline-masked'), out: masked, dryRun: true });
const maskedRuns = readFileSync(join(masked, 'runs.jsonl'), 'utf8').trim().split('\n').map(JSON.parse).filter((run) => run.arm === 'baseline');
assert.ok(maskedRuns.every((run) => run.ix_succeeded === 0 && run.ix_errors_by_code.COMMAND_NOT_FOUND === 1));
await assert.rejects(runBenchmark({ manifestPath: smoke, config: config('baseline-leak'), out: join(scratch, 'baseline-leak'), dryRun: true }), /Baseline ix call succeeded/);
});
integrationTest('hook-driven Ix calls are reported apart from the agent\'s own Ix calls', async () => {
const script = join(scratch, 'hook-agent.mjs');
writeFileSync(script, `
import { appendFileSync, readFileSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
const r = JSON.parse(readFileSync(process.argv[2], 'utf8'));
writeFileSync(join(r.repo_path, 'solution.txt'), 'synthetic solution\\n');
if (r.arm === 'ix') for (const hook of [{ type: 'hook', mode: 'claude', ix_calls: 2, seconds: 0.5 }, { type: 'ix_error', code: 'HOOK_FAILED', ix_calls: 1, seconds: 0.25 }]) appendFileSync(join(r.run_path, 'hooks.jsonl'), JSON.stringify(hook) + '\\n');
for (const e of [{ type: 'turn', id: 't' }, { type: 'tool', id: 'x', command: r.arm === 'ix' ? 'ix locate A' : 'rg A' }, { type: 'usage', input_tokens: 10, output_tokens: 1, cached_tokens: 0 }, { type: 'done' }]) console.log(JSON.stringify(e));
`);
const config = dryConfig(); config.agent.command = [process.execPath, script];
const out = join(scratch, 'hook-split');
const report = await runBenchmark({ manifestPath: smoke, config, out, dryRun: true });
const runs = readFileSync(join(out, 'runs.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
for (const run of runs.filter((r) => r.arm === 'ix')) assert.deepEqual([run.ix_calls, run.ix_errors, run.hook_ix_calls, run.hook_ix_errors, run.hook_seconds], [1, 0, 3, 1, 0.75]);
for (const run of runs.filter((r) => r.arm === 'baseline')) assert.deepEqual([run.ix_calls, run.hook_ix_calls, run.hook_ix_errors], [0, 0, 0]);
assert.equal(report.metrics.hook_ix_calls.mean_difference.estimate, 3);
});
integrationTest('the ix hook re-maps only after the checkout changed; the probe always maps', async () => {
const run_path = join(scratch, 'hook-refresh'), repo_path = join(run_path, 'repo'), home = join(run_path, 'home'), log = join(run_path, 'ix.log');
mkdirSync(repo_path, { recursive: true }); mkdirSync(home);
const git = (...args) => execFileSync('git', ['-C', repo_path, ...args], { encoding: 'utf8' });
git('init', '-q'); writeFileSync(join(repo_path, 'a.txt'), 'a\n'); git('add', '.');
git('-c', 'user.name=Fixture', '-c', '[email protected]', '-c', 'commit.gpgsign=false', 'commit', '-qm', 'fixture');
const fakeIx = join(run_path, 'fake-ix.mjs');
writeFileSync(fakeIx, `import { appendFileSync } from 'node:fs'; appendFileSync(${JSON.stringify(log)}, process.argv[2] + '\\n'); if (process.argv[2] === 'hook') console.log('{}');`);
const request = { arm: 'ix', run_path, repo_path, home, environment: { seconds: 10, ix_cli: fakeIx, endpoint: 'http://127.0.0.1:8090', tool_paths: ['/usr/bin', '/bin'] } };
const env = armEnvironment(request); writeFileSync(env.AGENT_EVAL_REQUEST, JSON.stringify(request));
const hook = async (mode) => {
writeFileSync(log, '');
await execute([process.execPath, join(root, 'hook.mjs'), mode], { cwd: repo_path, env, input: '{}' });
return readFileSync(log, 'utf8').trim().split('\n');
};
assert.deepEqual(await hook('probe'), ['map']);
assert.deepEqual(await hook('claude'), ['hook'], 'a read-only call on the probed checkout does not map');
writeFileSync(join(repo_path, 'a.txt'), 'b\n');
assert.deepEqual(await hook('claude'), ['map', 'hook']);
assert.deepEqual(await hook('claude'), ['hook']);
writeFileSync(join(repo_path, 'new.txt'), 'new\n');
assert.deepEqual(await hook('claude'), ['map', 'hook'], 'a new untracked file maps');
assert.deepEqual(await hook('claude'), ['hook']);
writeFileSync(join(repo_path, 'new.txt'), 'changed again\n');
assert.deepEqual(await hook('claude'), ['map', 'hook'], 'an edited untracked file maps');
assert.deepEqual(await hook('probe'), ['map'], 'the probe maps even when nothing changed');
const records = readFileSync(join(run_path, 'hooks.jsonl'), 'utf8').trim().split('\n').map((line) => JSON.parse(line));
assert.deepEqual(records.map((r) => [r.mapped, r.ix_calls]), [[true, 1], [false, 1], [true, 2], [false, 1], [true, 2], [false, 1], [true, 2], [true, 1]]);
});
integrationTest('the probe fails when a login shell profile resets PATH and undoes arm isolation', async () => {
const dir = join(scratch, 'login-shell'), ambient = join(dir, 'ambient'), keeps = join(dir, 'keeps'), resets = join(dir, 'resets'), broken = join(dir, 'broken');
for (const d of [ambient, keeps, resets, broken]) mkdirSync(d, { recursive: true });
writeFileSync(join(ambient, 'ix'), '#!/bin/sh\nexit 0\n', { mode: 0o755 });
// Stand-ins for `bash -l`: one keeps the inherited PATH, one resets it the way a distro /etc/profile can.
writeFileSync(join(keeps, 'bash'), '#!/bin/sh\n[ "$1" = -lc ] || exit 2\necho "profile banner"\nexec /bin/sh -c "$2"\n', { mode: 0o755 });
writeFileSync(join(resets, 'bash'), `#!/bin/sh\n[ "$1" = -lc ] || exit 2\nPATH=${shellQuote(ambient)}:/usr/bin:/bin; export PATH\nexec /bin/sh -c "$2"\n`, { mode: 0o755 });
copyFileSync(join(resets, 'bash'), join(resets, 'zsh'));
// Stand-in for a profile.d script written for another shell, as dash reports it.
const failing = '#!/bin/sh\necho \'profile.d/tool.sh: 5: eval: Syntax error: "(" unexpected\' >&2\nexit 2\n';
for (const name of ['bash', 'sh']) writeFileSync(join(broken, name), failing, { mode: 0o755 });
for (const arm of ['baseline', 'ix']) {
const run_path = join(dir, arm), bin = join(run_path, 'bin'); mkdirSync(bin, { recursive: true });
if (arm === 'ix') writeFileSync(join(bin, 'ix'), '#!/bin/sh\nexit 0\n', { mode: 0o755 });
const request = { arm, run_path, repo_path: dir, environment: { seconds: 10 } };
// The shells are found on the arm's PATH, as in a run; SHELL is unset so only `bash` is checked.
const env = (shells) => ({ ...process.env, SHELL: '', PATH: [bin, shells].join(delimiter) });
await checkLoginShells(request, env(keeps));
await assert.rejects(checkLoginShells(request, env(resets)), /login shell .*resets PATH/, arm);
// Claude Code only runs bash or zsh: a $SHELL of zsh is probed, while sh (whose profile may not even parse) is not.
await assert.rejects(checkLoginShells(request, { ...env(keeps), SHELL: join(resets, 'zsh') }), /zsh -l\) resolves ix/, arm);
await checkLoginShells(request, { ...env(keeps), SHELL: join(broken, 'sh') });
// A profile that fails under the arm's PATH reports the shell's own error instead of a bare exit status.
await assert.rejects(checkLoginShells(request, env(broken)), /bash -l\) exited 2 .*eval: Syntax error/, arm);
}
});
// The adapter spawns a descendant (same process group, like a Codex notify hook) and exits at once. The descendant
// waits for the adapter to exit, then `ms`, then writes `file`. Timing is relative to the adapter's exit rather than
// to process start-up, so a loaded machine (the full CI suite runs alongside) does not change the outcome.
// The descendant's command line ends with `tag`; `keep` matches the hook-fixture tag, as isHookProcess matches hook.mjs.
const lateDescendant = (file, ms, stdio = 'ignore', tag = 'hook-fixture') => {
const child = `const parent = Number(process.argv[1]); const wait = () => { try { process.kill(parent, 0); setTimeout(wait, 20); } catch { setTimeout(() => { console.log('hook'); require('node:fs').writeFileSync(${JSON.stringify(file)}, 'x'); }, ${ms}); } }; wait();`;
return [process.execPath, '-e', `require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify(child)}, String(process.pid), ${JSON.stringify(tag)}], { stdio: ${JSON.stringify(stdio)} }).unref(); console.log('agent')`];
};
const keep = (args) => args.endsWith(' hook-fixture');
const timed = async (promise) => { const started = performance.now(); const result = await promise; return { result, seconds: (performance.now() - started) / 1000 }; };
integrationTest('a drained stage lets an outliving descendant finish before the group is reaped, within a cap', async () => {
const marker = (name) => join(scratch, `drain-${name}`);
const drained = await timed(execute(lateDescendant(marker('drained'), 1000), { seconds: 10, drainSeconds: 10, keep }));
assert.ok(existsSync(marker('drained')));
// The descendant finished at least 1s after the adapter exited; none of that is stage time.
assert.ok(drained.seconds - drained.result.seconds >= 0.5, 'stage time excludes the drain');
await execute(lateDescendant(marker('reaped'), 1000), { seconds: 10 });
await new Promise((done) => setTimeout(done, 2000));
assert.ok(!existsSync(marker('reaped')), 'without a drain, descendants are still reaped');
const capped = await timed(execute(lateDescendant(marker('capped'), 30_000), { seconds: 10, drainSeconds: 0.5, keep }));
assert.ok(capped.seconds < 15, 'the drain is capped');
assert.ok(!existsSync(marker('capped')));
});
integrationTest('only kept (hook) processes and their children outlive the adapter; anything else is killed at its exit', async () => {
const marker = (name) => join(scratch, `cut-${name}`);
// A leftover that is not a hook gets no drain, even when the stage drains for hooks.
const other = await timed(execute(lateDescendant(marker('other'), 300, 'ignore', 'agent-leftover'), { seconds: 10, drainSeconds: 10, keep }));
assert.ok(other.seconds < 5, 'nothing kept, so no drain');
await new Promise((done) => setTimeout(done, 1000));
assert.ok(!existsSync(marker('other')), 'a non-hook leftover never writes after the adapter exits');
// One adapter leaves a hook (whose own child does the work, like the hook's `ix map`) and a leftover beside it.
const grandchild = `setTimeout(() => require('node:fs').writeFileSync(process.argv[1], 'x'), 300)`;
const hook = `const parent = Number(process.argv[1]); const wait = () => { try { process.kill(parent, 0); setTimeout(wait, 20); } catch { require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify(grandchild)}, ${JSON.stringify(marker('hook-child'))}], { stdio: 'ignore' }); } }; wait();`;
const leftover = `const parent = Number(process.argv[1]); const wait = () => { try { process.kill(parent, 0); setTimeout(wait, 20); } catch { setTimeout(() => require('node:fs').writeFileSync(${JSON.stringify(marker('leftover'))}, 'x'), 300); } }; wait();`;
const spawnBoth = `const { spawn } = require('node:child_process');
spawn(process.execPath, ['-e', ${JSON.stringify(hook)}, String(process.pid), 'hook-fixture'], { stdio: 'ignore' }).unref();
spawn(process.execPath, ['-e', ${JSON.stringify(leftover)}, String(process.pid), 'agent-leftover'], { stdio: 'ignore' }).unref();`;
await execute([process.execPath, '-e', spawnBoth], { seconds: 10, drainSeconds: 10, keep });
await new Promise((done) => setTimeout(done, 1000));
assert.ok(existsSync(marker('hook-child')), 'the hook and the child it starts run to completion');
assert.ok(!existsSync(marker('leftover')), 'the leftover beside the hook was killed at the adapter exit');
});
integrationTest('a descendant holding the stage stdout open is neither stage time nor a deadline overrun', async () => {
const marker = (name) => join(scratch, `inherit-${name}`);
// Like a notify hook spawned with inherited stdio: the adapter exits, the hook keeps the stdout pipe open past the deadline.
const drained = await execute(lateDescendant(marker('drained'), 2500, 'inherit'), { seconds: 2, drainSeconds: 10, allowTimeout: true, keep });
assert.equal(drained.timed_out, false); assert.ok(drained.seconds < 2, 'stage time ends when the adapter exits');
assert.ok(existsSync(marker('drained'))); assert.match(drained.stdout, /agent/);
// The descendant would hold stdout for 6s after the adapter exits; the stage closing well before that means it was reaped.
const reaped = await timed(execute(lateDescendant(marker('reaped'), 6000, 'inherit'), { seconds: 10, drainSeconds: 0.5, keep }));
assert.equal(reaped.result.timed_out, false); assert.ok(reaped.seconds - reaped.result.seconds < 4, 'the drain is capped');
assert.match(reaped.result.stdout, /agent/); assert.doesNotMatch(reaped.result.stdout, /hook/, 'the descendant is reaped when the drain ends');
});
integrationTest('an ix-arm hook that outlives the agent is counted; one killed mid-run fails the run', async () => {
const script = join(scratch, 'late-hook-agent.mjs'), leftovers = join(scratch, 'late-hook-leftovers');
mkdirSync(leftovers);
writeFileSync(script, `
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { spawn } from 'node:child_process';
const [mode, file] = process.argv.slice(2), r = JSON.parse(readFileSync(file, 'utf8'));
writeFileSync(join(r.repo_path, 'solution.txt'), 'synthetic solution\\n');
if (r.arm === 'ix' && mode === 'late') {
const record = JSON.stringify({ type: 'hook', mode: 'codex', ix_calls: 1, seconds: 0.5 }) + '\\n';
// The command line names hook.mjs, as a real hook's does; that is what lets it outlive the agent.
spawn(process.execPath, ['-e', 'setTimeout(() => require("node:fs").appendFileSync(process.argv[1], process.argv[2]), 500)', join(r.run_path, 'hooks.jsonl'), record, ${JSON.stringify(HOOK_SCRIPT)}], { stdio: 'ignore' }).unref();
}
// In both arms, a leftover that is not a hook must be gone by the time the drain ends.
if (mode === 'late') spawn(process.execPath, ['-e', 'setTimeout(() => require("node:fs").writeFileSync(process.argv[1], "x"), 500)', join(${JSON.stringify(leftovers)}, r.arm + '-' + r.repeat)], { stdio: 'ignore' }).unref();
if (r.arm === 'ix' && mode === 'killed') { mkdirSync(join(r.run_path, 'hooks.pending'), { recursive: true }); writeFileSync(join(r.run_path, 'hooks.pending', '1'), 'codex'); }
for (const e of [{ type: 'turn', id: 't' }, { type: 'usage', input_tokens: 10, output_tokens: 1, cached_tokens: 0 }, { type: 'done' }]) console.log(JSON.stringify(e));
`);
const config = (mode) => { const c = dryConfig(); c.repeats = 1; c.agent.command = [process.execPath, script, mode]; return c; };
const out = join(scratch, 'late-hook');
await runBenchmark({ manifestPath: smoke, config: config('late'), out, dryRun: true });
const runs = readFileSync(join(out, 'runs.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
assert.ok(runs.filter((r) => r.arm === 'ix').every((r) => r.hook_ix_calls === 1));
await new Promise((done) => setTimeout(done, 1000));
assert.deepEqual(readdirSync(leftovers), [], 'non-hook leftovers were killed at agent exit in both arms');
await assert.rejects(runBenchmark({ manifestPath: smoke, config: config('killed'), out: join(scratch, 'killed-hook'), dryRun: true }), /hook was still running/);
});
integrationTest('agent.mjs gives each Claude arm its own --allowedTools through the native binary', async () => {
const fake = join(scratch, 'fake-claude.cjs');
writeFileSync(fake, `#!${process.execPath}
const { writeFileSync } = require('node:fs');
if (process.argv[2] === '--version') console.log('9.9.9-fixture (Claude Code)');
else { writeFileSync(process.env.FAKE_CLAUDE_ARGV, JSON.stringify(process.argv.slice(2))); process.stdin.resume(); }
`.replace(/^\s+/gm, ''), { mode: 0o755 });
const argv = {};
for (const arm of ['baseline', 'ix']) {
const run_path = join(scratch, `allowed-${arm}`); mkdirSync(run_path, { recursive: true });
const request = { arm, run_path, task: { problem_statement: 'fixture' }, budget: { max_turns: 3 },
agent: { family: 'claude', binary: fake, version: '9.9.9-fixture', model: 'fixture-model', effort: 'medium', args: ['--permission-mode', 'acceptEdits'], allowed_tools: ['Bash'] } };
writeFileSync(join(run_path, 'request.json'), JSON.stringify(request));
const out = join(run_path, 'argv.json');
await execute([process.execPath, join(root, 'agent.mjs'), join(run_path, 'request.json')], { env: { ...process.env, FAKE_CLAUDE_ARGV: out }, seconds: 20 });
argv[arm] = JSON.parse(readFileSync(out, 'utf8'));
}
const allowed = (args) => args.filter((arg, i) => args[i - 1] === '--allowedTools');
assert.deepEqual(allowed(argv.ix), [`Bash,${CLAUDE_IX_MCP_RULE}`]);
assert.deepEqual(allowed(argv.baseline), ['Bash']);
assert.ok(argv.ix.includes('--plugin-dir') && !argv.baseline.includes('--plugin-dir'));
assert.ok(argv.baseline.includes('--strict-mcp-config'));
});
test('mapping probe rejects successful-but-empty, stale and partial maps', () => {
const status = { graphCompleted: true, mapCompleted: true, staleFiles: 0, graphHealth: { status: 'ok' } };
const stats = { nodes: { total: 3 } };
assertMapped(status, stats);
for (const patch of [{ graphCompleted: false }, { mapCompleted: null }, { staleFiles: 1 }, { graphHealth: { status: 'empty' } }, { replayedFiles: ['file.py'] }, { parseTimeouts: ['file.py'] }]) {
assert.throws(() => assertMapped({ ...status, ...patch }, stats), /unmapped/);
}
assert.throws(() => assertMapped(status, { nodes: { total: 0 } }), /unmapped/);
});
integrationTest('shipped provisioner installs pinned local integrations and isolates PATH in both native arms', async () => {
const source = join(scratch, 'ix-source'); mkdirSync(source);
const git = (...args) => execFileSync('git', args, { cwd: source, encoding: 'utf8', env: { ...process.env, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: join(scratch, 'absent-config') } });
git('init', '-q'); mkdirSync(join(source, 'skills', 'ix'), { recursive: true });
writeFileSync(join(source, 'skills', 'ix', 'SKILL.md'), '---\nname: ix\ndescription: Fixture\n---\nPinned skill\n');
git('add', '.'); git('-c', 'user.name=Fixture', '-c', '[email protected]', '-c', 'commit.gpgsign=false', 'commit', '-qm', 'fixture');
const revision = git('rev-parse', 'HEAD').trim();
const tools = join(scratch, 'ambient-bin'); mkdirSync(tools);
writeFileSync(join(tools, 'ix'), '#!/bin/sh\nexit 99\n', { mode: 0o755 });
// A profile-free `bash -l`: the host's /etc/profile is not under test here (the login-shell probe test covers profiles).
writeFileSync(join(tools, 'bash'), '#!/bin/sh\n[ "$1" = -lc ] && exec /bin/sh -c "$2"\nexec /bin/bash "$@"\n', { mode: 0o755 });
// execute() reports only the exit status; the provisioner's own message on stderr is the diagnosis.
const stage = async (argv, options) => {
const result = await execute(argv, { ...options, captureExit: true });
assert.equal(result.exit_code, 0, `${argv.slice(1, 3).join(' ')} exited ${result.exit_code}: ${result.stderr.trim() || 'no stderr'}`);
return result;
};
const fakeIx = join(scratch, 'fake-ix.mjs');
writeFileSync(fakeIx, `
import {existsSync,mkdirSync,writeFileSync} from 'node:fs';
import {join} from 'node:path';
const [cmd] = process.argv.slice(2), marker = join(process.env.IX_HOME,'mapped');
if(cmd==='--version') console.log('ix 1.0-fixture');
else if(cmd==='map') { mkdirSync(process.env.IX_HOME,{recursive:true}); writeFileSync(marker,process.cwd()); }
else if(cmd==='status') console.log(JSON.stringify({graphCompleted:existsSync(marker),mapCompleted:existsSync(marker),staleFiles:0,graphHealth:{status:'ok'}}));
else if(cmd==='stats') console.log(JSON.stringify({nodes:{total:existsSync(marker)?10:0}}));
else if(cmd==='hook') console.log('{}');
else process.exitCode=1;
`);
const fakeAgent = join(scratch, 'fake-native');
writeFileSync(fakeAgent, `#!${process.execPath}
import {existsSync,readdirSync,readFileSync,writeFileSync} from 'node:fs';
import {join} from 'node:path';
import {execFileSync} from 'node:child_process';
if(process.argv[2]==='--version') console.log('agent 1.0-fixture');
else if(process.argv[2]==='debug') {
// Mirrors codex debug prompt-input: the model-visible skill list for $CODEX_HOME/skills.
const dir=join(process.env.CODEX_HOME,'skills');
const skills=process.env.FAKE_CODEX_HIDE_SKILLS||!existsSync(dir)?[]:readdirSync(dir).map((name)=>'- '+readFileSync(join(dir,name,'SKILL.md'),'utf8').match(/^name: (.*)$/m)[1]+': Fixture (file: r0/'+name+'/SKILL.md)');
console.log(JSON.stringify([{type:'message',role:'developer',content:[{type:'input_text',text:'<skills_instructions>\\n'+skills.join('\\n')+'\\n</skills_instructions>'}]}]));
}
else {
const r=JSON.parse(readFileSync(process.env.AGENT_EVAL_REQUEST,'utf8'));
const args=process.argv.slice(2); writeFileSync(join(r.run_path,'native-args.json'),JSON.stringify(args));
if(r.arm==='ix') {
if(r.agent.family==='claude') {
const plugin=args[args.indexOf('--plugin-dir')+1];
const hooks=JSON.parse(readFileSync(join(plugin,'hooks','hooks.json'),'utf8'));
execFileSync('sh',['-c',hooks.hooks.PostToolUse[0].hooks[0].command],{input:'{}'});
} else {
const config=readFileSync(join(process.env.CODEX_HOME,'config.toml'),'utf8');
const notify=JSON.parse(config.split('\\n')[0].slice('notify = '.length));
execFileSync(notify[0],[...notify.slice(1),JSON.stringify({type:'agent-turn-complete'})]);
}
}
for(const e of [{type:'turn',id:'a'},{type:'usage',input_tokens:10,output_tokens:2,cached_tokens:0},{type:'done'}]) console.log(JSON.stringify(e));
}
`, { mode: 0o755 });
for (const family of ['claude', 'codex']) for (const arm of ['baseline', 'ix']) {
const run_path = join(scratch, `provision-${family}-${arm}`), repo_path = join(run_path, 'repo'), home = join(run_path, 'home');
mkdirSync(home, { recursive: true });
execFileSync('git', ['clone', '-q', source, repo_path]);
// This repository's skill is inert until installed in the agent-specific location.
const request = { synthetic: true, arm, run_path, repo_path, home, task: { base_commit: revision, problem_statement: 'fixture' }, agent: { family, binary: fakeAgent, version: '1.0-fixture', model: 'fixture-model', effort: 'medium' }, budget: { max_turns: 2 },
environment: { plugin_revision: revision, source_path: source, ix_cli: fakeIx, ix_version: '1.0-fixture', endpoint: 'http://127.0.0.1:8090', seconds: 10, tool_paths: [tools, '/usr/bin', '/bin'] } };
assert.throws(() => checkTaskIsolation({ ...request, synthetic: false }), (error) => /ancestor/.test(error.message) && error.message.includes(resolve(root, '..', '..')));
// SHELL is cleared so the outcome does not depend on the caller's login shell; `bash -l` is the profile-free stand-in.
const env = armEnvironment(request, { ...process.env, SHELL: '', IX_ENDPOINT: 'http://wrong.invalid', IX_WORKSPACE_ID: 'wrong', JARVIS_IX_SOCKET: 'wrong',
CLAUDE_CODE_OAUTH_TOKEN: 'fixture-not-a-token', CLAUDE_CODE_USE_BEDROCK: '1', CLAUDE_CODE_ENTRYPOINT: 'parent-session' });
assert.equal(env.IX_WORKSPACE_ID, undefined); assert.equal(env.JARVIS_IX_SOCKET, undefined);
// Native CLI auth/provider routing survives isolation in both arms; parent-session variables do not.
assert.equal(env.CLAUDE_CODE_OAUTH_TOKEN, 'fixture-not-a-token'); assert.equal(env.CLAUDE_CODE_USE_BEDROCK, '1'); assert.equal(env.CLAUDE_CODE_ENTRYPOINT, undefined);
assert.equal(env.IX_ENDPOINT, arm === 'ix' ? request.environment.endpoint : undefined);
const requestPath = join(run_path, 'request.json'); writeFileSync(requestPath, JSON.stringify(request));
const provision = join(root, 'provision.mjs');
await stage([process.execPath, provision, 'setup', requestPath], { cwd: repo_path, env });
const result = JSON.parse((await stage([process.execPath, provision, 'probe', requestPath], { cwd: repo_path, env })).stdout);
validateProbe(result, request);
const transcript = await stage([process.execPath, join(root, 'agent.mjs'), requestPath], { cwd: repo_path, env });
assert.equal(parseMetrics(transcript.stdout, 'normalized').turns, 1);
const args = JSON.parse(readFileSync(join(run_path, 'native-args.json'), 'utf8'));
assert.ok(args.includes('fixture-model'));
if (family === 'claude') assert.equal(args.includes('--plugin-dir'), arm === 'ix');
if (arm === 'baseline') {
await assert.rejects(execute(['ix', '--version'], { env }), /ENOENT/);
writeFileSync(join(repo_path, 'AGENTS.md'), 'ambient instructions');
await assert.rejects(execute([process.execPath, provision, 'probe', requestPath], { env }), /exited/);
} else {
for (const evidence of result.evidence) assert.equal(sha256(readFileSync(evidence.path)), evidence.sha256);
const files = integrationFiles(request), first = Object.keys(files)[0];
assert.deepEqual(readdirSync(join(run_path, 'hooks.pending')), []); // The probe's hook finished and cleared its marker.
if (family === 'codex') {
// Skills install where the product installer puts them, never into the task checkout.
assert.ok(existsSync(join(home, '.codex', 'skills', 'ix', 'SKILL.md'))); assert.ok(!existsSync(join(repo_path, '.agents')));
const log = join(run_path, 'hidden-skill-probe');
await assert.rejects(execute([process.execPath, provision, 'probe', requestPath], { env: { ...env, FAKE_CODEX_HIDE_SKILLS: '1' }, log }), /exited/);
assert.match(readFileSync(`${log}.stderr`, 'utf8'), /does not list the installed Ix skill/);
}
if (family === 'codex') assert.ok(files[join(home, '.codex', 'config.toml')].endsWith('[mcp_servers.ix-memory]\ncommand = "ix"\nargs = ["mcp"]\nenv = { IX_ENDPOINT = "http://127.0.0.1:8090" }\n'));
if (family === 'claude') {
const hooks = JSON.parse(files[join(run_path, 'plugin', 'hooks', 'hooks.json')]).hooks.PostToolUse[0].hooks[0];
assert.equal(hooks.timeout, 2 * request.environment.seconds + 30); // Outlasts both ix stage deadlines.
}
writeFileSync(first, 'tampered');
await assert.rejects(execute([process.execPath, provision, 'probe', requestPath], { env }), /exited/);
writeFileSync(first, files[first]);
rmSync(join(home, '.ix', 'mapped'));
// Direct assertMapped also covers map failures that return exit 0.
assert.ok(readFileSync(join(run_path, 'hooks.jsonl'), 'utf8').includes('probe'));
}
}
writeFileSync(join(source, 'skills', 'ix', 'SKILL.md'), 'drifted');
const r = join(scratch, 'provision-claude-ix', 'request.json');
await assert.rejects(execute([process.execPath, join(root, 'provision.mjs'), 'probe', r], { env: { ...process.env, PATH: join(scratch, 'provision-claude-ix', 'bin') } }), /exited/);
});
test('manifest pins agree with the hash-verified source export', () => {
const text = JSON.stringify(rows), manifest = importVerified(text);
checkManifestSource(manifest, text);
// The official evaluator picks its parser by extension, so a named export must match its format.
const jsonl = rows.map((row) => JSON.stringify(row)).join('\n');
assert.equal(parseExport(text, 'local/verified.json').length, rows.length);
assert.equal(parseExport(jsonl, 'local/verified.jsonl').length, rows.length);
assert.throws(() => parseExport(jsonl, 'local/verified.json'), /JSONL export must be named \*\.jsonl/);
assert.throws(() => checkManifestSource(manifest, text, 'local/verified.jsonl'), /JSON array export must be named \*\.json/);
assert.throws(() => importVerified(jsonl, { path: 'local/verified.json' }), /\.jsonl/);
for (const field of ['repo', 'base_commit', 'problem_statement']) {
const changed = structuredClone(manifest); changed.tasks[0][field] = 'drifted';
assert.throws(() => checkManifestSource(changed, text), /differs from source/);
}
});
test('native Ix failures retain JSON and LLM error codes', () => {
const metrics = parseMetrics(ndjson([{ type: 'turn.started' }, { type: 'item.completed', item: { id: 'i', type: 'command_execution', command: 'ix stats', exit_code: 1, aggregated_output: 'error code=backend_error message="unavailable"' } }, { type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 1, cached_input_tokens: 0 } }]), 'codex');
assert.deepEqual(metrics.ix_errors_by_code, { backend_error: 1 });
});
test('portable subset retains unit coverage and excludes POSIX integration tests', { skip: process.env.AGENT_EVAL_PORTABLE_ONLY === '1' }, () => {
const env = { ...process.env, AGENT_EVAL_PORTABLE_ONLY: '1' }; delete env.NODE_TEST_CONTEXT;
const output = execFileSync(process.execPath, ['--test', '--test-reporter=tap', fileURLToPath(import.meta.url)], { encoding: 'utf8', timeout: 60_000, env });
assert.match(output, /# fail 0/);
const skipped = Number(output.match(/# skipped (\d+)/)?.[1]);
assert.ok(skipped >= 6, `POSIX integrations ran in portable subset: ${skipped} skipped`);
assert.ok(Number(output.match(/# pass (\d+)/)?.[1]) >= 10);
});
test('partial budget telemetry preserves observations without invented token totals', () => {
const text = ndjson([{ type: 'turn.started' }, { type: 'item.completed', item: { id: 'tool', type: 'command_execution', command: 'rg needle', exit_code: 0 } }]) + '\n{"type":';
assert.throws(() => parseMetrics(text, 'codex'));
const metrics = parseMetrics(text, 'codex', { allowIncomplete: true });
assert.equal(metrics.turns, 1); assert.equal(metrics.search_calls, 1);
assert.equal(metrics.input_tokens, null); assert.equal(metrics.usage_complete, false);
const turnLimit = parseMetrics(ndjson([{ type: 'assistant', message: { id: 'a', content: [] } }, { type: 'result', subtype: 'error_max_turns', is_error: true }]), 'claude', { allowIncomplete: true });
assert.equal(turnLimit.budget_reason, 'turn_budget'); assert.equal(turnLimit.input_tokens, null);
});
test('missing usage keeps resolution denominators and reports complete-pair coverage', () => {
const runs = fixtureRuns();
for (const run of runs.filter((r) => r.instance_id === 'fixture-0' && r.arm === 'baseline')) {
Object.assign(run, { input_tokens: null, output_tokens: null, cached_tokens: null, usage_complete: false, termination: 'time_budget' });
}
const report = summarize(runs, { samples: 1000 });
assert.equal(report.metrics.resolved.coverage.paired_tasks, 2);
assert.equal(report.metrics.input_tokens.coverage.paired_tasks, 1);
assert.equal(report.metrics.input_tokens.coverage.baseline_observed_runs, 2);
assert.deepEqual(report.metrics.input_tokens.mean_difference.ci90, [2, 2]);
assert.equal(report.per_task[0].metrics.input_tokens.difference, null);
for (const run of runs) Object.assign(run, { input_tokens: null, output_tokens: null, cached_tokens: null, usage_complete: false });
const absent = summarize(runs, { samples: 100 });
assert.equal(absent.metrics.input_tokens.mean_difference.ci90, null);
assert.match(markdown(absent), /unavailable/);
});
test('agent deadlines can preserve output while other subprocess deadlines remain strict', async () => {
const result = await execute([process.execPath, '-e', 'console.log("observed"); setInterval(() => {}, 1000)'], { seconds: 3, allowTimeout: true });
assert.equal(result.timed_out, true); assert.match(result.stdout, /observed/);
await assert.rejects(execute([process.execPath, '-e', 'process.exit(7)'], { allowTimeout: true }), /exited 7/);
});
integrationTest('the evaluator runs with the inherited HOME and PATH, not the solver isolation', async () => {
const script = join(scratch, 'env-evaluator.mjs');
writeFileSync(script, `
import { readFileSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
const r = JSON.parse(readFileSync(process.argv[2], 'utf8'));
writeFileSync(join(r.run_path, 'evaluator-env.json'), JSON.stringify({ HOME: process.env.HOME, PATH: process.env.PATH }));
console.log(JSON.stringify({ instance_id: r.task.instance_id, resolved: true, evaluator: 'synthetic' }));
`);
const config = dryConfig(); config.repeats = 1; config.evaluator.command = [process.execPath, script];
const out = join(scratch, 'evaluator-env');
await runBenchmark({ manifestPath: smoke, config, out, dryRun: true });
const runs = readdirSync(out).filter((name) => existsSync(join(out, name, 'evaluator-env.json')));
assert.equal(runs.length, 6);
for (const name of runs) {
assert.deepEqual(JSON.parse(readFileSync(join(out, name, 'evaluator-env.json'), 'utf8')), { HOME: process.env.HOME, PATH: process.env.PATH });
assert.notEqual(JSON.parse(readFileSync(join(out, name, 'agent-request.json'), 'utf8')).home, process.env.HOME, 'the solver home stays private');
}
});
integrationTest('time-budget outcome evaluates its retained solution and continues subsequent jobs', async () => {
const adapter = join(scratch, 'deadline-agent.mjs');
writeFileSync(adapter, `
import {readFileSync,writeFileSync} from 'node:fs';
import {join} from 'node:path';
const r=JSON.parse(readFileSync(process.argv[2],'utf8'));
if(r.arm==='baseline' && r.task.instance_id==='synthetic-one') {
writeFileSync(join(r.repo_path,'solution.txt'),'synthetic solution\\n');
console.log(JSON.stringify({type:'turn',id:'partial'}));
process.stdout.write('{"type":'); setInterval(()=>{},1000);
} else { process.argv.splice(2,0,'agent'); await import(${JSON.stringify(join(root, 'stub.mjs'))}); }
`);
const config = dryConfig(); config.repeats = 1; config.budget.seconds = 1;
config.agent.command = [process.execPath, adapter];
const out = join(scratch, 'time-budget'), report = await runBenchmark({ manifestPath: smoke, config, out, dryRun: true });
const runs = readFileSync(join(out,'runs.jsonl'),'utf8').trim().split('\n').map(JSON.parse);
assert.equal(runs.length, 6);
const limited = runs.find((r) => r.termination === 'time_budget');
assert.equal(limited.resolved, 1); assert.equal(limited.input_tokens, null);
assert.equal(report.metrics.resolved.coverage.paired_tasks, 3);
assert.equal(report.metrics.input_tokens.coverage.paired_tasks, 2);
assert.ok(existsSync(join(out, 'summary.md')));
});
integrationTest('turn-budget overshoots are retained conservatively as unresolved', async () => {
const adapter = join(scratch, 'turn-agent.mjs');
writeFileSync(adapter, `
console.log(JSON.stringify({type:'turn',id:'extra'}));
process.argv.splice(2,0,'agent'); await import(${JSON.stringify(join(root, 'stub.mjs'))});
`);
const config = dryConfig(); config.repeats = 1; config.budget.max_turns = 1;
config.agent.command = [process.execPath, adapter];
const out = join(scratch, 'turn-budget');
await runBenchmark({ manifestPath: smoke, config, out, dryRun: true });
const runs = readFileSync(join(out,'runs.jsonl'),'utf8').trim().split('\n').map(JSON.parse);
assert.equal(runs.length, 6);
assert.ok(runs.every((r) => r.termination === 'turn_budget' && r.resolved === 0 && r.official_resolved === 1));
assert.ok(existsSync(join(out, 'summary.md')));
});
integrationTest('subagent messages do not push a completed Claude run over max_turns', async () => {
const adapter = join(scratch, 'subagent-agent.mjs');
writeFileSync(adapter, `
import {readFileSync,writeFileSync} from 'node:fs'; import {join} from 'node:path';
const r=JSON.parse(readFileSync(process.argv[2],'utf8'));
writeFileSync(join(r.repo_path,'solution.txt'),'synthetic solution\\n');
const events=[{type:'assistant',parent_tool_use_id:null,message:{id:'main-1',content:[{type:'tool_use',id:'task',name:'Task',input:{}}]}}];
for (const id of ['s1','s2','s3','s4']) events.push({type:'assistant',parent_tool_use_id:'task',message:{id,content:[]}});
events.push({type:'assistant',parent_tool_use_id:null,message:{id:'main-2',content:[]}});
events.push({type:'result',usage:{input_tokens:1,output_tokens:1,cache_read_input_tokens:0,cache_creation_input_tokens:0}});
for (const e of events) console.log(JSON.stringify(e));
`);
const config = dryConfig(); config.repeats = 1; config.budget.max_turns = 2; config.agent.telemetry = 'claude';
config.agent.command = [process.execPath, adapter];
const out = join(scratch, 'subagent-turns');
await runBenchmark({ manifestPath: smoke, config, out, dryRun: true });
const runs = readFileSync(join(out,'runs.jsonl'),'utf8').trim().split('\n').map(JSON.parse);
assert.ok(runs.every((r) => r.turns === 2 && r.termination === 'completed' && r.resolved === r.official_resolved));
});
integrationTest('native turn-limit exit receipts remain evaluable and ordinary crashes remain fatal', async () => {
const adapter = join(scratch, 'native-turn-limit.mjs');
writeFileSync(adapter, `
import {readFileSync,writeFileSync} from 'node:fs'; import {join} from 'node:path';
const r=JSON.parse(readFileSync(process.argv[2],'utf8'));
writeFileSync(join(r.repo_path,'solution.txt'),'synthetic solution\\n');
console.log(JSON.stringify({type:'assistant',message:{id:'a',content:[]}}));
console.log(JSON.stringify({type:'result',subtype:'error_max_turns',is_error:true}));
process.exitCode=1;
`);
const config = dryConfig(); config.repeats = 1; config.agent.telemetry = 'claude';
config.agent.command = [process.execPath, adapter];
const out = join(scratch, 'native-turn-limit'), report = await runBenchmark({ manifestPath: smoke, config, out, dryRun: true });
const runs = readFileSync(join(out,'runs.jsonl'),'utf8').trim().split('\n').map(JSON.parse);
assert.equal(runs.length, 6); assert.ok(runs.every((r) => r.termination === 'turn_budget' && r.input_tokens === null));
assert.equal(report.metrics.resolved.baseline.mean, 1);
assert.equal(report.metrics.input_tokens.mean_difference.ci90, null);
const crash = join(scratch, 'agent-crash.mjs'); writeFileSync(crash, 'process.exitCode=7;\n');
config.agent.command = [process.execPath, crash];
const invalid = join(scratch, 'agent-crash');
await assert.rejects(runBenchmark({ manifestPath: smoke, config, out: invalid, dryRun: true }), /without a recognized budget/);
assert.ok(!existsSync(join(invalid,'summary.md')));
});