Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

45.8 KB
Raw Blame
#!/usr/bin/env python3
"""Cleanup safety tests: real temporary Git worktrees, no Rust compilation."""
import importlib.util
import contextlib
import io
import json
import os
from pathlib import Path
import select
import signal
import subprocess
import tempfile
import time
import types
import unittest
from unittest.mock import patch

spec = importlib.util.spec_from_file_location("local_rust", Path(__file__).with_name("local-rust.py"))
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)


class Cleanup(unittest.TestCase):
    def setUp(self):
        self.temp = tempfile.TemporaryDirectory()
        self.addCleanup(self.temp.cleanup)
        self.base = Path(self.temp.name).resolve()
        self.repo = self.base / "repo"
        self.repo.mkdir()
        subprocess.run(["git", "init", "-q", str(self.repo)], check=True)
        m.git(self.repo, "config", "core.hooksPath", "/dev/null")
        m.git(self.repo, "config", "user.name", "Fixture")
        m.git(self.repo, "config", "user.email", "[email protected]")
        (self.repo / "source").write_text("preserve source")
        (self.repo / ".gitignore").write_text("target/\nartifacts/\n")
        m.git(self.repo, "add", ".")
        m.git(self.repo, "commit", "-qm", "fixture")
        self.store = m.Store(self.base / "state", self.base / "global.lock")

    def target(self, checkout):
        debug = checkout / "target/debug"
        debug.mkdir(parents=True)
        (debug / "binary").write_bytes(b"rebuildable")
        evidence = checkout / "target/e2e"
        evidence.mkdir()
        (evidence / "result.json").write_text('{"scenario":"red"}')

    def developer(self):
        self.target(self.repo)
        return self.store.retire(self.repo)

    def managed(self):
        key = m.uuid.uuid4().hex
        checkout = self.store.root / "checkouts" / key
        m.git(self.repo, "worktree", "add", "--detach", str(checkout), "HEAD")
        self.target(checkout)
        row = dict(id=key, kind="managed", repo=str(self.repo), checkout=str(checkout),
                   identity=m.identity(checkout), sha=m.git(checkout, "rev-parse", "HEAD"),
                   status="terminal", scenario="red", process_exit=1, keep=False,
                   evidence_saved=True, finished_at=0,
                   target_identity=m.identity(checkout / "target"),
                   debug_identity=m.identity(checkout / "target/debug"))
        self.store.save(row)
        return row

    def test_preview_is_non_destructive(self):
        row = self.developer()
        before = self.store.record_path(row['id']).read_bytes()
        self.assertEqual(self.store.preview()[0]['state'], 'candidate')
        self.assertEqual(before, self.store.record_path(row['id']).read_bytes())
        self.assertTrue((self.repo / 'target/debug/binary').exists())

    def targetless_admission(self):
        row = self.managed()
        m.shutil.rmtree(Path(row['checkout']) / 'target')
        row.pop('target_identity')
        row.pop('debug_identity')
        row['process_exit'] = None
        self.store.save(row)
        return row

    def test_targetless_admission_cleanup_preserves_run_evidence(self):
        row = self.targetless_admission()
        report = self.store.record_path(row['id']).parent / 'validation-report.json'
        report.write_text('{"scenario":"not_executed"}')
        candidate = self.store.preview_one(row)
        self.assertEqual(candidate['state'], 'candidate')
        self.assertEqual(candidate['bytes'], 0)
        self.store.apply(row['id'], candidate['token'])
        self.assertFalse(Path(row['checkout']).exists())
        self.assertTrue(self.store.load(row['id'])['removed'])
        self.assertIsNone(self.store.load(row['id'])['process_exit'])
        self.assertEqual(json.loads(report.read_text())['scenario'], 'not_executed')

    def test_targetless_admission_rejects_later_target_and_dangling_link(self):
        row = self.targetless_admission()
        candidate = self.store.preview_one(row)
        target = Path(row['checkout']) / 'target'
        for linked in (False, True):
            if linked:
                target.symlink_to(self.base / 'missing', target_is_directory=True)
            else:
                target.mkdir()
            with self.assertRaises((ValueError, KeyError)):
                self.store.apply(row['id'], candidate['token'])
            if linked:
                target.unlink()
            else:
                target.rmdir()
        self.assertTrue(Path(row['checkout']).exists())

    def test_missing_target_after_started_or_attested_run_stays_protected(self):
        row = self.targetless_admission()
        for extra in ({'command_started_at': 1}, {'process_exit': 1},
                      {'target_identity': [1, 2]}, {'debug_identity': [1, 3]}):
            self.assertEqual(self.store.preview_one(dict(row, **extra))['state'], 'skipped')

    def test_developer_cleanup_preserves_dirty_source_evidence_and_branch(self):
        row = self.developer()
        (self.repo / 'source').write_text('uncommitted')
        (self.repo / 'new-file').write_text('untracked')
        candidate = self.store.preview_one(row)
        self.store.apply(row['id'], candidate['token'])
        self.assertFalse((self.repo / 'target/debug').exists())
        self.assertEqual((self.repo / 'source').read_text(), 'uncommitted')
        self.assertTrue((self.repo / 'new-file').exists())
        self.assertTrue((self.repo / 'target/e2e/result.json').exists())
        self.assertEqual(m.git(self.repo, 'rev-parse', 'HEAD'), row['sha'])

    def test_activity_lock_blocks_preview_and_apply(self):
        row = self.developer()
        token = self.store.preview_one(row)['token']
        with self.store.locked(self.repo):
            self.assertEqual(self.store.preview_one(row)['state'], 'skipped')
            with self.assertRaises(BlockingIOError):
                self.store.apply(row['id'], token)

    def test_child_holding_global_lock_blocks_cleanup(self):
        row = self.developer()
        code = 'import fcntl,sys; f=open(sys.argv[1],"a"); fcntl.flock(f,fcntl.LOCK_EX); print("ready",flush=True); sys.stdin.read()'
        with subprocess.Popen([m.sys.executable, '-c', code, str(self.store.global_lock)],
                              stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) as child:
            self.assertEqual(child.stdout.readline().strip(), 'ready')
            self.assertEqual(self.store.preview_one(row)['state'], 'skipped')
            child.communicate('done')
            self.assertEqual(child.returncode, 0)

    def child(self, code, *arguments, env=None):
        process = subprocess.Popen([m.sys.executable, '-c', code, *map(str, arguments)],
                                   stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE, text=True, env=env)
        def reap():
            if process.poll() is None:
                process.kill()
            process.communicate(timeout=5)
        self.addCleanup(reap)
        return process

    def ready(self, process):
        self.assertTrue(select.select([process.stdout], [], [], 5)[0], 'child readiness timed out')
        # Read only the readiness line; TextIO.readline could prefetch the final
        # JSON that communicate() must later read from a fast nonblocking child.
        line = b''
        while not line.endswith(b'\n'):
            value = os.read(process.stdout.fileno(), 1)
            self.assertTrue(value, 'child exited before readiness')
            line += value
        self.assertEqual(line, b'ready\n')

    def hold_lock(self, path):
        child = self.child('import fcntl,sys; f=open(sys.argv[1],"a"); '
                           'fcntl.flock(f,fcntl.LOCK_EX); print("ready",flush=True); sys.stdin.read()', path)
        self.ready(child)
        return child

    def queued_run(self, wait_for_lock=None, available_bytes=60 * m.GIB):
        # Only Git, child processes and flock are real; these private compiler tools
        # record invocations instead of compiling Rust or touching the host's locks.
        binary = self.base / 'queued-bin'
        binary.mkdir()
        calls = self.base / 'compiler-calls.jsonl'
        cargo = binary / 'cargo'
        cargo.write_text('#!' + m.sys.executable + '\n'
                         'import json,pathlib,sys\n'
                         f'with pathlib.Path({str(calls)!r}).open("a") as out:\n'
                         ' out.write(json.dumps(sys.argv[1:])+"\\n")\n'
                         'target=pathlib.Path.cwd()/"target"\n'
                         'if sys.argv[1]=="metadata":\n'
                         ' print(json.dumps({"target_directory":str(target)}))\n'
                         'elif sys.argv[1]=="--version":\n'
                         ' print("cargo fixture")\n'
                         'else:\n'
                         ' (target/"debug").mkdir(parents=True)\n'
                         ' print("Running tests/fixture.rs (target/debug/deps/fixture-abcdef0123456789)")\n'
                         ' print("running 1 test")\n'
                         ' print("test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out")\n')
        cargo.chmod(0o700)
        for name in ('rustc', 'sccache'):
            tool = binary / name
            tool.write_text('#!/bin/sh\nexit 0\n')
            tool.chmod(0o700)
        space = self.base / 'available-bytes'
        space.write_text(str(available_bytes))
        args = dict(repo=str(self.repo), ref=m.git(self.repo, 'symbolic-ref', 'HEAD'),
                    sha=m.git(self.repo, 'rev-parse', 'HEAD'), command=['cargo', 'test'],
                    keep=True, cargo_only=True, timeout=5, source='local', jobs=2)
        if wait_for_lock is not None:
            args['wait_for_lock'] = wait_for_lock
        code = '''import importlib.util,json,sys,types
from pathlib import Path
sys.dont_write_bytecode=True
spec=importlib.util.spec_from_file_location('queued_local',sys.argv[1])
module=importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
store=module.Store(sys.argv[2],sys.argv[3])
store.ensure_space=lambda: None
store.gc=lambda *a,**k: (_ for _ in ()).throw(AssertionError('GC must not run inside the acquired lock'))
module.remote.stats=lambda *a,**k: {'cache_hits':1}
def cache_admission(*args,**kwargs):
 with Path(sys.argv[4]).with_name('cache-admitted').open('w') as out: out.write('entered')
 return {'state':'ready'}
module.remote.admit_cache_server=cache_admission
module.shutil.disk_usage=lambda path: types.SimpleNamespace(free=int(Path(sys.argv[4]).read_text()))
flock=module.remote.fcntl.flock
first=True
def observed_flock(handle,flags):
 global first
 if first:
  first=False
  print('ready',flush=True)
 return flock(handle,flags)
module.remote.fcntl.flock=observed_flock
result=module.run(store,types.SimpleNamespace(**json.loads(sys.argv[5])))
print(json.dumps(result),flush=True)
'''
        env = dict(os.environ, PATH=str(binary) + os.pathsep + os.environ['PATH'])
        child = self.child(code, m.__file__, self.store.root, self.store.global_lock,
                           space, json.dumps(args), env=env)
        self.ready(child)
        return child, calls, space

    def run_result(self, child):
        output, error = child.communicate(timeout=5)
        self.assertEqual(child.returncode, 0, error)
        row = json.loads(output)
        if row.get('temporary_directory'):
            self.addCleanup(m.remote.cleanup_temp_directory, row['temporary_directory'])
        return row

    def waiting_record(self, child, calls, require_queue=True):
        with self.assertRaises(subprocess.TimeoutExpired):
            child.wait(timeout=0.15)
        self.assertFalse(calls.exists(), 'compiler tools entered before capacity was acquired')
        records = list((self.store.root / 'runs').glob('*/result.json'))
        self.assertEqual(len(records), 1)
        row = json.loads(records[0].read_text())
        self.assertTrue(row['wait_for_lock'])
        if require_queue:
            self.assertIn('queued_at', row)
            self.assertLessEqual(row['checkout_finished_at'], row['queued_at'])
        self.assertNotIn('capacity_acquired_at', row)
        self.assertNotIn('command_started_at', row)
        return row

    def test_waiting_run_executes_once_after_child_releases_lock(self):
        holder = self.hold_lock(self.store.global_lock)
        child, calls, _ = self.queued_run(wait_for_lock=True, available_bytes=30 * m.GIB)
        queued = self.waiting_record(child, calls)
        released_at = time.time()
        holder.communicate('release', timeout=5)
        row = self.run_result(child)
        self.assertEqual(row['scenario'], 'green', row)
        self.assertEqual(row['process_exit'], 0)
        self.assertEqual(row['queued_at'], queued['queued_at'])
        self.assertGreaterEqual(row['capacity_acquired_at'], released_at)
        self.assertLessEqual(row['capacity_acquired_at'], row['command_started_at'])
        self.assertLessEqual(row['command_started_at'], row['command_finished_at'])
        self.assertEqual([json.loads(line) for line in calls.read_text().splitlines()].count(['test']), 1)
        saved = self.store.load(row['id'])
        self.assertTrue(saved['wait_for_lock'])
        self.assertEqual(saved['queued_at'], row['queued_at'])
        with self.store.locked(row['checkout']):
            pass

    def test_default_run_refuses_busy_lock_without_execution(self):
        holder = self.hold_lock(self.store.global_lock)
        child, calls, _ = self.queued_run()
        row = self.run_result(child)
        self.assertFalse(row['wait_for_lock'])
        self.assertEqual(row['scenario'], 'red')
        self.assertIsNone(row['process_exit'])
        self.assertIn('temporarily unavailable', row['error'])
        self.assertNotIn('capacity_acquired_at', row)
        self.assertFalse(calls.exists())
        self.assertFalse((self.base / 'cache-admitted').exists())
        self.assertIsNone(holder.poll(), 'the test must leave the competing owner active')

    def test_interrupted_waiting_run_has_no_execution_or_lock_leak(self):
        holder = self.hold_lock(self.store.global_lock)
        child, calls, _ = self.queued_run(wait_for_lock=True)
        self.waiting_record(child, calls, require_queue=False)
        child.send_signal(signal.SIGINT)
        row = self.run_result(child)
        self.assertEqual(row['scenario'], 'red')
        self.assertIsNone(row['process_exit'])
        self.assertNotIn('capacity_acquired_at', row)
        self.assertFalse(calls.exists())
        holder.communicate('release', timeout=5)
        with self.store.locked(row['checkout']):
            pass

    def waiting_store(self):
        checkout_lock = self.store.root / 'locks' / (m.hashlib.sha256(str(self.repo).encode()).hexdigest() + '.lock')
        holder = self.hold_lock(checkout_lock)
        code = '''import importlib.util,sys
spec=importlib.util.spec_from_file_location('waiting_local',sys.argv[1])
module=importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
store=module.Store(sys.argv[2],sys.argv[3])
flock=module.remote.fcntl.flock
count=0
def observed_flock(handle,flags):
 global count
 count+=1
 if count==2: print('ready',flush=True)
 return flock(handle,flags)
module.remote.fcntl.flock=observed_flock
with store.locked(sys.argv[4],wait=True): print('entered',flush=True)
'''
        child = self.child(code, m.__file__, self.store.root, self.store.global_lock, self.repo)
        self.ready(child)
        return holder, child

    def test_waiting_store_enters_once_after_checkout_child_releases_lock(self):
        holder, child = self.waiting_store()
        with self.assertRaises(subprocess.TimeoutExpired):
            child.wait(timeout=0.15)
        holder.communicate('release', timeout=5)
        output, error = child.communicate(timeout=5)
        self.assertEqual(child.returncode, 0, error)
        self.assertEqual(output.splitlines(), ['entered'])
        with self.store.locked(self.repo):
            pass

    def test_waiting_store_holds_global_until_checkout_and_releases_on_termination(self):
        holder, child = self.waiting_store()
        with self.assertRaises(subprocess.TimeoutExpired):
            child.wait(timeout=0.15)
        with self.assertRaises(BlockingIOError):
            with self.store.locked(self.base / 'other-checkout'):
                self.fail('global lock was released while waiting on the checkout lock')
        child.terminate()
        output, _ = child.communicate(timeout=5)
        self.assertEqual(child.returncode, -signal.SIGTERM)
        self.assertNotIn('entered', output)
        with self.store.locked(self.base / 'other-checkout'):
            pass
        holder.communicate('release', timeout=5)
        with self.store.locked(self.repo):
            pass

    def test_cleanup_still_refuses_busy_child_lock_without_waiting(self):
        row = self.developer()
        token = self.store.preview_one(row)['token']
        holder = self.hold_lock(self.store.global_lock)
        code = '''import importlib.util,json,sys
spec=importlib.util.spec_from_file_location('cleanup_local',sys.argv[1])
module=importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
store=module.Store(sys.argv[2],sys.argv[3])
row=store.load(sys.argv[4])
preview=store.preview_one(row)
try: store.apply(row['id'],sys.argv[5])
except BlockingIOError: rejected=True
else: rejected=False
print(json.dumps({'preview':preview['state'],'apply_rejected':rejected}))
'''
        child = self.child(code, m.__file__, self.store.root, self.store.global_lock, row['id'], token)
        output, error = child.communicate(timeout=2)
        self.assertEqual(child.returncode, 0, error)
        self.assertEqual(json.loads(output), {'preview': 'skipped', 'apply_rejected': True})
        self.assertIsNone(holder.poll())
        self.assertTrue((self.repo / 'target/debug/binary').exists())

    def test_waiting_run_rechecks_capacity_without_gc_before_compiler(self):
        holder = self.hold_lock(self.store.global_lock)
        child, calls, space = self.queued_run(wait_for_lock=True)
        self.waiting_record(child, calls, require_queue=False)
        space.write_text(str(30 * m.GIB - 1))
        holder.communicate('release', timeout=5)
        row = self.run_result(child)
        self.assertEqual(row['scenario'], 'red')
        self.assertIn('Less than 30 GiB free after waiting', row['error'])
        self.assertIsNone(row['process_exit'])
        self.assertIn('capacity_acquired_at', row)
        self.assertNotIn('command_started_at', row)
        self.assertNotIn('temporary_directory', row)
        self.assertFalse(calls.exists())
        with self.store.locked(row['checkout']):
            pass

    def test_default_run_does_not_add_a_second_capacity_check(self):
        # Preflight is stubbed in this compiler fixture. Only wait mode adds the
        # new capacity check after flock; the original default stays compatible.
        child, calls, _ = self.queued_run(available_bytes=0)
        row = self.run_result(child)
        self.assertFalse(row['wait_for_lock'])
        self.assertEqual(row['scenario'], 'green', row)
        self.assertTrue(calls.exists())

    def test_local_cli_parses_explicit_wait_and_preserves_default(self):
        for enabled in (False, True):
            argv = ['local-rust.py', 'run', '--ref', 'refs/heads/test', '--sha', 'a' * 40]
            if enabled:
                argv.append('--wait-for-lock')
            argv += ['--', 'cargo', 'test']
            with self.subTest(wait=enabled), patch.object(m.sys, 'argv', argv), \
                    patch.object(m, 'Store', return_value=self.store), \
                    patch.object(m, 'run', return_value={'scenario': 'green'}) as run, \
                    patch('builtins.print'):
                self.assertEqual(m.main(), 0)
                self.assertEqual(run.call_args.args[1].wait_for_lock, enabled)

    def test_keep_and_missing_evidence_are_protected(self):
        row = self.managed()
        for field, value in [('keep', True), ('evidence_saved', False), ('status', 'running')]:
            changed = dict(row, **{field: value})
            self.assertEqual(self.store.preview_one(changed)['state'], 'skipped')

    def test_snapshot_changes_reject_apply(self):
        row = self.developer()
        token = self.store.preview_one(row)['token']
        (self.repo / 'target/debug/binary').write_text('changed contents after preview')
        with self.assertRaisesRegex(ValueError, 'Candidate changed'):
            self.store.apply(row['id'], token)

    def test_replaced_target_or_symlink_is_rejected(self):
        row = self.developer()
        token = self.store.preview_one(row)['token']
        original = self.repo / 'target'
        original.rename(self.repo / 'saved')
        original.symlink_to(self.repo / 'saved', target_is_directory=True)
        with self.assertRaises(ValueError):
            self.store.apply(row['id'], token)
        self.assertTrue((self.repo / 'saved/debug/binary').exists())

    def test_debug_inner_symlink_never_deletes_external_target(self):
        row = self.developer()
        outside = self.base / 'outside'
        outside.mkdir()
        (outside / 'precious').write_text('keep')
        (self.repo / 'target/debug/link').symlink_to(outside, target_is_directory=True)
        self.store.apply(row['id'], self.store.preview_one(row)['token'])
        self.assertTrue((outside / 'precious').exists())

    def test_git_tracked_target_is_rejected(self):
        self.target(self.repo)
        m.git(self.repo, 'add', '-f', 'target/debug/binary')
        with self.assertRaisesRegex(ValueError, 'tracked'):
            self.store.retire(self.repo)

    def test_gc_ages_only_managed_and_keep_protects(self):
        dev = self.developer()
        row = self.managed()
        row['finished_at'] = time.time()
        self.store.save(row)
        self.assertEqual(self.store.gc(), [])
        row.update(finished_at=0, keep=True)
        self.store.save(row)
        self.assertEqual(self.store.gc(apply=True)[0]['state'], 'skipped')
        self.assertTrue((self.repo / 'target/debug').exists())
        row['keep'] = False
        self.store.save(row)
        self.assertEqual(self.store.gc(apply=True)[0]['state'], 'removed')
        self.assertTrue((self.repo / 'target/debug').exists())
        evidence = self.store.record_path(row['id']).parent / 'artifacts/e2e/result.json'
        self.assertEqual(json.loads(evidence.read_text())['scenario'], 'red')
        self.assertFalse(Path(row['checkout']).exists())
        self.assertFalse(self.store.load(dev['id']).get('removed', False))

    def test_managed_source_changes_block_cleanup(self):
        row = self.managed()
        (Path(row['checkout']) / 'source').write_text('do not lose')
        self.assertEqual(self.store.preview_one(row)['state'], 'skipped')

    def test_artifact_links_retain_checkout(self):
        row = self.managed()
        (Path(row['checkout']) / 'target/e2e/link').symlink_to(self.repo / 'source')
        with self.assertRaisesRegex(ValueError, 'Artifact contains links'):
            self.store.apply(row['id'], self.store.preview_one(row)['token'])
        self.assertTrue(Path(row['checkout']).exists())

    def test_pressure_never_removes_developer_target(self):
        self.developer()
        low = m.shutil._ntuple_diskusage(100 * m.GIB, 99 * m.GIB, m.GIB)
        with patch.object(m.shutil, 'disk_usage', return_value=low):
            with self.assertRaisesRegex(ValueError, 'build not started'):
                self.store.ensure_space()
        self.assertTrue((self.repo / 'target/debug/binary').exists())

    def test_unloaded_toolchain_refuses_before_record_worktree_or_space_check(self):
        empty = self.base / 'empty-bin'
        empty.mkdir()
        args = types.SimpleNamespace(repo=str(self.repo), ref=m.git(self.repo, 'symbolic-ref', 'HEAD'),
                                     sha=m.git(self.repo, 'rev-parse', 'HEAD'),
                                     command=['cargo', 'test'], keep=False, cargo_only=True, timeout=10)
        with patch.dict(os.environ, {'PATH': str(empty)}), \
             patch.object(m, 'TOOLCHAIN_BIN', empty), \
             patch.object(self.store, 'ensure_space') as space:
            with self.assertRaisesRegex(ValueError, 'Missing on PATH: cargo, rustc, sccache, git'):
                m.run(self.store, args)
            space.assert_not_called()
        self.assertEqual(list((self.store.root / 'runs').iterdir()), [])
        self.assertEqual(list((self.store.root / 'checkouts').iterdir()), [])

    def test_toolchain_lookup_appends_rustup_bin_after_loaded_path(self):
        loaded = self.base / 'loaded-bin'
        loaded.mkdir()
        fallback = self.base / 'rustup-bin'
        fallback.mkdir()
        for name in ('cargo', 'rustc', 'sccache', 'git'):
            tool = fallback / name
            tool.write_text('#!/bin/sh\n')
            tool.chmod(0o700)
        with patch.dict(os.environ, {'PATH': str(loaded)}), \
             patch.object(m, 'TOOLCHAIN_BIN', fallback):
            self.assertEqual(m.toolchain_environment(),
                             os.pathsep.join([str(loaded), str(fallback)]))

    def run_fixture(self, exit_code=0, cargo_only=True, delay=0, source="origin", jobs=2, command=None,
                    keep=False, profile='debug', cache_error=None, no_target=False, prepare=None):
        # Real git origin/worktree and subprocess lifecycle; fake compiler means NO Rust proof.
        m.git(self.repo, 'remote', 'add', 'origin', str(self.base / 'unpublished') if source == 'local' else str(self.repo))
        branch = m.git(self.repo, 'symbolic-ref', 'HEAD')
        binary = self.base / 'bin'
        binary.mkdir()
        cargo = binary / 'cargo'
        cargo.write_text('#!' + m.sys.executable + '\n'
                         'import sys,json,pathlib,os\n'
                         'p=pathlib.Path.cwd()/"target"\n'
                         'if sys.argv[1]=="metadata":\n'
                         ' print(json.dumps({"target_directory":str(p)}))\n'
                         'elif sys.argv[1]=="--version":\n'
                         ' print("cargo fixture")\n'
                         'else:\n'
                         + ('' if no_target else
                         ' (p/' + repr(profile) + ').mkdir(parents=True)\n'
                         ' (p/' + repr(profile) + '/"binary").write_text("rebuildable fixture")\n'
                         ' (p/"e2e").mkdir()\n'
                         ' (p/"e2e/result.json").write_text(\'{"observed":"fixture"}\')\n') +
                         ' (pathlib.Path.cwd()/"artifacts").mkdir()\n'
                         ' (pathlib.Path.cwd()/"artifacts/unique.json").write_text("checkout evidence")\n'
                         ' (pathlib.Path(os.environ["TMPDIR"])/"diagnostic.log").write_text("private runtime evidence")\n'
                         ' import time; time.sleep(' + str(delay) + ')\n'
                         ' print("Running tests/fixture.rs (target/debug/deps/fixture-abcdef0123456789)")\n'
                         ' print("running 1 test")\n'
                         ' print("test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out")\n'
                         ' sys.exit(' + str(exit_code) + ')\n')
        cargo.chmod(0o700)
        for name in ('rustc', 'sccache'):
            tool = binary / name
            tool.write_text('#!/bin/sh\nexit 0\n')
            tool.chmod(0o700)
        args = types.SimpleNamespace(repo=str(self.repo), ref=branch,
                                     sha=m.git(self.repo, 'rev-parse', 'HEAD'),
                                     command=command or ['cargo', 'test'], keep=keep,
                                     cargo_only=cargo_only, timeout=1 if delay else 10,
                                     source=source, jobs=jobs)
        with patch.dict(os.environ, {'PATH': str(binary) + os.pathsep + os.environ['PATH']}), \
             patch.object(m.remote, 'stats', return_value={'cache_hits': 1}), \
             patch.object(m.remote, 'admit_cache_server', side_effect=cache_error, return_value={'state': 'ready'}), \
             patch.object(self.store, 'ensure_space'):
            result = m.run(self.store, args, prepare=prepare)
        if result.get('temporary_directory'):
            self.addCleanup(m.remote.cleanup_temp_directory, result['temporary_directory'])
        return result

    def test_workspace_default_failure_policy_is_preserved(self):
        command = ['cargo', 'test', '--workspace', '--exclude', 'hiroute-desktop']
        result = self.run_fixture(command=command)
        self.assertEqual(result['process_exit'], 0)
        self.assertEqual(result['command'], command)

    def test_explicit_collect_all_failure_policy_is_preserved(self):
        command = ['cargo', 'test', '--no-fail-fast', '--workspace', '--exclude', 'hiroute-desktop']
        result = self.run_fixture(command=command)
        self.assertEqual(result['process_exit'], 0)
        self.assertEqual(result['command'], command)

    def test_local_candidate_needs_no_published_origin_and_preserves_dirty_tree(self):
        (self.repo / 'source').write_text('uncommitted source must stay outside build')
        result = self.run_fixture(source='local', cargo_only=False, jobs=3)
        self.assertEqual(result['process_exit'], 0)
        self.assertEqual(result['scenario'], 'unassessed')
        self.assertEqual(result['candidate_source'], 'local')
        self.assertEqual(result['build_jobs'], 3)
        self.assertFalse(Path(result['checkout']).exists())
        self.assertEqual((self.repo / 'source').read_text(), 'uncommitted source must stay outside build')

    def test_managed_cargo_run_keeps_feedback_after_automatic_cleanup(self):
        result = self.run_fixture(source='local')
        self.assertEqual(result['process_exit'], 0)
        self.assertFalse(Path(result['checkout']).exists())
        report = json.loads(Path(result['validation_report']['path']).read_text())
        self.assertEqual(report['sha'], result['sha'])
        self.assertEqual(report['process_exit'], 0)
        self.assertTrue(report['complete'])
        self.assertEqual(report['modules'][0]['counts'], dict(passed=1, failed=0, ignored=0))
        self.assertIsNotNone(report['timing']['command_seconds'])

    def test_invalid_jobs_refuses_before_run_record(self):
        with self.assertRaisesRegex(ValueError, 'positive build jobs'):
            self.run_fixture(jobs=0)
        self.assertEqual(list((self.store.root / 'runs').iterdir()), [])

    def test_managed_run_success_exports_log_and_removes_checkout(self):
        result = self.run_fixture()
        self.assertEqual(result['process_exit'], 0)
        self.assertEqual(result['scenario'], 'green')
        self.assertTrue(result['removed'])
        self.assertTrue((self.store.record_path(result['id']).parent / 'command.log').exists())
        self.assertFalse(Path(result['checkout']).exists())

    def test_managed_run_failure_exports_evidence_then_removes_checkout_and_temp(self):
        result = self.run_fixture(exit_code=4)
        self.assertEqual(result['process_exit'], 4)
        self.assertEqual(result['scenario'], 'red')
        self.assertFalse(Path(result['checkout']).exists())
        self.assertFalse(Path(result['temporary_directory']['path']).exists())
        run = self.store.record_path(result['id']).parent
        self.assertEqual((run / 'artifacts/e2e/result.json').read_text(), '{"observed":"fixture"}')
        self.assertEqual((run / 'evidence/temporary/diagnostic.log').read_text(), 'private runtime evidence')
        self.assertFalse((run / 'artifacts/debug').exists())
        self.assertEqual(json.loads((run / 'validation-report.json').read_text())['process_exit'], 4)
        self.assertEqual(result['evidence_export']['state'], 'saved')

    def test_zero_exit_without_scenario_assessment_cleans_without_claiming_green(self):
        result = self.run_fixture(cargo_only=False)
        self.assertEqual(result['process_exit'], 0)
        self.assertEqual(result['scenario'], 'unassessed')
        self.assertTrue(result['evidence_saved'])
        self.assertFalse(Path(result['checkout']).exists())

    def test_timeout_reaps_child_before_cleanup_and_preserves_failure_evidence(self):
        result = self.run_fixture(delay=30)
        self.assertEqual(result['process_exit'], -9)
        self.assertEqual(result['scenario'], 'red')
        self.assertFalse(Path(result['checkout']).exists())
        self.assertFalse(Path(result['temporary_directory']['path']).exists())
        self.assertEqual((self.store.record_path(result['id']).parent / 'evidence/temporary/diagnostic.log').read_text(),
                         'private runtime evidence')
        with self.store.locked(result['checkout']):
            pass

    def test_explicit_keep_retains_failed_run_for_diagnosis(self):
        result = self.run_fixture(exit_code=101, keep=True)
        self.assertEqual((result['scenario'], result['process_exit']), ('red', 101))
        self.assertTrue(Path(result['checkout']).exists())
        self.assertTrue(Path(result['temporary_directory']['path']).exists())

    def test_kept_release_run_exports_all_evidence_when_the_owner_finishes_and_releases(self):
        result = self.run_fixture(exit_code=101, keep=True, profile='release',
                                  command=['cargo', 'test', '--release'])
        for arguments in (['finish', result['id'], '--verdict', 'expected_red', '--evidence-saved'],
                          ['release', result['id']]):
            with patch.object(m.sys, 'argv', ['local-rust.py', *arguments]), \
                 patch.object(m, 'Store', return_value=self.store), contextlib.redirect_stdout(io.StringIO()):
                self.assertEqual(m.main(), 0)
        row = self.store.load(result['id'])
        candidate = self.store.preview_one(row)
        self.assertEqual(candidate['state'], 'candidate', candidate)
        self.store.apply(row['id'], candidate['token'])
        saved = self.store.load(row['id'])
        run = self.store.record_path(row['id']).parent
        self.assertEqual((saved['scenario'], saved['process_exit']), ('expected_red', 101))
        self.assertFalse(Path(row['checkout']).exists())
        self.assertFalse(Path(row['temporary_directory']['path']).exists())
        self.assertEqual(saved.get('evidence_export', {}).get('state'), 'saved')
        self.assertTrue((run / 'evidence.json').is_file())
        self.assertEqual((run / 'artifacts/e2e/result.json').read_text(), '{"observed":"fixture"}')
        self.assertEqual((run / 'evidence/artifacts/unique.json').read_text(), 'checkout evidence')
        self.assertEqual((run / 'evidence/temporary/diagnostic.log').read_text(), 'private runtime evidence')
        self.assertFalse((run / 'artifacts/release').exists())

    def assert_targetless_cleanup(self, command, exit_code):
        result = self.run_fixture(command=command, exit_code=exit_code, no_target=True)
        self.assertEqual(result['process_exit'], exit_code)
        self.assertIn('command_started_at', result)
        self.assertTrue(result.get('removed', False), result.get('cleanup'))
        self.assertFalse(Path(result['checkout']).exists())
        self.assertFalse(Path(result['temporary_directory']['path']).exists())
        run = self.store.record_path(result['id']).parent
        self.assertEqual((run / 'evidence/temporary/diagnostic.log').read_text(), 'private runtime evidence')

    def test_targetless_format_check_is_reclaimed(self):
        self.assert_targetless_cleanup(['cargo', 'fmt', '--check'], 0)

    def test_targetless_package_selection_failure_is_reclaimed(self):
        self.assert_targetless_cleanup(['cargo', 'test', '-p', 'no-such-package'], 101)

    def test_inherited_wasi_target_is_recorded_and_compiler_output_is_reclaimed(self):
        with patch.dict(os.environ, CARGO_BUILD_TARGET='wasm32-wasip1'):
            result = self.run_fixture(profile='wasm32-wasip1/debug', command=['cargo', 'check'])
        self.assertEqual(result.get('cargo_build_target'), 'wasm32-wasip1')
        self.assertTrue(result.get('removed', False))
        run = self.store.record_path(result['id']).parent
        self.assertFalse((run / 'artifacts/wasm32-wasip1/debug').exists())
        self.assertTrue((run / 'artifacts/e2e/result.json').is_file())

    def test_report_and_cleanup_finish_before_releasing_the_original_compiler_lease(self):
        reporter = m.remote.reporting()
        original = reporter.write_report
        probes = []
        def write_report(*args, **kwargs):
            probe = subprocess.run([m.sys.executable, '-c',
                'import fcntl,sys\nf=open(sys.argv[1],"a")\n'
                'try: fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB)\n'
                'except BlockingIOError: sys.exit(11)\n', str(self.store.global_lock)])
            probes.append(probe.returncode)
            return original(*args, **kwargs)
        with patch.object(m.remote, 'reporting', return_value=reporter), \
             patch.object(reporter, 'write_report', side_effect=write_report):
            result = self.run_fixture()
        self.assertTrue(result['removed'])
        self.assertEqual(probes, [11])

    def test_precommand_cache_block_is_cleaned_without_fabricating_execution(self):
        result = self.run_fixture(cache_error=ValueError('fixture cache admission failed'))
        self.assertIsNone(result['process_exit'])
        self.assertNotIn('command_started_at', result)
        self.assertEqual(result['scenario'], 'red')
        self.assertFalse(Path(result['checkout']).exists())
        self.assertEqual(result['evidence_export']['state'], 'saved')

    def test_export_failure_retains_resources_and_the_actual_test_failure(self):
        with patch.object(m.tempfile, 'NamedTemporaryFile', side_effect=OSError('fixture export failed')):
            result = self.run_fixture(exit_code=101)
        self.assertEqual((result['scenario'], result['process_exit']), ('red', 101))
        self.assertTrue(Path(result['checkout']).exists())
        self.assertTrue(Path(result['temporary_directory']['path']).exists())
        self.assertFalse(result['evidence_saved'])
        self.assertEqual(result['cleanup']['state'], 'retained')
        self.assertIn('fixture export failed', result['cleanup']['reason'])

    def test_link_profile_marker_retains_the_checkout_instead_of_losing_unique_evidence(self):
        def prepare(checkout, *_args):
            evidence = checkout / 'target/unique-evidence'
            evidence.mkdir(parents=True)
            (evidence / 'diagnostic.json').write_text('unique runtime evidence')
            (evidence / '.cargo-lock').symlink_to(self.repo / 'source')
        result = self.run_fixture(command=['cargo', 'check'], prepare=prepare)
        self.assertEqual((result['scenario'], result['process_exit']), ('green', 0))
        self.assertEqual(result['cleanup']['state'], 'retained')
        self.assertEqual(result['evidence_export']['state'], 'failed')
        self.assertIn('link or special file', result['evidence_export']['error'])
        self.assertTrue(Path(result['temporary_directory']['path']).is_dir())
        self.assertEqual((Path(result['checkout']) / 'target/unique-evidence/diagnostic.json').read_text(),
                         'unique runtime evidence')
        self.assertEqual((self.repo / 'source').read_text(), 'preserve source')

    def test_release_only_target_is_reclaimed_without_copying_compiler_outputs(self):
        result = self.run_fixture(profile='release', command=['cargo', 'build', '--release'])
        self.assertEqual(result['process_exit'], 0)
        self.assertTrue(result.get('removed', False))
        self.assertFalse(Path(result['checkout']).exists())
        artifacts = self.store.record_path(result['id']).parent / 'artifacts'
        self.assertFalse((artifacts / 'release').exists())
        self.assertTrue((artifacts / 'e2e/result.json').is_file())

    def test_short_canonical_temporary_directory(self):
        info = m.private_temp()
        self.addCleanup(m.remote.cleanup_temp_directory, info)
        p = Path(info['path'])
        self.assertEqual(p.resolve(), p)
        self.assertEqual(p.stat().st_mode & 0o777, 0o700)
        self.assertLessEqual(len(os.fsencode(p)), m.remote.MAX_TMPDIR_BYTES)

    @unittest.skipUnless(os.environ.get('HIROUTE_REAL_CARGO_TEST') == '1',
                         'Opt-in tiny platform-runner integration, not HiRoute product tests')
    def test_real_cargo_pushed_fixture_lifecycle(self):
        # Own a short endpoint/scratch/cache: never touch the daily server. macOS
        # daemon-start notification sockets also need room below Unix SUN_LEN.
        endpoint = tempfile.TemporaryDirectory(prefix='h202-', dir='/tmp')
        self.addCleanup(endpoint.cleanup)
        self.store = m.Store(Path(endpoint.name) / 'state', m.HOST_LOCK)
        if os.environ.get('HIROUTE_RUNNER_EVIDENCE'):
            destination = Path(os.environ['HIROUTE_RUNNER_EVIDENCE'])
            self.addCleanup(lambda: m.shutil.copytree(self.store.root / 'runs', destination))
        server_env = dict(os.environ, SCCACHE_SERVER_UDS=str(Path(endpoint.name) / 's'),
                          SCCACHE_DIR=str(self.base / 'cache'), SCCACHE_IDLE_TIMEOUT='0')
        environment = patch.dict(os.environ, server_env)
        environment.start()
        self.addCleanup(environment.stop)
        self.addCleanup(lambda: subprocess.run(['sccache', '--stop-server'], env=server_env,
                                              capture_output=True, timeout=30))
        (self.repo / 'Cargo.toml').write_text(
            '[package]\nname="retention-fixture"\nversion="0.1.0"\nedition="2021"\n')
        (self.repo / 'src').mkdir()
        (self.repo / 'src/lib.rs').write_text(
            '#[test] fn proof() { assert_eq!(2 + 2, 4); }\n')
        subprocess.run(['cargo', 'generate-lockfile', '--offline'], cwd=self.repo, check=True)
        m.git(self.repo, 'add', 'Cargo.toml', 'Cargo.lock', 'src')
        m.git(self.repo, 'commit', '-qm', 'tiny local runner fixture')
        origin = self.base / 'origin.git'
        subprocess.run(['git', 'init', '--bare', '-q', str(origin)], check=True)
        m.git(self.repo, 'remote', 'add', 'origin', str(origin))
        branch = m.git(self.repo, 'symbolic-ref', 'HEAD')
        m.git(self.repo, 'push', 'origin', branch)
        self.store.global_lock = m.HOST_LOCK
        self.store.global_lock.parent.mkdir(parents=True, exist_ok=True)
        args = types.SimpleNamespace(repo=str(self.repo), ref=branch,
                                     sha=m.git(self.repo, 'rev-parse', 'HEAD'),
                                     command=['cargo', 'test', '--locked', '--offline'],
                                     keep=False, cargo_only=True, timeout=120)
        result = m.run(self.store, args)
        self.assertEqual(result['process_exit'], 0, result)
        self.assertEqual(result['scenario'], 'green', result)
        self.assertTrue(result['removed'], result)
        self.assertFalse(Path(result['checkout']).exists())
        def report(row):
            values = {k: row[k] for k in ('sha', 'command', 'process_exit', 'tests', 'scenario')}
            values['cache'] = {phase: {
                'hits': row[phase]['stats']['cache_hits']['counts'],
                'misses': row[phase]['stats']['cache_misses']['counts'],
                'max_bytes': row[phase]['max_cache_size'],
            } for phase in ('cache_before', 'cache_after')}
            print(json.dumps(values))
        report(result)
        self.assertFalse(Path(result['temporary_directory']['path']).exists())
        second = m.run(self.store, args)
        self.assertEqual(second['scenario'], 'green', second)
        self.assertTrue(second['removed'])
        self.assertEqual(second['cache_admission']['state'], 'ready')
        self.assertGreater(second['cache_before']['stats']['compile_requests'],
                           result['cache_after']['stats']['compile_requests'])  # Existing server retained.
        report(second)
        # An obsolete pre-existing server is refused, never silently restarted.
        subprocess.run(['sccache', '--stop-server'], env=server_env, check=True, capture_output=True)
        obsolete = Path(endpoint.name) / 'obsolete-scratch'
        obsolete.mkdir()
        obsolete_env = dict(server_env, TMPDIR=str(obsolete), TMP=str(obsolete), TEMP=str(obsolete))
        subprocess.run(['sccache', '--start-server'], env=obsolete_env, check=True, capture_output=True)
        obsolete.rmdir()
        blocked = m.run(self.store, args)
        self.assertEqual(blocked['scenario'], 'red', blocked)
        self.assertIsNone(blocked['process_exit'])
        evidence = self.store.record_path(blocked['id']).parent
        self.assertEqual(json.loads((evidence / 'cache-admission.json').read_text())['state'], 'blocked')
        self.assertTrue((evidence / 'cache-probe.log').stat().st_size)
        self.assertFalse(obsolete.exists())
        self.assertFalse(Path(blocked['checkout']).exists())
        # Explicit owner recovery is only for this test's isolated, idle endpoint.
        subprocess.run(['sccache', '--stop-server'], env=server_env, check=True, capture_output=True)
        (self.repo / 'src/lib.rs').write_text('#[test] fn intended_failure() { panic!("fixture"); }\n')
        m.git(self.repo, 'add', 'src/lib.rs')
        m.git(self.repo, 'commit', '-qm', 'intentional diagnostic retention fixture')
        m.git(self.repo, 'push', 'origin', branch)
        args.sha = m.git(self.repo, 'rev-parse', 'HEAD')
        failed = m.run(self.store, args)
        self.addCleanup(m.remote.cleanup_temp_directory, failed['temporary_directory'])
        self.assertEqual(failed['process_exit'], 101)
        self.assertEqual(failed['scenario'], 'red')
        self.assertFalse(Path(failed['checkout']).exists())
        self.assertFalse(Path(failed['temporary_directory']['path']).exists())
        report(failed)
        args.keep = True
        retained = m.run(self.store, args)
        self.addCleanup(m.remote.cleanup_temp_directory, retained['temporary_directory'])
        self.assertEqual(retained['process_exit'], 101)
        self.assertEqual(retained['scenario'], 'red')
        self.assertTrue(Path(retained['checkout']).is_dir())
        self.assertTrue(Path(retained['temporary_directory']['path']).is_dir())
        report(retained)


if __name__ == '__main__':
    unittest.main()