Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.
Rust ai-agentsai-gatewayclaude-codecodexcoding-agentcost-optimizationdeepseekllm-gatewayllm-routerlocal-firstmodel-routingrusttask-routingtauri
#!/usr/bin/env python3
"""Opt-in local platform validation and explicit retired-debug reclamation (Unix)."""
import argparse
import contextlib
import hashlib
import importlib.util
import json
import os
import platform
from pathlib import Path
import re
import shutil
import signal
import subprocess
import sys
import tempfile
import time
import uuid
spec = importlib.util.spec_from_file_location("remote_rust", Path(__file__).with_name("remote-rust.py"))
remote = importlib.util.module_from_spec(spec)
spec.loader.exec_module(remote)
DEFAULT_ROOT = Path.home() / ".cache/hiroute/local-rust"
MAC_LOCK = Path.home() / ".cache/hiroute/mac-rust-validation.lock"
HOST_LOCK = MAC_LOCK if sys.platform == "darwin" else Path.home() / ".cache/hiroute/local-rust-validation.lock"
TOOLCHAIN_BIN = Path.home() / ".cargo/bin"
GIB = 1024 ** 3
def private_temp():
system = Path(tempfile.gettempdir()).resolve(strict=True)
if len(os.fsencode(system / "hr-xxxxxxxx")) > remote.MAX_TMPDIR_BYTES:
# macOS per-user /var/folders paths exceed socket budgets. /tmp is an OS
# convention, resolved before use, never a machine-specific mounted path.
system = Path("/tmp").resolve(strict=True)
if len(os.fsencode(system / "hr-xxxxxxxx")) > remote.MAX_TMPDIR_BYTES:
raise ValueError("No short system temporary directory; configure TMPDIR")
path = Path(tempfile.mkdtemp(prefix="hr-", dir=system))
os.chmod(path, 0o700)
return dict(path=str(path), system_directory=str(system))
def git(repo, *args):
return remote.command(["git", "-C", str(repo), *args])
def toolchain_environment():
"""PATH that locates the required tools, appending the standard rustup bin only
when a tool is missing from PATH. A toolchain already on PATH is never overridden."""
path = os.environ.get("PATH", "")
required = ("cargo", "rustc", "sccache", "git")
missing = [name for name in required if shutil.which(name, path=path) is None]
if missing and TOOLCHAIN_BIN.is_dir():
path = os.pathsep.join(part for part in (path, str(TOOLCHAIN_BIN)) if part)
missing = [name for name in required if shutil.which(name, path=path) is None]
if missing:
raise ValueError("Missing on PATH: %s; load the Cargo environment first, "
'for example with . "$HOME/.cargo/env"' % ", ".join(missing))
return path
def identity(path):
info = path.lstat()
if path.is_symlink() or not path.is_dir() or info.st_uid != os.geteuid():
raise ValueError("Not an owned real directory: " + str(path))
return [info.st_dev, info.st_ino]
def fingerprint(path):
"""Snapshot metadata without following links; no content or secrets in reports."""
digest = hashlib.sha256()
size = 0
for parent, dirs, files in os.walk(path, followlinks=False):
dirs.sort()
for name in sorted(dirs + files):
item = Path(parent) / name
info = item.lstat()
digest.update(repr((str(item.relative_to(path)), info.st_dev, info.st_ino,
info.st_mode, info.st_size, info.st_mtime_ns)).encode())
size += info.st_blocks * 512
return digest.hexdigest(), size
class Store:
def __init__(self, root=DEFAULT_ROOT, global_lock=HOST_LOCK):
self.root = Path(root).expanduser().resolve()
self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
if identity(self.root) and self.root.stat().st_mode & 0o077:
raise ValueError("State root must be private (0700)")
for name in ("runs", "checkouts", "locks"):
(self.root / name).mkdir(exist_ok=True, mode=0o700)
identity(self.root / name)
self.global_lock = Path(global_lock)
self.global_lock.parent.mkdir(parents=True, exist_ok=True)
def record_path(self, key):
if not re.fullmatch(r"[a-f0-9]{32}", key):
raise ValueError("Invalid record ID")
return self.root / "runs" / key / "result.json"
def load(self, key):
return json.loads(self.record_path(key).read_text())
def save(self, row):
path = self.record_path(row["id"])
path.parent.mkdir(exist_ok=True, mode=0o700)
remote.save(path, row)
@contextlib.contextmanager
def locked(self, checkout, *, wait=False):
# Only an explicitly queued run waits; cleanup always uses the default fast failure.
key = hashlib.sha256(str(checkout).encode()).hexdigest()
with contextlib.ExitStack() as stack:
handles = []
for path in (self.global_lock, self.root / "locks" / (key + ".lock")):
handle = stack.enter_context(path.open("a"))
remote.fcntl.flock(handle, remote.fcntl.LOCK_EX | (0 if wait else remote.fcntl.LOCK_NB))
handles.append(handle)
yield handles
def verify(self, row):
checkout = Path(row["checkout"])
if checkout.resolve() != checkout or identity(checkout) != row["identity"]:
raise ValueError("Checkout was replaced")
if git(checkout, "rev-parse", "--show-toplevel") != str(checkout):
raise ValueError("Not the recorded worktree root")
if git(checkout, "rev-parse", "HEAD") != row["sha"]:
raise ValueError("Checkout revision changed")
if row["kind"] == "managed":
if checkout.parent != self.root / "checkouts" or checkout.name != row["id"]:
raise ValueError("Not a managed checkout")
if "retention" in row:
remote.verify_disposable_source(checkout)
else:
if git(checkout, "status", "--porcelain", "--untracked-files=all"):
raise ValueError("Source changes/untracked files retained")
ignored = git(checkout, "ls-files", "--others", "--ignored", "--exclude-standard")
if any(not p.startswith("target/") for p in ignored.splitlines()):
raise ValueError("Ignored files outside target retained")
target = checkout / "target"
# Some commands never create a target, including failed package selection
# and cargo fmt. New managed records attest that absence at completion.
# A later-created target (including a dangling link) still fails closed.
if (row["kind"] == "managed" and ("retention" in row or
row.get("process_exit") is None and not row.get("command_started_at"))
and "target_identity" not in row and "debug_identity" not in row
and not os.path.lexists(target)):
if git(checkout, "ls-files", "target"):
raise ValueError("Target contains tracked files")
return checkout, target / "debug"
identity(target)
if target.resolve() != target or identity(target) != row["target_identity"]:
raise ValueError("Target was replaced")
debug = target / "debug"
if "retention" in row and "debug_identity" not in row and not os.path.lexists(debug):
return checkout, debug
if identity(debug) != row["debug_identity"]:
raise ValueError("Debug directory was replaced")
if git(checkout, "ls-files", "target"):
raise ValueError("Target contains tracked files")
return checkout, debug
def preview_one(self, row, *, lease=None):
report = {"id": row["id"], "checkout": row["checkout"], "state": "skipped"}
try:
with self.locked(row["checkout"]) if lease is None else contextlib.nullcontext(lease):
if row.get("removed") or row.get("keep") or row["status"] not in ("terminal", "retired"):
raise ValueError("Active, kept, removed, or unknown lifecycle")
if row.get("pilot_reuse") and not row.get("pilot_cleanup_ready"):
raise ValueError("Pilot owners have not finished their registered sessions")
if not row.get("evidence_saved"):
raise ValueError("Evidence not confirmed/exported")
checkout, debug = self.verify(row)
artifact = checkout / "target" if row["kind"] == "managed" and "retention" in row else debug
stamp, size = fingerprint(artifact)
token = hashlib.sha256((json.dumps(row, sort_keys=True) + stamp).encode()).hexdigest()
report.update(state="candidate", token=token, bytes=size,
target=str(artifact), sha=row["sha"], kind=row["kind"])
except (OSError, ValueError, KeyError, subprocess.SubprocessError) as error:
report["reason"] = str(error)
return report
def preview(self):
return [self.preview_one(self.load(p.parent.name))
for p in sorted((self.root / "runs").glob("*/result.json"))]
def apply(self, key, token, *, lease=None):
row = self.load(key)
# Hold locks throughout revalidation and deletion, including snapshot comparison.
with self.locked(row["checkout"]) if lease is None else contextlib.nullcontext(lease):
current_row = self.load(key)
if current_row["checkout"] != row["checkout"]:
raise ValueError("Record checkout changed")
row = current_row
if row.get("keep") or row.get("removed") or row["status"] not in ("terminal", "retired"):
raise ValueError("Not an inactive cleanup candidate")
if row.get("pilot_reuse") and not row.get("pilot_cleanup_ready"):
raise ValueError("Pilot owners have not finished their registered sessions")
if not row.get("evidence_saved"):
raise ValueError("Evidence not saved")
checkout, debug = self.verify(row)
artifact = checkout / "target" if row["kind"] == "managed" and "retention" in row else debug
stamp, size = fingerprint(artifact)
current = hashlib.sha256((json.dumps(row, sort_keys=True) + stamp).encode()).hexdigest()
if token != current:
raise ValueError("Candidate changed; preview again")
if row["kind"] == "managed":
# New records export runtime evidence even without a target tree.
if "retention" in row or os.path.lexists(checkout / "target"):
self.export_target(row)
remote.command(["git", "-C", row["repo"], "worktree", "remove", "--force", str(checkout)])
else:
shutil.rmtree(debug) # Never remove a developer worktree or its release/evidence.
row.update(removed=True, cleaned_at=time.time(), reclaimed_debug_bytes=size)
if "retention" in row:
row["cleanup"] = dict(state="removed", bytes=size)
if row.get("temporary_directory"):
row["temporary_cleanup"] = remote.cleanup_temp_directory(row["temporary_directory"])
self.save(row)
return {"id": key, "state": "removed", "bytes": size}
def export_target(self, row):
if "retention" in row:
row["evidence_export"] = remote.reporting().preserve_evidence(self.record_path(row["id"]).parent, row)
return
target = Path(row["checkout"]) / "target"
destination = self.record_path(row["id"]).parent / "artifacts"
for entry in target.iterdir():
if entry.name == "debug":
continue
# Preserve even unknown artifacts; links are refused, not followed outside the tree.
items = [entry] + (list(entry.rglob("*")) if entry.is_dir() else [])
if any(p.is_symlink() for p in items):
raise ValueError("Artifact contains links; retained for manual inspection")
destination.mkdir(exist_ok=True)
dest = destination / entry.name
if dest.exists():
if dest.is_dir():
shutil.rmtree(dest)
else:
dest.unlink()
if entry.is_dir():
shutil.copytree(entry, dest)
else:
shutil.copy2(entry, dest)
def retire(self, checkout):
checkout = Path(checkout).resolve(strict=True)
with self.locked(checkout):
row = dict(id=uuid.uuid4().hex, kind="developer", checkout=str(checkout),
identity=identity(checkout), sha=git(checkout, "rev-parse", "HEAD"),
status="retired", evidence_saved=True, keep=False, finished_at=time.time(),
target_identity=identity(checkout / "target"),
debug_identity=identity(checkout / "target/debug"))
self.verify(row)
self.save(row)
return row
def gc(self, apply=False, older_days=7, target_free=None):
results = []
for candidate in self.preview():
row = self.load(candidate["id"])
if row["kind"] != "managed" or row.get("finished_at", time.time()) > time.time() - older_days * 86400:
continue # Developer retirement is always manual, never pressure GC.
if candidate["state"] != "candidate":
results.append(candidate)
continue
if apply:
if target_free and shutil.disk_usage(self.root).free >= target_free:
break
try:
candidate = self.apply(row["id"], candidate["token"])
except (OSError, ValueError, subprocess.SubprocessError) as error:
candidate = dict(id=row["id"], state="retained", reason=str(error))
results.append(candidate)
return results
def ensure_space(self):
if shutil.disk_usage(self.root).free < 30 * GIB:
self.gc(apply=True, target_free=60 * GIB)
if shutil.disk_usage(self.root).free < 30 * GIB:
raise ValueError("Less than 30 GiB free; build not started")
def run(store, args, prepare=None, on_record=None):
repo = Path(args.repo).resolve(strict=True)
# A non-interactive shell often omits the Cargo environment; resolve it before any
# record, worktree or tool call, refusing with a remedy when it cannot be resolved.
os.environ["PATH"] = toolchain_environment()
cargo = args.command[1:] if args.command[:1] == ["--"] else args.command
reporter = remote.reporting()
feedback = reporter.options(args)
related = reporter.related_reports(store.root / "runs", feedback)
remote.validate(args.ref, args.sha, cargo)
source = getattr(args, "source", "origin")
jobs = getattr(args, "jobs", 2)
if source not in ("origin", "local") or not isinstance(jobs, int) or jobs < 1:
raise ValueError("Expected source origin/local and positive build jobs")
if args.timeout <= 0:
raise ValueError("Timeout must be positive")
if any(k in os.environ for k in ("CARGO_TARGET_DIR", "CARGO_BUILD_TARGET_DIR")):
raise ValueError("Remove target directory overrides")
store.ensure_space()
key = uuid.uuid4().hex
checkout = store.root / "checkouts" / key
row = dict(id=key, kind="managed", repo=str(repo), checkout=str(checkout), sha=args.sha,
command=cargo, status="preparing", keep=args.keep, evidence_saved=False,
retention="always" if args.keep else "never",
process_exit=None, scenario="unassessed", candidate_source=source,
build_jobs=jobs, wait_for_lock=getattr(args, "wait_for_lock", False),
cargo_build_target=os.environ.get("CARGO_BUILD_TARGET"),
platform=sys.platform, architecture=platform.machine(),
submitted_at=time.time(), **feedback)
row["executor_sha256"] = {
name: hashlib.sha256(Path(__file__).with_name(name).read_bytes()).hexdigest()
for name in ("local-rust.py", "remote-rust.py", "validation-report.py")
}
reporter.preflight(row, related)
store.save(row)
if on_record is not None:
on_record(row)
# Local execution can validate a committed candidate without publishing it.
# Fetch into a private ref in either case; never compile the caller's dirty tree.
ref = "refs/hiroute-local-validation/" + key
try:
row["checkout_started_at"] = time.time()
git(repo, "fetch", "--no-tags", "origin" if source == "origin" else str(repo), args.ref + ":" + ref)
git(repo, "merge-base", "--is-ancestor", args.sha, ref)
git(repo, "worktree", "add", "--detach", str(checkout), args.sha)
except subprocess.SubprocessError as error:
row.update(status="terminal", scenario="red", error=str(error), finished_at=time.time())
reporter.write_report(store.record_path(key).parent, row, related)
store.save(row)
return row
finally:
git(repo, "update-ref", "-d", ref)
row["identity"] = identity(checkout)
row["checkout_finished_at"] = time.time()
row["status"] = "running"
store.save(row)
# Cache admission and Cargo share the existing exclusive compiler lease.
env = dict(os.environ, CARGO_BUILD_JOBS=str(jobs), CARGO_INCREMENTAL="0", RUSTC_WRAPPER="sccache")
with contextlib.ExitStack() as leases:
process = None
handles = None
try:
server_env = remote.cache_server_environment(store.root)
row["cache_server_environment"] = {
"temporary_directory": server_env["TMPDIR"],
"idle_timeout": "0",
}
row["queued_at"] = time.time()
store.save(row)
handles = leases.enter_context(store.locked(checkout, wait=row["wait_for_lock"]))
row["capacity_acquired_at"] = time.time()
if row["wait_for_lock"] and shutil.disk_usage(store.root).free < 30 * GIB:
raise ValueError("Less than 30 GiB free after waiting; build not started")
row["cache_admission"] = remote.admit_cache_server(
store.record_path(key).parent, server_env, checkout, handles=handles)
before = remote.stats(store.record_path(key).parent, "cache-before", server_env)
info = private_temp()
row["temporary_directory"] = info
env["TMPDIR"] = info["path"]
store.save(row)
metadata = json.loads(subprocess.check_output(
["cargo", "metadata", "--locked", "--no-deps", "--format-version", "1"],
cwd=checkout, env=env, text=True))
remote.check_target(checkout, metadata["target_directory"])
for tool in ("rustc", "cargo"):
row[tool] = subprocess.check_output([tool, "--version"], cwd=checkout, env=env, text=True).strip()
if prepare is not None:
prepare(checkout, env, row, tuple(h.fileno() for h in handles))
store.save(row)
with (store.record_path(key).parent / "command.log").open("w") as output:
row["command_started_at"] = time.time()
process = remote.start_locked_command(cargo, checkout, env, output, handles)
try:
row["process_exit"] = process.wait(timeout=args.timeout)
finally:
if process.poll() is None:
os.killpg(process.pid, signal.SIGKILL)
row["process_exit"] = process.wait()
row["command_finished_at"] = time.time()
row["tests"] = remote.test_summary((store.record_path(key).parent / "command.log").read_text(errors="replace"), cargo)
bad = row["tests"]["state"] in ("tests_failed", "zero_tests_passed", "unrecognized_test_output")
row["scenario"] = "red" if row["process_exit"] != 0 or bad else ("green" if args.cargo_only else "unassessed")
row["evidence_saved"] = bool(args.cargo_only)
row["cache_before"] = before
row["cache_after"] = remote.stats(
store.record_path(key).parent,
"cache-after",
server_env,
)
except (Exception, KeyboardInterrupt) as error:
row["error"] = str(error)
row["scenario"] = "red"
finally:
row.update(status="terminal", finished_at=time.time())
reporter.write_report(store.record_path(key).parent, row, related)
for name, path in (("target_identity", checkout / "target"), ("debug_identity", checkout / "target/debug")):
if path.is_dir() and not path.is_symlink():
row[name] = identity(path)
store.save(row)
if not row["keep"]:
try:
with store.locked(checkout) if handles is None else contextlib.nullcontext(handles):
store.verify(row)
store.export_target(row)
row["evidence_saved"] = True
store.save(row)
candidate = store.preview_one(row, lease=handles)
if candidate["state"] == "candidate":
store.apply(key, candidate["token"], lease=handles)
else:
row = store.load(key)
row["cleanup"] = dict(state="retained", reason=candidate["reason"])
store.save(row)
except (OSError, ValueError, KeyError, subprocess.SubprocessError) as error:
row = store.load(key)
row["evidence_saved"] = False
row["evidence_export"] = dict(state="failed", error=str(error))
row["cleanup"] = dict(state="retained", reason=str(error))
store.save(row)
return store.load(key)
def argument_parser():
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest="action", required=True)
p = sub.add_parser("run", help="Exact committed candidate in an isolated worktree")
p.add_argument("--repo", default=".")
p.add_argument("--ref", required=True)
p.add_argument("--sha", required=True)
p.add_argument("--source", choices=("origin", "local"), default="origin")
p.add_argument("--jobs", type=int, default=2)
p.add_argument("--wait-for-lock", action="store_true",
help="Wait on the existing exclusive host/checkout locks before starting Cargo")
p.add_argument("--keep", action="store_true", help="Explicitly retain the checkout and artifacts after this run")
p.add_argument("--cargo-only", action="store_true", help="Assert pure Cargo check: no external scenario verdict or unique debug artifacts")
p.add_argument("--timeout", type=int, default=3600)
remote.reporting().add_arguments(p)
p.add_argument("command", nargs=argparse.REMAINDER)
sub.add_parser("preview", help="Read-only registered candidate list with snapshot tokens")
sub.add_parser("status", help="Read saved result and validation feedback").add_argument("id")
p = sub.add_parser("retire", help="Explicitly retire developer debug; never auto-detect inactivity")
p.add_argument("--checkout", required=True)
p.add_argument("--evidence-saved", action="store_true", required=True,
help="Confirm owner stopped use and unique debug evidence is saved elsewhere")
p = sub.add_parser("apply", help="Recheck and delete one unchanged preview candidate")
p.add_argument("id")
p.add_argument("--token", required=True)
p = sub.add_parser("gc", help="Aged managed validation only; never developer worktrees")
p.add_argument("--apply", action="store_true")
p.add_argument("--older-than", type=int, default=7)
for action in ("keep", "release"):
sub.add_parser(action).add_argument("id")
p = sub.add_parser("finish", help="Record actual scenario verdict after exporting unique debug evidence")
p.add_argument("id")
p.add_argument("--verdict", choices=("green", "expected_red", "red"), required=True)
p.add_argument("--evidence-saved", action="store_true", required=True)
return parser
def main():
parser = argument_parser()
args = parser.parse_args()
store = Store()
if args.action == "run":
result = run(store, args)
elif args.action == "status":
result = store.load(args.id)
elif args.action == "preview":
result = store.preview()
elif args.action == "retire":
result = store.retire(args.checkout)
elif args.action == "apply":
result = store.apply(args.id, args.token)
elif args.action == "gc":
if args.older_than < 0:
parser.error("Age cannot be negative")
result = store.gc(args.apply, args.older_than)
else:
row = store.load(args.id)
with store.locked(row["checkout"]):
row = store.load(args.id)
if row["status"] not in ("terminal", "retired"):
raise ValueError("Active or unknown run")
if args.action == "finish":
if args.verdict == "green" and row.get("process_exit") != 0:
raise ValueError("Nonzero/no process exit cannot become green")
if args.verdict == "green" and row.get("tests", {}).get("state") in (
"tests_failed", "zero_tests_passed", "unrecognized_test_output"
):
raise ValueError("Failed/zero/unrecognized Cargo tests cannot become green")
row.update(scenario=args.verdict, evidence_saved=True)
else:
row["keep"] = args.action == "keep"
store.save(row)
result = row
print(json.dumps(result, indent=2))
return 1 if args.action == "run" and result["scenario"] != "green" else 0
if __name__ == "__main__":
try:
sys.exit(main())
except (ValueError, OSError, subprocess.SubprocessError) as error:
print(str(error), file=sys.stderr)
sys.exit(2)