Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.
Rust ai-agentsai-gatewayclaude-codecodexcoding-agentcost-optimizationdeepseekllm-gatewayllm-routerlocal-firstmodel-routingrusttask-routingtauri
#!/usr/bin/env python3
"""Regression diagnostics from #94-shaped logs; no Cargo/product validation claims."""
import importlib.util
import contextlib
import io
import json
import os
from pathlib import Path
import tempfile
import types
import unittest
spec = importlib.util.spec_from_file_location("report", Path(__file__).with_name("validation-report.py"))
report = importlib.util.module_from_spec(spec)
spec.loader.exec_module(report)
def target(name="p0_gateway_runtime", failed=False, seconds=42.11, test="reasoning::late_model"):
return (f" Running tests/{name}.rs (target/debug/deps/{name}-abcdef0123456789)\n"
"running 1 test\n"
f"test {test} ... {'FAILED' if failed else 'ok'}\n"
f"test result: {'FAILED' if failed else 'ok'}. {0 if failed else 1} passed; "
f"{1 if failed else 0} failed; 0 ignored; 0 measured; 31 filtered out; finished in {seconds}s\n")
def row(**kwargs):
value = dict(id="run-a", plan="issue-94-b", phase="focused", sha="a" * 40,
process_exit=0, command=["cargo", "test", "-p", "hiroute-e2e", "--all-features"],
platform="linux", architecture="x86_64", rustc="rustc fixture", cargo="cargo fixture")
value.update(kwargs)
return value
class Parsing(unittest.TestCase):
def test_nested_daemon_summary_is_not_added_to_parent(self):
log = (" Finished `test` profile [unoptimized + debuginfo] target(s) in 1m 09s\n"
" Running unittests src/lib.rs (target/debug/deps/hiroute_daemon-abcdef0123456789)\n"
"running 2 tests\nrunning 1 test\ntest child ... ok\n"
"test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 181 filtered out; finished in 266.98s\n"
"test parent ... ok\n"
"test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 267.00s\n")
result = report.build_report(row(), log + target(seconds=220.11))
self.assertTrue(result["timing_complete"])
self.assertEqual([m["seconds"] for m in result["top_modules"]], [267., 220.11])
self.assertEqual(result["modules"][0]["counts"]["passed"], 2)
self.assertEqual(result["modules"][0]["child_summaries"], 1)
self.assertEqual(result["build_events"], [dict(seconds=69., line=1, scope="outer")])
def test_child_summary_does_not_complete_interrupted_parent(self):
for count in (1, 2):
with self.subTest(parent_count=count):
log = ("Running unittests src/lib.rs (target/debug/deps/hiroute_daemon-abcdef0123456789)\n"
f"running {count} tests\nrunning 1 test\ntest child ... ok\n"
"test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 20s\n")
result = report.build_report(row(process_exit=-9), log)
self.assertFalse(result["complete"])
self.assertFalse(result["modules"][0]["complete"])
self.assertEqual(result["top_modules"], [])
def test_failure_and_unfinished_target_remain_distinct(self):
log = target(failed=True) + "Running tests/next.rs (target/debug/deps/next-abcdef0123456789)\nrunning 1 test\n"
result = report.build_report(row(process_exit=101), log)
self.assertFalse(result["complete"])
self.assertEqual(result["modules"][0]["failures"], ["reasoning::late_model"])
self.assertTrue(result["modules"][0]["complete"])
self.assertIsNone(result["modules"][1]["seconds"])
self.assertFalse(result["modules"][1]["complete"])
def test_compile_failure_has_no_invented_test_duration(self):
result = report.build_report(row(process_exit=101), "error[E0425]: missing symbol\n")
self.assertFalse(result["complete"])
self.assertEqual(result["modules"], [])
self.assertIsNone(result["timing"]["command_seconds"])
def test_multiple_failed_binaries_are_retained(self):
result = report.build_report(row(process_exit=101), target(failed=True) + target("routing_plans", True, test="golden"))
self.assertEqual([m["failures"] for m in result["modules"]], [["reasoning::late_model"], ["golden"]])
def test_94_same_named_targets_have_separate_timings_but_ambiguous_cross_run_identity(self):
log = target() + target().replace("abcdef0123456789", "123456789abcdef0")
result = report.build_report(row(), log)
self.assertTrue(result["complete"])
self.assertEqual(len(result['modules']), 2)
self.assertNotEqual(result['modules'][0]['module'], result['modules'][1]['module'])
self.assertTrue(all(not m['comparison_identity_unique'] for m in result['modules']))
failed_log = target(failed=True) + target(failed=True).replace("abcdef0123456789", "123456789abcdef0")
final = report.build_report(row(phase='final', process_exit=101), failed_log,
[report.build_report(row(), target())])
self.assertTrue(all(f['focused_observation'] == 'ambiguous_target_identity' for f in final['findings']))
def test_repeated_identical_binary_is_not_assumed_independent(self):
result = report.build_report(row(), target() + target())
self.assertFalse(result["complete"])
self.assertTrue(all(not m["complete"] for m in result["modules"]))
def test_unassigned_summary_is_not_attributed_to_a_module(self):
result = report.build_report(row(), "test result: ok. 1 passed; 0 failed; 0 ignored\n")
self.assertFalse(result["complete"])
self.assertEqual(result["modules"], [])
self.assertEqual(len(result["unassigned_summaries"]), 1)
def test_ansi_doc_tests_and_missing_time(self):
log = "\x1b[32m Doc-tests hiroute_domain\x1b[0m\nrunning 0 tests\ntest result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out\n"
result = report.build_report(row(), log)
self.assertTrue(result["complete"])
self.assertFalse(result["timing_complete"])
self.assertIsNone(result["modules"][0]["seconds"])
def test_unassigned_summary_keeps_otherwise_successful_report_incomplete(self):
log = "test result: ok. 1 passed; 0 failed; 0 ignored\n" + target()
self.assertFalse(report.build_report(row(), log)["observed_tests_passed"])
class Feedback(unittest.TestCase):
def test_harness_settings_distinguish_omitted_environment_argument_and_unknown(self):
command = ["cargo", "test", "--locked"]
omitted = report.harness_concurrency(command, {})
self.assertEqual(omitted["requested_threads"], [])
self.assertTrue(omitted["environment_recorded"])
self.assertEqual(omitted["configuration_source"], "default")
self.assertIsNone(omitted["configured_thread_limit"])
old = report.context(row(cpu_count=16))["harness_concurrency"]
self.assertFalse(old["environment_recorded"])
self.assertEqual(old["configuration_source"], "unknown")
self.assertIsNone(old["effective_threads"])
for arguments in (["--test-threads=2"], ["--test-threads", "2"]):
context = report.harness_concurrency([*command, "--", *arguments], {"RUST_TEST_THREADS": "8"})
self.assertEqual(context["requested_threads"], ["2"])
self.assertEqual(context["environment_threads"], "8")
self.assertEqual(context["configuration_source"], "argument")
self.assertEqual(context["configured_thread_limit"], 2)
self.assertIsNone(context["effective_threads"])
def test_ambiguous_or_invalid_thread_setting_does_not_invent_limit(self):
for arguments in (["--test-threads"], ["--test-threads=0"], ["--test-threads=oops"],
["--test-threads=1", "--test-threads=2"]):
context = report.harness_concurrency(["cargo", "test", "--", *arguments], {"RUST_TEST_THREADS": "8"})
self.assertIsNone(context["configured_thread_limit"])
self.assertIsNone(context["effective_threads"])
self.assertIsNone(report.harness_concurrency(["cargo", "check"], {}))
def test_same_command_with_different_launch_environment_reports_context_gap(self):
command = row()["command"]
focused = report.build_report(row(harness_concurrency=report.harness_concurrency(
command, {"RUST_TEST_THREADS": "1"})), target())
final = report.build_report(row(phase="final", process_exit=101,
harness_concurrency=report.harness_concurrency(command, {})),
target(failed=True), [focused])
self.assertEqual(final["findings"][0]["context_differences"], ["harness_concurrency"])
def test_child_failures_are_not_attributed_to_parent_regressions(self):
focused = report.build_report(row(), target())
# A parent can intentionally spawn a failing test and still pass its own assertion.
nested = target(failed=True) + "test result: ok. 1 passed; 0 failed; 0 ignored; finished in 43s\n"
final = report.build_report(row(phase="final", process_exit=101), nested, [focused])
self.assertEqual([f["kind"] for f in final["findings"]], ["full_failure_unattributed"])
earlier = report.build_report(row(), nested)
final = report.build_report(row(phase="final", process_exit=101), target(failed=True), [earlier])
self.assertEqual(final["findings"][0]["focused_observation"], "ambiguous_child_output")
def test_94_package_pass_workspace_failure_is_context_gap_not_flake(self):
focused = report.build_report(row(), target())
final = report.build_report(row(phase="final", process_exit=101,
command=["cargo", "test", "--workspace", "--exclude", "hiroute-desktop", "--all-features"]),
target(failed=True), [focused])
finding = final["findings"][0]
self.assertEqual(finding["kind"], "focused_full_gap")
self.assertEqual(finding["focused_observation"], "ok")
self.assertEqual(finding["context_differences"], ["package_scope"])
self.assertNotIn("flaky", json.dumps(final))
def test_missing_case_and_ignored_case_are_not_success(self):
for focused_log, expected in [(target(test="different"), "not_observed"),
(target().replace("... ok", "... ignored").replace("1 passed; 0 failed; 0 ignored", "0 passed; 0 failed; 1 ignored"), "ignored")]:
focused = report.build_report(row(), focused_log + target('other_target', test='other_passing_case'))
final = report.build_report(row(phase="final", process_exit=101), target(failed=True), [focused])
self.assertEqual(final["findings"][0]["focused_observation"], expected)
def test_prior_sha_or_other_plan_is_not_same_candidate_evidence(self):
focused = report.build_report(row(), target())
for changed, expected in [(dict(sha="b" * 40), "different_revision"), (dict(plan="another"), "unrelated_plan")]:
result = report.build_report(row(phase="final", process_exit=101, **changed), target(failed=True), [focused])
self.assertEqual([f["kind"] for f in result["findings"]], [expected])
def test_same_flags_with_missing_toolchain_remain_unknown(self):
focused = report.build_report(row(rustc=None), target())
result = report.build_report(row(phase="final", process_exit=101, rustc=None), target(failed=True), [focused])
self.assertIn("rustc", result["findings"][0]["unknown_context"])
def test_failed_focused_run_does_not_establish_escape(self):
focused = report.build_report(row(process_exit=101), target(failed=True))
result = report.build_report(row(phase="final", process_exit=101), target(failed=True), [focused])
self.assertEqual(result["findings"], [])
def test_repeated_final_warns_without_changing_exit(self):
previous = report.build_report(row(phase="final"), target())
result = report.build_report(row(phase="final"), target(), [previous])
self.assertEqual(result["findings"][0]["kind"], "repeated_final")
self.assertEqual(result["process_exit"], 0)
record = row(phase='final')
with contextlib.redirect_stderr(io.StringIO()) as error:
report.preflight(record, [previous])
self.assertIn('Repeated final', error.getvalue())
self.assertIn('validation_advice', record)
def test_zero_selected_focused_run_is_not_coverage(self):
focused = report.build_report(row(), target().replace('running 1 test', 'running 0 tests')
.replace('1 passed', '0 passed').replace('test reasoning::late_model ... ok\n', ''))
result = report.build_report(row(phase='final', process_exit=101), target(failed=True), [focused])
self.assertFalse(focused['observed_tests_passed'])
self.assertEqual(result['findings'], [])
def test_final_failure_without_test_output_requests_diagnosis(self):
focused = report.build_report(row(), target())
result = report.build_report(row(phase="final", process_exit=101), "compile error", [focused])
self.assertEqual(result["findings"][0]["kind"], "full_failure_unattributed")
class Storage(unittest.TestCase):
def test_evidence_export_preserves_reports_and_runtime_without_copying_compiler_caches(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
checkout, run = root / 'checkout', root / 'run'
run.mkdir()
expected = {'target/e2e/result.json': 'actual failure',
'target/smoke/run-a/private/diagnostic.log': 'private evidence',
'target/e2e/debug/details.json': 'nested evidence, not a Cargo profile',
'artifacts/screenshot.png': 'captured pixels',
'target/smoke/builds/' + 'a' * 64 + '/hiroute-cli/receipt.json': '{"recipe":"selected"}'}
skipped = ('target/debug', 'target/release', 'target/custom',
'target/x86_64-pc-windows-msvc/debug', 'target/wasm32-wasip1/debug',
'target/validation-schedule', 'target/smoke/builds/' + 'a' * 64 + '/hiroute-cli/binary')
for name, value in expected.items():
path = checkout / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(value)
for name in skipped:
path = checkout / name / 'large-binary'
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text('rebuildable')
record = dict(checkout=str(checkout), sha='a' * 40,
command=['cargo', 'test', '--profile=custom', '--target=wasm32-wasip1'],
process_exit=101, scenario='red')
first = report.preserve_evidence(run, record)
self.assertEqual(first, report.preserve_evidence(run, record))
self.assertEqual((record['process_exit'], record['scenario']), (101, 'red'))
manifest = json.loads(Path(first['manifest']).read_text())
self.assertEqual(manifest['source_revision'], record['sha'])
self.assertEqual(set(manifest['files']), set(expected))
for name, value in expected.items():
self.assertEqual((run / manifest['files'][name]['path']).read_text(), value)
for name in skipped:
self.assertFalse((run / 'artifacts' / Path(name).relative_to('target')).exists())
(checkout / 'target/e2e/result.json').write_text('changed evidence')
with self.assertRaisesRegex(ValueError, 'Previously exported evidence differs'):
report.preserve_evidence(run, record)
self.assertEqual((run / 'artifacts/e2e/result.json').read_text(), 'actual failure')
def test_inherited_and_configured_cargo_targets_do_not_export_compiler_profiles(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
checkout, run = root / 'checkout', root / 'run'
run.mkdir()
for target in ('wasm32-wasip1', 'private_json_target'):
profile = checkout / 'target' / target / 'debug'
profile.mkdir(parents=True)
(profile / 'binary').write_bytes(b'rebuildable compiler output')
if target == 'private_json_target':
(profile / '.cargo-lock').touch()
nested = checkout / 'target/e2e/debug/details.json'
nested.parent.mkdir(parents=True)
nested.write_text('unique runtime evidence')
record = dict(checkout=str(checkout), command=['cargo', 'check'],
cargo_build_target='wasm32-wasip1')
report.preserve_evidence(run, record)
manifest = json.loads((run / 'evidence.json').read_text())
self.assertEqual(set(manifest['files']), {'target/e2e/debug/details.json'})
self.assertEqual((run / 'artifacts/e2e/debug/details.json').read_text(), 'unique runtime evidence')
self.assertFalse((run / 'artifacts/wasm32-wasip1/debug').exists())
self.assertFalse((run / 'artifacts/private_json_target/debug').exists())
def test_evidence_export_refuses_links_and_missing_reports_before_any_resource_deletion(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
checkout, run = root / 'checkout', root / 'run'
(checkout / 'target/e2e').mkdir(parents=True)
run.mkdir()
outside = root / 'outside'
outside.write_text('borrowed bytes')
link = checkout / 'target/e2e/link'
link.symlink_to(outside)
record = dict(checkout=str(checkout), command=['cargo', 'test'])
with self.assertRaisesRegex(ValueError, 'link or special file'):
report.preserve_evidence(run, record)
self.assertEqual(outside.read_text(), 'borrowed bytes')
self.assertTrue(link.is_symlink())
link.unlink()
record['validation_report'] = dict(error='fixture report write failure')
with self.assertRaisesRegex(ValueError, 'report was not saved'):
report.preserve_evidence(run, record)
self.assertTrue(checkout.is_dir())
record.pop('validation_report')
(run / 'evidence.json').symlink_to(outside)
with self.assertRaisesRegex(ValueError, 'manifest is a link'):
report.preserve_evidence(run, record)
self.assertEqual(outside.read_text(), 'borrowed bytes')
def test_previous_custom_profiles_stay_disposable_when_the_command_changes(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
checkout = root / 'checkout'
for profile in ('ci', 'private_json_target/ci'):
directory = checkout / 'target' / profile
directory.mkdir(parents=True)
(directory / '.cargo-lock').touch()
(directory / 'binary').write_bytes(b'rebuildable custom compiler output')
evidence = checkout / 'target/e2e/debug/details.json'
evidence.parent.mkdir(parents=True)
evidence.write_text('unique runtime evidence')
for index, command in enumerate((['cargo', 'check', '--profile', 'ci'], ['cargo', 'check'])):
run = root / str(index)
run.mkdir()
report.preserve_evidence(run, dict(checkout=str(checkout), command=command))
manifest = json.loads((run / 'evidence.json').read_text())
self.assertEqual(set(manifest['files']), {'target/e2e/debug/details.json'})
self.assertFalse((run / 'artifacts/ci').exists())
self.assertFalse((run / 'artifacts/private_json_target/ci').exists())
def test_link_and_special_profile_markers_cannot_hide_unique_evidence(self):
for kind in ('link', 'dangling-link', *(['fifo'] if hasattr(os, 'mkfifo') else [])):
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as folder:
root = Path(folder)
checkout, run = root / 'checkout', root / 'run'
evidence = checkout / 'target/unique-evidence'
evidence.mkdir(parents=True)
(evidence / 'diagnostic.json').write_text('unique runtime evidence')
outside = root / 'outside'
if kind == 'link':
outside.write_text('external bytes')
marker = evidence / '.cargo-lock'
if kind == 'fifo':
os.mkfifo(marker)
else:
marker.symlink_to(outside)
run.mkdir()
with self.assertRaisesRegex(ValueError, 'link or special file'):
report.preserve_evidence(run, dict(checkout=str(checkout), command=['cargo', 'check']))
self.assertEqual((evidence / 'diagnostic.json').read_text(), 'unique runtime evidence')
self.assertFalse((run / 'evidence.json').exists())
if kind == 'link':
self.assertEqual(outside.read_text(), 'external bytes')
def test_report_survives_checkout_removal_and_never_changes_verdict(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
run = root / "run-a"
run.mkdir()
(run / "command.log").write_text(target())
original = row(phase="final", scenario="expected_red")
info = report.write_report(run, original)
self.assertEqual(original["scenario"], "expected_red")
self.assertTrue(Path(info["markdown"]).is_file())
loaded = report.related_reports(root, dict(plan="issue-94-b", related_runs=["run-a"]))
self.assertEqual(loaded[0]["sha"], original["sha"])
self.assertTrue(info["follow_up"])
with self.assertRaisesRegex(ValueError, "another plan"):
report.related_reports(root, dict(plan="other", related_runs=["run-a"]))
def test_related_id_cannot_escape_run_store(self):
for value in ("../secret", "/tmp/secret", "a/b"):
with self.assertRaises(ValueError):
report.options(types.SimpleNamespace(plan="task", related_run=[value]))
with self.assertRaises(ValueError):
report.options(types.SimpleNamespace(related_run=["id"]))
def test_report_write_failure_does_not_mask_process_failure(self):
with tempfile.TemporaryDirectory() as folder:
record = row(process_exit=101)
info = report.write_report(Path(folder) / "missing", record)
self.assertFalse(info["complete"])
self.assertIn("error", info)
self.assertEqual(record["process_exit"], 101)
def test_smoke_import_requires_owned_path_and_matching_revision(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
checkout = root / "checkout"
checkout.mkdir()
value = dict(schema="hiroute.smoke.result/v1", source_revision="a" * 40, run_id="smoke-1",
cases=[dict(id="gateway", execution="executed", state="green", build_ms=120,
execution_ms=50, unclassified_ms=0, timing_complete=True, secret="not copied")])
path = checkout / "smoke.json"
path.write_text(json.dumps(value))
result = report.smoke_timings("smoke report=" + str(path), checkout, "a" * 40)
self.assertEqual(result[0]["cases"][0]["build_ms"], 120)
self.assertNotIn("secret", result[0]["cases"][0])
self.assertEqual(report.smoke_timings("smoke report=" + str(path), checkout, "b" * 40), [])
outside = root / "outside.json"
outside.write_text(json.dumps(value))
(checkout / "link").symlink_to(outside)
self.assertEqual(report.smoke_timings("smoke report=" + str(checkout / "link"), checkout, "a" * 40), [])
def test_successful_smoke_captured_stdout_uses_current_command_window(self):
with tempfile.TemporaryDirectory() as folder:
checkout = Path(folder)
for name, start in [('current', 1100), ('old', 100)]:
directory = checkout / 'target/smoke' / name
directory.mkdir(parents=True)
(directory / 'report.json').write_text(json.dumps(dict(schema='hiroute.smoke.result/v1',
source_revision='a' * 40, started_unix_ms=start, finished_unix_ms=start + 100,
run_id=name, cases=[])))
values = report.smoke_timings('', checkout, 'a' * 40, 1, 2)
self.assertEqual([v['run_id'] for v in values], ['current'])
self.assertEqual(report.smoke_timings('', checkout, 'a' * 40), [])
if __name__ == "__main__":
unittest.main()