Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.
Rust ai-agentsai-gatewayclaude-codecodexcoding-agentcost-optimizationdeepseekllm-gatewayllm-routerlocal-firstmodel-routingrusttask-routingtauri
use super::*;
use crate::server::core_runtime::profiles::fixed_reasoning;
use crate::server::request_plan::IngressProtocol;
use std::os::unix::fs::PermissionsExt;
use std::time::{Duration, Instant};
fn setup() -> PathBuf {
let mut random = [0u8; 16];
getrandom::fill(&mut random).unwrap();
let root = fs::canonicalize(std::env::temp_dir())
.unwrap()
.join(format!("hiroute-capture-{:x}", u128::from_le_bytes(random)));
fs::create_dir(&root).unwrap();
fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap();
let session = Session {
source_sha: "a".repeat(40),
binary_sha256: "b".repeat(64),
client: "fixture".into(),
expires_at: now() + 60,
delete_after: now() + 120,
};
private_file(&root.join("session.json"))
.unwrap()
.write_all(&serde_json::to_vec(&session).unwrap())
.unwrap();
root
}
fn profile() -> CandidateProtocolProfile {
CandidateProtocolProfile::exact_portable_path(
IngressProtocol::Messages,
IngressProtocol::ChatCompletions,
"physical",
fixed_reasoning("fixed"),
)
}
fn promoted_correlation(index: u32) -> CaptureCorrelation {
CaptureCorrelation {
request_token: Some(hiroute_diagnostics::identity::CorrelationToken::from_bytes(
[1; 32],
)),
request_id: "private-request".into(),
attempt_index: index,
attempt_token: Some(hiroute_diagnostics::identity::CorrelationToken::from_bytes(
[index as u8; 32],
)),
}
}
pub(super) fn correlation_checkpoint(root: &Path, stage: &str) {
if std::env::var_os("HIROUTE_CAPTURE_INTERRUPT_ROOT").as_deref() == Some(root.as_os_str())
&& std::env::var("HIROUTE_CAPTURE_INTERRUPT_STAGE").as_deref() == Ok(stage)
{
private_file(&root.join("checkpoint"))
.unwrap()
.write_all(stage.as_bytes())
.unwrap();
loop {
std::thread::park();
}
}
}
#[tokio::test]
async fn sealed_correlation_survives_process_termination() {
const TEST: &str =
"runtime::stream_capture::tests::sealed_correlation_survives_process_termination";
if let Some(root) = std::env::var_os("HIROUTE_CAPTURE_INTERRUPT_ROOT") {
let root = PathBuf::from(root);
let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();
capture.record(1, b"{}");
capture.record(2, &[]);
capture.record(3, &200u16.to_le_bytes());
capture.record(4, b"data: {invalid\n\n");
capture.failed();
PendingCapture(Arc::downgrade(&capture.0)).promoted(promoted_correlation(1));
panic!("missing interruption checkpoint");
}
for stage in ["partial", "ready", "published"] {
let root = setup();
let mut child = std::process::Command::new(std::env::current_exe().unwrap())
.args(["--exact", TEST, "--nocapture"])
.env("HIROUTE_CAPTURE_INTERRUPT_ROOT", &root)
.env("HIROUTE_CAPTURE_INTERRUPT_STAGE", stage)
.stdout(std::process::Stdio::null())
.spawn()
.unwrap();
let deadline = Instant::now() + Duration::from_secs(10);
while !root.join("checkpoint").exists() && Instant::now() < deadline {
if child.try_wait().unwrap().is_some() {
break;
}
std::thread::sleep(Duration::from_millis(5));
}
let reached = root.join("checkpoint").exists();
let stopped = root.join("stopped").exists();
// SIGKILL cannot execute a destructor or finish an in-progress write.
let _ = child.kill();
let status = child.wait().unwrap();
assert!(
reached && stopped,
"{stage}: checkpoint not reached: {status}"
);
let path = root.join("attempt-1.capture");
let bytes = read_private(&path, MAX_FILE).unwrap();
let records = replay::records(&bytes).unwrap();
assert_eq!(records.last().unwrap().0, 8, "{stage}");
assert_eq!(records.iter().filter(|r| r.0 == 8).count(), 1, "{stage}");
assert_eq!(
records.iter().filter(|r| r.0 == 7).count(),
usize::from(stage == "published"),
"{stage}"
);
for chunk in [0, 1, 4096] {
let replay = replay_capture(&path, chunk).await.unwrap();
assert_eq!(replay["decoder"]["reason"], "invalid_json", "{stage}");
assert_eq!(replay["captured_gateway_failure"], true, "{stage}");
}
fs::remove_dir_all(root).unwrap();
}
}
#[test]
fn promoted_failure_appends_only_one_bounded_correlation_without_reopening_content() {
let root = setup();
let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();
capture.record(1, b"{}");
capture.record(2, &[]);
capture.record(3, &200u16.to_le_bytes());
capture.record(4, b"original response");
capture.failed();
let path = root.join("attempt-1.capture");
let initial = fs::read(&path).unwrap();
assert!(root.join("stopped").exists());
assert_eq!(replay::records(&initial).unwrap().last().unwrap().0, 8);
PendingCapture(Arc::downgrade(&capture.0)).promoted(promoted_correlation(1));
let bound = fs::read(&path).unwrap();
assert_eq!(&initial[..initial.len() - 9], &bound[..initial.len() - 9]);
let records = replay::records(&bound).unwrap();
assert_eq!(records.iter().filter(|r| r.0 == 7).count(), 1);
assert_eq!(records.iter().filter(|r| r.0 == 8).count(), 1);
assert_eq!(records.last().unwrap().0, 8);
let correlation: serde_json::Value =
serde_json::from_slice(records.iter().find(|r| r.0 == 7).unwrap().1).unwrap();
assert_eq!(correlation["attempt_index"], 1);
assert!(!capture.0.lock().unwrap().active);
PendingCapture(Arc::downgrade(&capture.0)).promoted(promoted_correlation(2));
capture.record(4, b"must not capture a later response");
capture.failed();
assert_eq!(fs::read(&path).unwrap(), bound);
let weak = PendingCapture(Arc::downgrade(&capture.0));
drop(capture);
weak.promoted(promoted_correlation(3));
assert_eq!(fs::read(&path).unwrap(), bound);
fs::remove_dir_all(root).unwrap();
}
#[test]
fn sealed_correlation_respects_original_bounds_and_file_identity() {
for guard in [
"file_limit",
"session_limit",
"temporary_copy_limit",
"records",
"expiry",
"identity",
] {
let root = setup();
let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();
capture.record(1, b"{}");
capture.record(2, &[]);
capture.failed();
let path = root.join("attempt-1.capture");
let initial = fs::read(&path).unwrap();
match guard {
"file_limit" => {
let mut w = capture.0.lock().unwrap();
w.limit = w.written;
}
"session_limit" => {
private_file(&root.join("other.capture"))
.unwrap()
.set_len(MAX_TOTAL - initial.len() as u64)
.unwrap();
}
"temporary_copy_limit" => {
// The final delta fits, but both complete copies cannot coexist.
let correlation_bytes = serde_json::to_vec(&promoted_correlation(1)).unwrap().len();
let existing_bytes: u64 = fs::read_dir(&root)
.unwrap()
.map(|entry| entry.unwrap().metadata().unwrap().len())
.sum();
private_file(&root.join("other.capture"))
.unwrap()
.set_len(MAX_TOTAL - existing_bytes - correlation_bytes as u64 - 9)
.unwrap();
}
"records" => capture.0.lock().unwrap().records = MAX_RECORDS,
"expiry" => capture.0.lock().unwrap().expires_at = now() - 1,
"identity" => {
fs::rename(&path, root.join("original.capture")).unwrap();
private_file(&path).unwrap().write_all(&initial).unwrap();
}
_ => unreachable!(),
}
PendingCapture(Arc::downgrade(&capture.0)).promoted(promoted_correlation(1));
assert_eq!(fs::read(&path).unwrap(), initial, "{guard}");
assert!(!capture.0.lock().unwrap().active, "{guard}");
drop(capture);
fs::remove_dir_all(root).unwrap();
}
}
// Only after the test has dropped its previous writer. A concurrent process
// spawn can inherit the open-file description until CLOEXEC closes it. Wait
// for that actual kernel ownership to end; all other errors remain failures.
fn open_after_capture_release(root: &Path, mut on_blocked: impl FnMut()) -> Capture {
let deadline = Instant::now() + Duration::from_secs(2);
loop {
match Capture::open(root, &profile(), None, 2, true) {
Ok(capture) => return capture,
Err(error)
if error.kind() == io::ErrorKind::WouldBlock && Instant::now() < deadline =>
{
on_blocked();
std::thread::sleep(Duration::from_millis(1));
}
Err(error) => panic!("capture ownership did not release: {error}"),
}
}
}
#[test]
fn completed_capture_waits_for_inherited_lock_descriptor_release() {
let root = setup();
let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();
// A descriptor duplicate models the same open-file description inherited
// across fork, even when the originating writer has already been dropped.
let inherited = capture.0.lock().unwrap()._lock.try_clone().unwrap();
drop(capture);
assert_eq!(
Capture::open(&root, &profile(), None, 2, true)
.err()
.unwrap()
.kind(),
io::ErrorKind::WouldBlock,
"live descriptor ownership must still reject another writer"
);
let (release, observed) = std::sync::mpsc::channel();
let holder = std::thread::spawn(move || {
observed.recv_timeout(Duration::from_secs(2)).unwrap();
std::thread::sleep(Duration::from_millis(50));
drop(inherited);
});
let mut blocked = 0;
let next = open_after_capture_release(&root, || {
blocked += 1;
if blocked == 1 {
release.send(()).unwrap();
}
});
holder.join().unwrap();
assert!(
blocked > 0,
"re-admission must observe the retained kernel lock"
);
drop(next);
fs::remove_dir_all(root).unwrap();
}
#[tokio::test]
async fn capture_preserves_bytes_and_replays_request_bound_tools_across_chunks() {
let root = setup();
let request = serde_json::json!({"model":"alias", "max_tokens":64,"messages":[{"role":"user","content":"hello"}], "tools":[{"name":"known","description":"tool", "input_schema":{"type":"object"}}]});
let ir = adapters::decode_ingress_request(IngressProtocol::Messages, &request).unwrap();
let tools = adapters::ChatToolProjection::for_request(&ir).unwrap();
let body = serde_json::to_vec(&request).unwrap();
let capture = Capture::open(&root, &profile(), Some(&tools), body.len(), true).unwrap();
capture.record(1, &body[..9]);
capture.record(1, &body[9..]);
capture.record(2, &[]);
capture.record(3, &103u16.to_le_bytes());
capture.record(3, &200u16.to_le_bytes());
let sse = b"data: {\"id\":\"resp\",\"model\":\"physical\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call\",\"type\":\"function\",\"function\":{\"name\":\"unknown_secret_tool\",\"arguments\":\"{}\"}}]},\"finish_reason\":null}]}\n\ndata: [DONE]\n\n";
capture.record(4, &sse[..17]);
capture.record(4, &sse[17..]);
capture.failed();
// The supervisor may stop the process as soon as this marker appears.
// A failed sample must already be durable and sealed while other owners still exist.
assert!(root.join("stopped").exists());
assert!(root.join("active.lock").exists());
let at_stop = read_private(&root.join("attempt-1.capture"), MAX_FILE).unwrap();
assert_eq!(replay::records(&at_stop).unwrap().last().unwrap().0, 8);
for size in [0, 1, 4096] {
let replay = replay_capture(&root.join("attempt-1.capture"), size)
.await
.unwrap();
assert_eq!(replay["decoder"]["result"], "rejected");
assert_eq!(replay["decoder"]["reason"], "missing_tool_identity");
}
capture.record(4, b"cannot append after seal");
drop(capture);
let path = root.join("attempt-1.capture");
assert_eq!(
fs::metadata(&path).unwrap().permissions().mode() & 0o777,
0o600
);
let bytes = read_private(&path, MAX_FILE).unwrap();
let records = replay::records(&bytes).unwrap();
let actual: Vec<u8> = records
.iter()
.filter(|r| r.0 == 1)
.flat_map(|r| r.1.iter().copied())
.collect();
assert_eq!(actual, body);
let chunks: Vec<_> = records.iter().filter(|r| r.0 == 4).map(|r| r.1).collect();
assert_eq!(chunks, vec![&sse[..17], &sse[17..]]);
for size in [0, 1, 4096] {
let result = replay_capture(&path, size).await.unwrap();
assert_eq!(result["decoder"]["result"], "rejected");
assert!(!result.to_string().contains("unknown_secret_tool"));
}
assert!(
Capture::open(&root, &profile(), None, 2, true).is_err(),
"first failure stops further captures"
);
fs::remove_dir_all(root).unwrap();
}
#[tokio::test]
async fn bounded_capture_is_fail_closed_without_mutating_business_input() {
let root = setup();
let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();
assert!(
Capture::open(&root, &profile(), None, 2, true).is_err(),
"one writer per session"
);
{
let mut writer = capture.0.lock().unwrap();
writer.limit = writer.written + 10;
}
capture.record(1, b"{}");
drop(capture);
assert!(
replay_capture(&root.join("attempt-1.capture"), 0)
.await
.is_err()
);
fs::set_permissions(root.join("session.json"), fs::Permissions::from_mode(0o644)).unwrap();
let capture = open_after_capture_release(&root, || {});
assert_eq!(
fs::metadata(root.join("session.json"))
.unwrap()
.permissions()
.mode()
& 0o777,
0o644
);
drop(capture);
fs::remove_dir_all(root).unwrap();
}
#[test]
fn capture_redacts_transport_headers_and_enforces_expiry() {
let root = setup();
let mut profile = profile();
if let crate::server::core_runtime::profiles::CriticalFact::Exact(headers) =
&mut profile.connector.headers
{
headers
.required_headers
.push(("x-api-key".into(), "SECRET_CREDENTIAL".into()));
}
let capture = Capture::open(&root, &profile, None, 2, true).unwrap();
{
let mut writer = capture.0.lock().unwrap();
writer.expires_at = now() - 1;
}
capture.record(1, b"{}");
drop(capture);
let bytes = read_private(&root.join("attempt-1.capture"), MAX_FILE).unwrap();
assert!(!String::from_utf8_lossy(&bytes).contains("SECRET_CREDENTIAL"));
assert_ne!(replay::records(&bytes).unwrap().last().unwrap().0, 8);
fs::remove_dir_all(root).unwrap();
}
#[tokio::test]
async fn capture_limits_and_partial_request_cannot_produce_replay_evidence() {
// Even all expected bytes are insufficient without the reader's verified
// EOF: an early upstream error is allowed to seal an incomplete capture.
for request in [b"{".as_slice(), b"{}".as_slice()] {
let root = setup();
assert!(Capture::open(&root, &profile(), None, MAX_FILE as usize, true).is_err());
assert!(root.join("active.lock").exists());
let capture = open_after_capture_release(&root, || {});
capture.record(1, request);
capture.record(3, &200u16.to_le_bytes());
capture.failed();
drop(capture);
assert_eq!(
replay_capture(&root.join("attempt-1.capture"), 0)
.await
.unwrap_err(),
"request body capture incomplete"
);
fs::remove_dir_all(root).unwrap();
}
let root = setup();
for _ in 0..MAX_ATTEMPTS {
drop(open_after_capture_release(&root, || {}));
}
assert!(Capture::open(&root, &profile(), None, 2, true).is_err());
assert!(root.join("active.lock").exists());
fs::remove_dir_all(root).unwrap();
let root = setup();
private_file(&root.join("existing.capture"))
.unwrap()
.set_len(MAX_TOTAL)
.unwrap();
assert!(Capture::open(&root, &profile(), None, 2, true).is_err());
assert!(root.join("active.lock").exists());
fs::remove_dir_all(root).unwrap();
let root = setup();
let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();
capture.0.lock().unwrap().records = MAX_RECORDS;
capture.record(1, b"{}");
drop(capture);
assert!(
replay_capture(&root.join("attempt-1.capture"), 0)
.await
.is_err()
);
assert!(replay::records(&[4, 2, 0, 0, 0, 0, 0, 0, 0, 1]).is_err());
fs::remove_dir_all(root).unwrap();
}
#[tokio::test]
async fn replay_uses_final_head_and_rejects_paths_without_decoder_equivalence() {
let body = b"{}";
let response = b"data: {\"id\":\"resp\",\"model\":\"physical\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"resp\",\"model\":\"physical\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
let mut mismatches = Vec::new();
for native in [false, true] {
for (heads, invalid) in [
(&[200u16][..], None),
(&[103u16, 200][..], None),
(&[103, 200, 200][..], Some("multiple final statuses")),
(&[429][..], Some("unsupported_capture_path")),
(&[103, 429][..], Some("unsupported_capture_path")),
(&[503][..], Some("unsupported_capture_path")),
(&[103, 503][..], Some("unsupported_capture_path")),
(&[103][..], Some("missing final status")),
(
&[200, 103][..],
Some("informational head after final status"),
),
] {
let root = setup();
let mut profile = profile();
if native {
profile.ingress_protocol = profile.capability.upstream_protocol;
}
let capture = Capture::open(&root, &profile, None, body.len(), true).unwrap();
capture.record(1, body);
capture.record(2, &[]);
for head in heads {
capture.record(3, &head.to_le_bytes());
}
capture.record(4, &response[..23]);
capture.record(4, &response[23..]);
capture.record(5, &[]);
drop(capture);
for size in [0, 1, 4096] {
let result = replay_capture(&root.join("attempt-1.capture"), size).await;
let expected = invalid.or(native.then_some("unsupported_capture_path"));
let matched = match (&result, expected) {
(Err(actual), Some(expected)) => *actual == expected,
(Ok(actual), None) => actual["decoder"]["result"] == "accepted",
_ => false,
};
if !matched {
mismatches.push(format!(
"heads={heads:?} native={native} chunk={size} expected={expected:?} actual={result:?}"
));
}
}
fs::remove_dir_all(root).unwrap();
}
}
assert!(mismatches.is_empty(), "{}", mismatches.join("\n"));
}
#[test]
fn sealed_capture_correlation_accepts_accessible_directory_and_file_modes() {
let root = setup();
let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();
capture.record(1, b"{}");
capture.record(2, &[]);
capture.failed();
let path = root.join("attempt-1.capture");
let initial = fs::read(&path).unwrap();
fs::set_permissions(&root, fs::Permissions::from_mode(0o755)).unwrap();
fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap();
PendingCapture(Arc::downgrade(&capture.0)).promoted(promoted_correlation(1));
assert_ne!(fs::read(&path).unwrap(), initial);
assert!(!capture.0.lock().unwrap().active);
assert_eq!(
fs::metadata(&root).unwrap().permissions().mode() & 0o777,
0o755
);
drop(capture);
fs::remove_dir_all(root).unwrap();
}