Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.
Rust ai-agentsai-gatewayclaude-codecodexcoding-agentcost-optimizationdeepseekllm-gatewayllm-routerlocal-firstmodel-routingrusttask-routingtauri
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use hiroute_application_api::{
DelegationTaskInputV1, WORKER_CONTINUE_SCHEMA_V1, WorkerContinueRequestV1,
};
use hiroute_domain::WorkspaceId;
use hiroute_domain::delegation::{
DELEGATION_NATIVE_ROOT_MARKER_FILE_V1, DelegationBodyRefV1, DelegationCheckpointV1,
DelegationNativeCleanupFailureKindV1, DelegationNativeRootReadyV1, DelegationNativeRootStateV1,
DelegationProcessBindingV1, DelegationRuntimePort, DelegationSessionBindingV1, RunEventV1,
RunProcessObservationV1, RunStateV1, RunStopEvidenceV1, RunStopScopeV1,
};
use hiroute_observation::managed_text::{
ManagedTextInput, ManagedTextNativeCleanup, ManagedTextPurpose, ManagedTextScope, RETENTION_MS,
};
use super::*;
use crate::control::runtime::ProductionControlRuntime;
use crate::delegation::content::read_required_body;
use crate::delegation::profile::{SessionRootUse, TaskSessionRoot, native_history};
const BLOCKED_TEST: &str = "control::runtime::delegation_maintenance::tests::blocked_native_deletion_remains_claimed_and_unacknowledged";
const SUCCESS_TEST: &str = "control::runtime::delegation_maintenance::tests::owned_native_deletion_commits_removed_before_exact_ack";
#[derive(Clone, Copy)]
enum BlockedFixture {
ReplacedIdentity,
MarkerFinalizationInterrupted,
}
struct CleanupFixture {
runtime: ProductionControlRuntime,
_directory: tempfile::TempDir,
workspace_id: WorkspaceId,
task_id: String,
native_root: PathBuf,
job: ManagedTextNativeCleanup,
}
#[test]
fn blocked_native_deletion_remains_claimed_and_unacknowledged() {
if crate::test_support::isolated_agent_home(BLOCKED_TEST) {
return;
}
assert_blocked_cleanup(BlockedFixture::ReplacedIdentity, "identity");
assert_blocked_cleanup(
BlockedFixture::MarkerFinalizationInterrupted,
"marker-interrupted",
);
}
fn assert_blocked_cleanup(fixture: BlockedFixture, suffix: &str) {
let cleanup = cleanup_fixture(suffix);
let preserved_entry = match fixture {
BlockedFixture::ReplacedIdentity => {
std::fs::remove_file(
cleanup
.native_root
.join(DELEGATION_NATIVE_ROOT_MARKER_FILE_V1),
)
.unwrap();
std::fs::remove_dir(&cleanup.native_root).unwrap();
private_directory(&cleanup.native_root);
true
}
BlockedFixture::MarkerFinalizationInterrupted => {
std::fs::remove_file(
cleanup
.native_root
.join(DELEGATION_NATIVE_ROOT_MARKER_FILE_V1),
)
.unwrap();
false
}
};
if preserved_entry {
std::fs::write(cleanup.native_root.join("must-remain"), b"preserved").unwrap();
}
let mut remaining = DELETE_BUDGET;
cleanup
.runtime
.adapter
.process_native_root_for_job(&cleanup.job, 101, &mut remaining)
.unwrap();
let blocked = DelegationRuntimePort::native_root(
cleanup.runtime.adapter.as_ref(),
&cleanup.workspace_id,
&cleanup.task_id,
)
.unwrap()
.unwrap();
assert_eq!(blocked.root.state, DelegationNativeRootStateV1::Deleting);
let failure = blocked.root.last_cleanup_failure.unwrap();
assert_eq!(
failure.kind,
DelegationNativeCleanupFailureKindV1::IdentityMismatch
);
assert_eq!(failure.attempted_at_ms, 101);
assert_eq!(failure.attempts, 1);
assert_eq!(remaining, DELETE_BUDGET);
assert_eq!(
cleanup
.runtime
.observation
.managed_text_pending_native_cleanup_page(None, 10)
.unwrap(),
vec![cleanup.job]
);
assert!(cleanup.native_root.exists());
if preserved_entry {
assert_eq!(
std::fs::read(cleanup.native_root.join("must-remain")).unwrap(),
b"preserved"
);
}
}
#[test]
fn owned_native_deletion_commits_removed_before_exact_ack() {
if crate::test_support::isolated_agent_home(SUCCESS_TEST) {
return;
}
for mode in [0o700, 0o755] {
assert_owned_cleanup(mode);
}
}
fn assert_owned_cleanup(mode: u32) {
let cleanup = cleanup_fixture(&format!("success-{mode}"));
std::fs::set_permissions(&cleanup.native_root, std::fs::Permissions::from_mode(mode)).unwrap();
std::fs::write(cleanup.native_root.join("history.jsonl"), b"native history").unwrap();
let mut remaining = DELETE_BUDGET;
cleanup
.runtime
.adapter
.process_native_root_for_job(&cleanup.job, 101, &mut remaining)
.unwrap();
let removed = DelegationRuntimePort::native_root(
cleanup.runtime.adapter.as_ref(),
&cleanup.workspace_id,
&cleanup.task_id,
)
.unwrap()
.unwrap();
assert_eq!(removed.root.state, DelegationNativeRootStateV1::Removed);
assert!(removed.root.last_cleanup_failure.is_none());
assert!(remaining < DELETE_BUDGET);
assert!(!cleanup.native_root.exists());
assert!(
cleanup
.runtime
.observation
.managed_text_pending_native_cleanup_page(None, 10)
.unwrap()
.is_empty()
);
}
fn cleanup_fixture(suffix: &str) -> CleanupFixture {
let directory = tempfile::tempdir().unwrap();
let runtime = maintenance_runtime(directory.path());
let task_id = format!("cleanup-{suffix}-task");
let run_id = format!("cleanup-{suffix}-run");
let scope = ManagedTextScope {
workspace_id: WorkspaceId::default(),
task_id: task_id.clone(),
run_id: run_id.clone(),
};
let input = ManagedTextInput {
scope: scope.clone(),
purpose: ManagedTextPurpose::Result,
source_event_id: format!("cleanup-{suffix}-body"),
source_revision: 1,
original_created_at_ms: 10,
import_origin: None,
};
let reference = runtime.observation.managed_text_put(&input, 100).unwrap();
runtime
.observation
.managed_text_append(&scope, &reference, 0, b"result", 100)
.unwrap();
let reference = runtime
.observation
.managed_text_finish(&scope, &reference, 1, 100)
.unwrap();
let body = DelegationBodyRefV1 {
opaque_id: reference.opaque_id,
scope_run_id: run_id.clone(),
visibility_generation: reference.visibility_generation,
original_retention_deadline_ms: reference.original_retention_deadline_ms,
};
let acceptance = super::super::tests::worker_list_acceptance(&task_id, &run_id, body, 10);
let accepted = DelegationRuntimePort::accept(runtime.adapter.as_ref(), &acceptance).unwrap();
let native_base = std::fs::canonicalize(directory.path())
.unwrap()
.join("native");
private_directory(&native_base);
let creating = DelegationRuntimePort::native_root(
runtime.adapter.as_ref(),
&accepted.workspace_id,
&accepted.task_id,
)
.unwrap()
.unwrap();
let session = crate::delegation::profile::TaskSessionRoot::prepare(
&native_base,
&accepted.workspace_id,
&creating.root.workspace_root_identity,
&accepted.task_id,
creating.root.harness,
crate::delegation::profile::SessionRootUse::New,
)
.unwrap();
let identity = session
.create_ownership_marker(&native_base, &creating.root)
.unwrap();
let native_root = session.path().to_owned();
DelegationRuntimePort::commit_native_root_ready(
runtime.adapter.as_ref(),
&DelegationNativeRootReadyV1 {
workspace_id: accepted.workspace_id.clone(),
task_id: accepted.task_id.clone(),
root_generation: creating.root.root_generation,
creation_nonce: creating.root.creation_nonce,
managed_base_path: native_base.to_str().unwrap().into(),
filesystem_identity: identity,
},
)
.unwrap();
DelegationRuntimePort::checkpoint(
runtime.adapter.as_ref(),
&accepted.workspace_id,
&accepted.run_id,
accepted.progress.revision,
&format!("cleanup-{suffix}-stopped"),
&DelegationCheckpointV1::Progress {
event: RunEventV1::LaunchFailedBeforeSpawn,
},
)
.unwrap();
let preview = runtime
.observation
.managed_text_delete_preview(&scope, 100, 100)
.unwrap();
let deleted = runtime
.observation
.managed_text_delete_apply(&preview)
.unwrap();
assert!(deleted.native_gc_pending);
let mut jobs = runtime
.observation
.managed_text_pending_native_cleanup_page(None, 10)
.unwrap();
assert_eq!(jobs.len(), 1);
CleanupFixture {
runtime,
_directory: directory,
workspace_id: accepted.workspace_id,
task_id: accepted.task_id,
native_root,
job: jobs.pop().unwrap(),
}
}
#[test]
fn deleted_worker_body_revokes_continue_and_cleans_only_owned_metadata() {
if crate::test_support::isolated_agent_home(
"control::runtime::delegation_maintenance::tests::deleted_worker_body_revokes_continue_and_cleans_only_owned_metadata",
) {
return;
}
assert_borrowed_retention(RetentionAction::Delete);
}
#[test]
fn expired_worker_body_revokes_continue_and_cleans_only_owned_metadata() {
if crate::test_support::isolated_agent_home(
"control::runtime::delegation_maintenance::tests::expired_worker_body_revokes_continue_and_cleans_only_owned_metadata",
) {
return;
}
assert_borrowed_retention(RetentionAction::Expire);
}
enum RetentionAction {
Delete,
Expire,
}
struct RetainedTask {
reference: ManagedTextRef,
session: TaskSessionRoot,
native_session_id: String,
}
// These are retention-consumer fixtures: lifecycle checkpoints stand in for a completed Worker.
// They exercise the real body barrier and cleanup consumer, not a public sessions.delete API.
fn assert_borrowed_retention(action: RetentionAction) {
let directory = tempfile::tempdir().unwrap();
let runtime = maintenance_runtime(directory.path());
let base = std::fs::canonicalize(directory.path()).unwrap();
for name in ["native", "home", "workspace"] {
private_directory(&base.join(name));
}
let home = base.join("home");
for name in [".codex", ".codex/skills", ".codex/sessions"] {
private_directory(&home.join(name));
}
let shared_files: Vec<_> = [
("user-note.txt", b"user-owned home bytes".as_slice()),
(".codex/config.toml", b"user-owned configuration".as_slice()),
(".codex/skills/skill.md", b"user-owned skill".as_slice()),
(
".codex/sessions/target.jsonl",
b"opaque target native history".as_slice(),
),
(
".codex/sessions/neighbor.jsonl",
b"opaque neighbor native history".as_slice(),
),
]
.into_iter()
.map(|(relative, bytes)| {
let path = home.join(relative);
std::fs::write(&path, bytes).unwrap();
(path, bytes.to_vec())
})
.collect();
let now = i64::try_from(
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_millis(),
)
.unwrap();
let target = retained_borrowed_task(&runtime, &base, "target", now);
// A later body deadline keeps the neighbor visible during target expiry.
let neighbor = retained_borrowed_task(&runtime, &base, "neighbor", now + 1_000);
let workspace = &target.reference.scope.workspace_id;
let target_id = &target.reference.scope.task_id;
let before = DelegationRuntimePort::task(runtime.adapter.as_ref(), workspace, target_id)
.unwrap()
.unwrap();
assert!(before.resume_until_ms > now as u64);
assert!(
runtime
.adapter
.required_bodies_visible(&before, now)
.unwrap()
);
assert_eq!(
read_retained_body(&runtime, &target, now).unwrap(),
b"retained Worker body"
);
let neighbor_before = DelegationRuntimePort::task(
runtime.adapter.as_ref(),
workspace,
&neighbor.reference.scope.task_id,
)
.unwrap()
.unwrap();
let neighbor_metadata = owned_metadata_bytes(neighbor.session.path());
let effect_time = match action {
RetentionAction::Expire => now + RETENTION_MS,
RetentionAction::Delete => now + 1_001,
};
if matches!(action, RetentionAction::Expire) {
assert_eq!(
runtime
.observation
.managed_text_expire_pending(effect_time, 10)
.unwrap(),
1
);
} else {
let preview = runtime
.observation
.managed_text_delete_preview(&target.reference.scope, effect_time, effect_time)
.unwrap();
assert!(
runtime
.observation
.managed_text_delete_apply(&preview)
.unwrap()
.native_gc_pending
);
}
// A retained native binding cannot restore text hidden by HiRoute's body authority, even
// before cleanup removes the owned metadata. This is the same reader Continue invokes.
target
.session
.verify_native_session(&target.native_session_id)
.unwrap();
assert_eq!(
read_retained_body(&runtime, &target, effect_time),
Err(DelegationErrorV1::ResumeUnavailable)
);
assert_eq!(
runtime
.observation
.managed_text_pending_native_cleanup_page(None, 10)
.unwrap()
.len(),
1
);
runtime.adapter.scan_task_maintenance(effect_time).unwrap();
runtime.adapter.scan_native_cleanup(effect_time).unwrap();
let after = DelegationRuntimePort::task(runtime.adapter.as_ref(), workspace, target_id)
.unwrap()
.unwrap();
assert_eq!(after.resume_until_ms, 0);
assert!(after.title.is_none());
// Check the closed admission path after revocation. This fixture does not publish a Plan
// or prove a successful Continue; the body reader above covers the pre-cleanup barrier.
let request = WorkerContinueRequestV1 {
schema: WORKER_CONTINUE_SCHEMA_V1.into(),
task_id: target_id.clone(),
expected_latest_run_id: target.reference.scope.run_id.clone(),
cwd: None,
permission_policy: Default::default(),
run_timeout_secs: 60,
input: DelegationTaskInputV1 {
goal: "Continue the retained task".into(),
context: String::new(),
constraints: String::new(),
acceptance_criteria: String::new(),
},
submission_key: "retention-continue".into(),
};
assert!(matches!(
runtime.adapter.continue_worker(&request),
Err(DelegationErrorV1::ResumeUnavailable)
));
let removed =
DelegationRuntimePort::native_root(runtime.adapter.as_ref(), workspace, target_id)
.unwrap()
.unwrap();
assert_eq!(removed.root.state, DelegationNativeRootStateV1::Removed);
assert!(!target.session.path().exists());
assert!(
runtime
.observation
.managed_text_pending_native_cleanup_page(None, 10)
.unwrap()
.is_empty()
);
assert!(
DelegationRuntimePort::pending_continuation_releases(runtime.adapter.as_ref(), 10)
.unwrap()
.is_empty()
);
assert_eq!(
DelegationRuntimePort::task(
runtime.adapter.as_ref(),
workspace,
&neighbor.reference.scope.task_id
)
.unwrap()
.unwrap(),
neighbor_before
);
assert_eq!(
owned_metadata_bytes(neighbor.session.path()),
neighbor_metadata
);
neighbor
.session
.verify_native_session(&neighbor.native_session_id)
.unwrap();
assert_eq!(
read_retained_body(&runtime, &neighbor, effect_time).unwrap(),
b"retained Worker body"
);
for (path, expected) in shared_files {
assert_eq!(
std::fs::read(&path).unwrap(),
expected,
"borrowed file changed: {}",
path.display()
);
}
}
fn retained_borrowed_task(
runtime: &ProductionControlRuntime,
base: &Path,
suffix: &str,
created_at_ms: i64,
) -> RetainedTask {
let task_id = format!("retention-{suffix}-task");
let run_id = format!("retention-{suffix}-run");
let scope = ManagedTextScope {
workspace_id: WorkspaceId::default(),
task_id,
run_id,
};
let reference = runtime
.observation
.managed_text_put(
&ManagedTextInput {
scope: scope.clone(),
purpose: ManagedTextPurpose::Result,
source_event_id: format!("retention-{suffix}-body"),
source_revision: 1,
original_created_at_ms: created_at_ms,
import_origin: None,
},
created_at_ms,
)
.unwrap();
runtime
.observation
.managed_text_append(
&scope,
&reference,
0,
b"retained Worker body",
created_at_ms,
)
.unwrap();
let reference = runtime
.observation
.managed_text_finish(&scope, &reference, 1, created_at_ms)
.unwrap();
let body = DelegationBodyRefV1 {
opaque_id: reference.opaque_id.clone(),
scope_run_id: scope.run_id.clone(),
visibility_generation: reference.visibility_generation,
original_retention_deadline_ms: reference.original_retention_deadline_ms,
};
let mut acceptance = super::super::tests::worker_list_acceptance(
&scope.task_id,
&scope.run_id,
body.clone(),
created_at_ms as u64,
);
acceptance.run.idempotency_key = format!("retention-{suffix}-start");
acceptance.run.request_digest = hiroute_domain::CanonicalDigest::of_bytes(suffix.as_bytes());
acceptance.run.execution_owner_ref = format!("retention-{suffix}-owner");
acceptance.run.lease_id = format!("retention-{suffix}-lease");
acceptance.run.launch_nonce = format!("retention-{suffix}-launch");
acceptance.title_lookup_key = Some(format!("retention-{suffix}-title"));
let mut run = DelegationRuntimePort::accept(runtime.adapter.as_ref(), &acceptance).unwrap();
let creating = DelegationRuntimePort::native_root(
runtime.adapter.as_ref(),
&scope.workspace_id,
&scope.task_id,
)
.unwrap()
.unwrap();
let native_base = base.join("native");
let session = TaskSessionRoot::prepare(
&native_base,
&scope.workspace_id,
&creating.root.workspace_root_identity,
&scope.task_id,
creating.root.harness,
SessionRootUse::New,
)
.unwrap();
session
.bind_borrowed_context(
&base.join("home"),
&base.join("home/.codex"),
&base.join("workspace"),
)
.unwrap();
let identity = session
.create_ownership_marker(&native_base, &creating.root)
.unwrap();
DelegationRuntimePort::commit_native_root_ready(
runtime.adapter.as_ref(),
&DelegationNativeRootReadyV1 {
workspace_id: scope.workspace_id.clone(),
task_id: scope.task_id.clone(),
root_generation: creating.root.root_generation,
creation_nonce: creating.root.creation_nonce,
managed_base_path: native_base.to_str().unwrap().into(),
filesystem_identity: identity,
},
)
.unwrap();
let native_session_id = format!("retention-{suffix}-native");
let checkpoints = [
DelegationCheckpointV1::Progress {
event: RunEventV1::Preparing,
},
DelegationCheckpointV1::ProcessSpawned {
binding: DelegationProcessBindingV1 {
launch_nonce: run.launch_nonce.clone(),
handle_id: format!("retention-{suffix}-process"),
creation_identity: format!("retention-{suffix}-created"),
},
},
DelegationCheckpointV1::SessionBound {
binding: DelegationSessionBindingV1 {
acp_session_id: native_session_id.clone(),
native_session_id: Some(native_session_id.clone()),
},
},
DelegationCheckpointV1::Progress {
event: RunEventV1::PromptSendIntent,
},
DelegationCheckpointV1::ResultRecorded {
body: Some(body.clone()),
incomplete: false,
},
DelegationCheckpointV1::Progress {
event: RunEventV1::PromptCompleted,
},
DelegationCheckpointV1::ProcessStopped {
evidence: RunStopEvidenceV1 {
scope: RunStopScopeV1::ProcessGroup,
observation: RunProcessObservationV1::Exited { code: Some(0) },
scope_stopped: true,
residual_unknown: false,
},
},
];
for (index, checkpoint) in checkpoints.iter().enumerate() {
run = DelegationRuntimePort::checkpoint(
runtime.adapter.as_ref(),
&scope.workspace_id,
&scope.run_id,
run.progress.revision,
&format!("retention-{suffix}-{index}"),
checkpoint,
)
.unwrap();
}
assert_eq!(run.progress.state, RunStateV1::Succeeded);
assert!(run.progress.workspace_releasable());
let history =
native_history(session.path(), creating.root.harness, &native_session_id).unwrap();
DelegationRuntimePort::set_resume_materials(
runtime.adapter.as_ref(),
&scope.workspace_id,
&scope.task_id,
&scope.run_id,
reference.original_retention_deadline_ms as u64,
&[body],
&history,
)
.unwrap();
RetainedTask {
reference,
session,
native_session_id,
}
}
fn read_retained_body(
runtime: &ProductionControlRuntime,
task: &RetainedTask,
now_ms: i64,
) -> Result<Vec<u8>, DelegationErrorV1> {
read_required_body(
&runtime.observation,
&task.reference.scope,
&task.reference,
now_ms,
1_024,
|_| Ok(()),
)
}
fn owned_metadata_bytes(root: &Path) -> Vec<(PathBuf, Vec<u8>)> {
let mut files: Vec<_> = std::fs::read_dir(root)
.unwrap()
.map(|entry| {
let path = entry.unwrap().path();
let bytes = std::fs::read(&path).unwrap();
(path, bytes)
})
.collect();
files.sort();
files
}
fn maintenance_runtime(directory: &Path) -> ProductionControlRuntime {
let storage = directory.join("storage");
private_directory(&storage);
let mut runtime = ProductionControlRuntime::open_with_release_catalog(
&storage,
crate::release_catalog::fixture_catalog(),
)
.unwrap();
runtime._observation_maintenance.take();
for _ in 0..100 {
if !runtime.observation.maintenance_status().0 {
break;
}
std::thread::sleep(Duration::from_millis(2));
}
assert!(!runtime.observation.maintenance_status().0);
runtime
}
fn private_directory(path: &Path) {
std::fs::create_dir(path).unwrap();
std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)).unwrap();
}