Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.
Rust ai-agentsai-gatewayclaude-codecodexcoding-agentcost-optimizationdeepseekllm-gatewayllm-routerlocal-firstmodel-routingrusttask-routingtauri
//! The failure entry of one accepted run: an early failure is durable before it is reported,
//! and a failure that could not be recorded claims nothing.
use std::sync::atomic::{AtomicBool, Ordering};
use async_trait::async_trait;
use hiroute_application::publication::versions::ExactPlanVersionPort;
use hiroute_diagnostics::runtime::{DiagnosticRuntime, RuntimeConfig};
use hiroute_domain::delegation::{
DELEGATION_RUN_CONFIGURATION_VERSION_V1, DelegationAcceptanceV1, DelegationCancelReceiptV1,
DelegationCheckpointV1, DelegationRunConfigurationV1, WorkerExecutionIntentV1, WorkerNetworkV1,
WorkerPermissionPolicyV1, WorkerToolV1, WorkspaceAccessV1,
};
use hiroute_domain::{
AgentPlanId, CanonicalDigest, OperationId, PlanExecutionRef, PlanHeadV1, PlanVersionError,
PlanVersionV1, VersionOwnerRefV1, VersionReservationV1, WorkspaceId,
};
use hiroute_observation::DigestAuthority;
use super::*;
use crate::delegation::platform::{
ReadyWorker, WorkerObservation, WorkerPlatformCapabilities, WorkerProcessIdentity,
WorkerStopEvidence,
};
/// One accepted run and nothing else: every failure below happens before a Worker exists.
struct MemoryRuntime {
run: Mutex<DelegationRunV1>,
checkpoints_fail: AtomicBool,
}
impl MemoryRuntime {
fn accepted() -> Self {
Self {
run: Mutex::new(accepted_run()),
checkpoints_fail: AtomicBool::new(false),
}
}
fn state(&self) -> (hiroute_domain::delegation::RunStateV1, u64) {
let run = self.run.lock().unwrap();
(run.progress.state, run.progress.revision)
}
fn unrecordable(&self) {
self.checkpoints_fail.store(true, Ordering::SeqCst);
}
}
impl DelegationRuntimePort for MemoryRuntime {
fn task(
&self,
_: &WorkspaceId,
_: &str,
) -> Result<Option<DelegationTaskV1>, DelegationErrorV1> {
Ok(None)
}
fn run(
&self,
workspace: &WorkspaceId,
run_id: &str,
) -> Result<Option<DelegationRunV1>, DelegationErrorV1> {
let run = self
.run
.lock()
.map_err(|_| DelegationErrorV1::StorageUnavailable)?;
Ok((run.workspace_id == *workspace && run.run_id == run_id).then(|| run.clone()))
}
fn find_submission(
&self,
_: &WorkspaceId,
_: bool,
_: &str,
) -> Result<Option<DelegationRunV1>, DelegationErrorV1> {
Ok(None)
}
fn accept(&self, _: &DelegationAcceptanceV1) -> Result<DelegationRunV1, DelegationErrorV1> {
Err(DelegationErrorV1::Conflict)
}
fn checkpoint(
&self,
workspace: &WorkspaceId,
run_id: &str,
expected_revision: u64,
_: &str,
event: &DelegationCheckpointV1,
) -> Result<DelegationRunV1, DelegationErrorV1> {
if self.checkpoints_fail.load(Ordering::SeqCst) {
return Err(DelegationErrorV1::StorageUnavailable);
}
let mut run = self
.run
.lock()
.map_err(|_| DelegationErrorV1::StorageUnavailable)?;
if run.workspace_id != *workspace
|| run.run_id != run_id
|| run.progress.revision != expected_revision
{
return Err(DelegationErrorV1::Conflict);
}
match event {
DelegationCheckpointV1::Progress { event } => run.progress.advance(*event)?,
_ => return Err(DelegationErrorV1::PermissionDenied),
}
run.progress.revision = expected_revision
.checked_add(1)
.ok_or(DelegationErrorV1::Conflict)?;
Ok(run.clone())
}
fn request_cancel(
&self,
_: &WorkspaceId,
_: &str,
_: &OperationId,
_: &str,
) -> Result<DelegationCancelReceiptV1, DelegationErrorV1> {
Err(DelegationErrorV1::CapabilityUnavailable)
}
fn unreconciled(&self, _: &WorkspaceId) -> Result<Vec<DelegationRunV1>, DelegationErrorV1> {
Ok(Vec::new())
}
fn set_resume_materials(
&self,
_: &WorkspaceId,
_: &str,
_: &str,
_: u64,
_: &[DelegationBodyRefV1],
_: &[String],
) -> Result<(), DelegationErrorV1> {
Err(DelegationErrorV1::CapabilityUnavailable)
}
}
/// No exact plan version is ever published here, so a run that reaches that step fails before
/// a Worker journal exists.
struct NoVersions;
impl ExactPlanVersionPort for NoVersions {
fn current_plan(
&self,
_: &hiroute_application::publication::admission::AdmissionGuard<'_>,
_: &AgentPlanId,
) -> Result<PlanHeadV1, PlanVersionError> {
Err(PlanVersionError::Unavailable)
}
fn acquire_exact(
&self,
_: &hiroute_application::publication::admission::AdmissionGuard<'_>,
_: &VersionReservationV1,
) -> Result<Arc<PlanVersionV1>, PlanVersionError> {
Err(PlanVersionError::Unavailable)
}
fn lookup_exact(&self, _: &PlanExecutionRef) -> Result<Arc<PlanVersionV1>, PlanVersionError> {
Err(PlanVersionError::Unavailable)
}
fn renew(
&self,
_: &WorkspaceId,
_: &VersionOwnerRefV1,
_: i64,
) -> Result<(), PlanVersionError> {
Err(PlanVersionError::Unavailable)
}
fn release(&self, _: &WorkspaceId, _: &VersionOwnerRefV1) -> Result<(), PlanVersionError> {
Ok(())
}
fn reconcile(
&self,
_: &WorkspaceId,
_: &[VersionReservationV1],
_: i64,
) -> Result<(), PlanVersionError> {
Ok(())
}
}
/// No Worker can ever be launched: reaching this backend fails the test's premise loudly.
struct UnusedPlatform;
#[async_trait]
impl WorkerPlatformPort for UnusedPlatform {
fn capabilities(
&self,
_: &CandidateWorkerProfile,
) -> Result<WorkerPlatformCapabilities, DelegationErrorV1> {
Err(DelegationErrorV1::CapabilityUnavailable)
}
async fn launch(&self, _: WorkerLaunchRequest) -> Result<ReadyWorker, DelegationErrorV1> {
Err(DelegationErrorV1::CapabilityUnavailable)
}
async fn observe(
&self,
_: &WorkerProcessIdentity,
) -> Result<WorkerObservation, DelegationErrorV1> {
Err(DelegationErrorV1::CapabilityUnavailable)
}
async fn terminate(
&self,
_: &WorkerProcessIdentity,
_: u64,
) -> Result<WorkerStopEvidence, DelegationErrorV1> {
Err(DelegationErrorV1::CapabilityUnavailable)
}
}
fn accepted_run() -> DelegationRunV1 {
DelegationRunV1 {
workspace_id: WorkspaceId::default(),
task_id: "task/one".into(),
run_id: "run/one".into(),
ordinal: 1,
continued_from: None,
idempotency_key: "start/one".into(),
request_digest: CanonicalDigest::of_bytes(b"request"),
admission_sequence: 1,
accepted_at_ms: Some(1),
execution_owner_ref: "delegation-run/run/one".into(),
lease_id: "lease/one".into(),
daemon_epoch: "epoch".into(),
permit_id: "run-config/run/one".into(),
permit_generation: 1,
configuration: DelegationRunConfigurationV1 {
format_version: DELEGATION_RUN_CONFIGURATION_VERSION_V1,
scope_id: "run-config/run/one".into(),
generation: 1,
canonical_workspace_path: "/workspace".into(),
permission_policy: WorkerPermissionPolicyV1::ApproveAll,
},
execution: WorkerExecutionIntentV1 {
root_identity: "root".into(),
access: WorkspaceAccessV1::TrustedNative,
tools: vec![WorkerToolV1::Read],
network: WorkerNetworkV1::Allowed,
duration_ms: 30_000,
delegation_depth: 1,
},
deadline_ms: 30_000,
lease_revoked: false,
launch_nonce: "launch/one".into(),
process: None,
session: None,
progress: Default::default(),
stop_evidence: None,
result_body: None,
result_incomplete: false,
}
}
fn diagnostic_report(root: &std::path::Path) -> DiagnosticRuntime {
DiagnosticRuntime::start(RuntimeConfig {
root: root.join("diagnostics"),
role: hiroute_diagnostics::event::ProcessRole::Daemon,
component: hiroute_diagnostics::record::Component::Daemon,
parent_session_id: None,
level_override: Some(hiroute_diagnostics::level::DiagnosticLevel::Debug),
})
}
fn executor(
runtime: Arc<MemoryRuntime>,
observation: Arc<LocalObservationStore>,
safety: Arc<RunSafetyProjection>,
) -> DelegationRunExecutor {
DelegationRunExecutor::new(
runtime,
Arc::new(NoVersions),
Arc::new(SharedAdmissionGate::new()),
safety,
observation,
Arc::new(DelegationRunAuthority::default()),
Arc::new(DelegationCancellationDispatcher::default()),
Arc::new(crate::delegation::finalization::DelegationFinalization::default()),
Arc::new(UnusedPlatform),
Arc::new(UnavailableWorkerProfileSource),
)
}
/// Drive the executor's own failure entry, exactly as `wake` does on its worker thread.
fn run_failure_entry(executor: &DelegationRunExecutor, context: &DiagnosticContext) {
let handle = Builder::new_current_thread().enable_all().build().unwrap();
handle.block_on(executor.execute(
context,
&WorkspaceId::default(),
"run/one",
PathBuf::from("/workspace"),
));
}
fn task_lifecycles(log: &str) -> Vec<serde_json::Value> {
log.lines()
.map(|line| serde_json::from_str::<serde_json::Value>(line).expect("record is JSON"))
.filter_map(|record| {
let event = record.get("event")?.get("task_lifecycle")?.clone();
Some(serde_json::json!({"event": event, "span_id": record["span_id"]}))
})
.collect()
}
fn fixture(
temporary: &crate::test_support::PrivateTempDir,
seed: u8,
) -> (DiagnosticRuntime, Arc<MemoryRuntime>, DelegationRunExecutor) {
let report = diagnostic_report(temporary.path());
let runtime = Arc::new(MemoryRuntime::accepted());
let observation = Arc::new(
LocalObservationStore::open(
temporary.path().join("observation"),
DigestAuthority::new([seed; 32]),
)
.unwrap(),
);
let safety = Arc::new(
RunSafetyProjection::new(Arc::new(SharedAdmissionGate::new()), "epoch".into()).unwrap(),
);
safety.finish_startup_recovery();
let executor = executor(Arc::clone(&runtime), observation, safety);
(report, runtime, executor)
}
/// R6: a failure before a run journal exists records the real durable terminal first, then
/// reports that same terminal on this run's own context with the authoritative task/run
/// tokens; a second pass over the now-terminal run adds nothing.
#[test]
fn an_early_failure_records_one_terminal_on_the_run_context() {
let temporary = crate::test_support::private_tempdir();
let (report, runtime, executor) = fixture(&temporary, 9);
let context = report.port().handle().root_context();
let expected_task = serde_json::to_value(context.token(CorrelationDomain::Task, "task/one"))
.expect("task token");
let expected_run =
serde_json::to_value(context.token(CorrelationDomain::Run, "run/one")).expect("run token");
let expected_span = serde_json::to_value(context.span_id()).expect("span");
run_failure_entry(&executor, &context);
assert_eq!(
runtime.state(),
(hiroute_domain::delegation::RunStateV1::Failed, 2),
"the durable state is the authority for the terminal"
);
// The run is terminal now: the same entry point cannot record a second terminal.
run_failure_entry(&executor, &context);
assert_eq!(runtime.state().1, 2);
report.shutdown();
let log = std::fs::read_to_string(temporary.path().join("diagnostics/daemon/current.jsonl"))
.expect("read the diagnostic log");
let terminals = task_lifecycles(&log);
assert_eq!(terminals.len(), 1, "exactly one terminal: {log}");
let terminal = &terminals[0];
assert_eq!(terminal["event"]["phase"], "end");
assert_eq!(terminal["event"]["state"], "failed");
assert_eq!(
terminal["event"]["task"], expected_task,
"the authoritative task token of this run"
);
assert_eq!(terminal["event"]["run"], expected_run);
assert_eq!(
terminal["span_id"], expected_span,
"the terminal belongs to the same context as the run's other events"
);
assert_ne!(
terminal["event"]["task"], terminal["event"]["run"],
"the two domains never share one token"
);
assert!(
!log.contains("run/one"),
"the raw run id never reaches a record"
);
}
/// R6: a failure whose checkpoint cannot be written, and a run that already left acceptance,
/// are both left unexplained in the log instead of being reported as launch failures.
#[test]
fn an_early_failure_that_cannot_be_recorded_claims_nothing() {
let temporary = crate::test_support::private_tempdir();
let (report, runtime, executor) = fixture(&temporary, 10);
runtime.unrecordable();
let context = report.port().handle().root_context();
run_failure_entry(&executor, &context);
assert_eq!(
runtime.state(),
(hiroute_domain::delegation::RunStateV1::Accepted, 1),
"a failed checkpoint changes nothing"
);
// A run that already left acceptance is a conflict, not a launch failure.
{
let mut run = runtime.run.lock().unwrap();
run.progress.advance(RunEventV1::Preparing).unwrap();
}
run_failure_entry(&executor, &context);
assert_eq!(
runtime.state(),
(hiroute_domain::delegation::RunStateV1::Preparing, 2),
"no checkpoint was written for the conflict"
);
report.shutdown();
let log = std::fs::read_to_string(temporary.path().join("diagnostics/daemon/current.jsonl"))
.expect("read the diagnostic log");
assert!(
task_lifecycles(&log).is_empty(),
"no terminal is claimed for a failure that was not recorded: {log}"
);
}
#[test]
fn cancel_before_executor_preparation_is_cancelled_without_launching() {
let temporary = crate::test_support::private_tempdir();
let (report, runtime, executor) = fixture(&temporary, 11);
{
let mut run = runtime.run.lock().unwrap();
run.progress.advance(RunEventV1::CancelRequested).unwrap();
run.lease_revoked = true;
}
let context = report.port().handle().root_context();
run_failure_entry(&executor, &context);
let run = runtime.run.lock().unwrap();
assert_eq!(
run.progress.state,
hiroute_domain::delegation::RunStateV1::Cancelled
);
assert!(run.progress.workspace_releasable());
assert!(run.process.is_none());
assert!(!run.progress.prompt_may_have_executed);
drop(run);
run_failure_entry(&executor, &context);
report.shutdown();
let log =
std::fs::read_to_string(temporary.path().join("diagnostics/daemon/current.jsonl")).unwrap();
let terminals = task_lifecycles(&log);
assert!(
terminals.is_empty(),
"the cancel transaction already settled the run"
);
}