Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.
Rust ai-agentsai-gatewayclaude-codecodexcoding-agentcost-optimizationdeepseekllm-gatewayllm-routerlocal-firstmodel-routingrusttask-routingtauri
use std::collections::VecDeque;
use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener};
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::{Duration, Instant};
use hiroute_diagnostics::event::{
CpaFailureCode, CpaLifecycle, CpaPhase, CpaStage, CpaStageKind, CpaStageOutcome,
DiagnosticEvent,
};
use hiroute_diagnostics::runtime::DiagnosticsPort;
use hiroute_integrations::{
CpaAccountMaterializationV1, CpaRegisteredSourceV1, CpaSupervisorError, CpaSupervisorPort,
TrustedReleaseCatalog, register_cpa_account,
};
use parking_lot::Mutex;
use crate::BorrowedSubscriptionEvidence;
use crate::MANAGED_CPA_ARTIFACT_VERSION;
use crate::accounts::{
AccountDiscoveryError, AccountSnapshotRecord, CpaControlPlane, StockCpaControlPlane,
};
use crate::artifact::{CpaArtifactError, CpaBinaryLocator, VerifiedCpaBinary};
use crate::borrowed_codex::ManagedAuthLease;
use crate::config::{
InstanceSecrets, SecretText, ensure_private_dir, private_atomic_write, render_managed_config,
validate_private_file,
};
use crate::errors::CpaLifecycleError;
use crate::owner::{OwnerClaim, OwnerLease, OwnerRecord};
use crate::process::{
CpaExit, CpaLaunch, CpaProcessBackend, CpaProcessHandle, StdCpaProcessBackend,
};
use crate::state::{load_account_state, merge_accounts, save_account_state};
mod admission;
mod oauth;
mod recovery;
mod routing_batch;
mod spec;
mod subscriptions;
pub use oauth::{CpaOAuthLogin, CpaOAuthStatus};
pub use routing_batch::CpaRoutingBatch;
use spec::validate_spec;
pub use spec::{CpaRuntimeSpec, RestartPolicy};
pub use subscriptions::CpaSourceManagementState;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum CpaHealth {
Stopped {
last_exit: Option<CpaExit>,
},
Ready {
pid: u32,
address: SocketAddr,
restart_count: u64,
adopted: bool,
},
Crashed {
exit: CpaExit,
restart_count: u64,
},
Unhealthy {
pid: u32,
restart_count: u64,
},
CrashLoop {
last_exit: CpaExit,
restart_count: u64,
},
}
pub struct ManagedCpaRuntime {
pub(crate) spec: CpaRuntimeSpec,
pub(crate) catalog: Arc<TrustedReleaseCatalog>,
locator: Arc<dyn CpaBinaryLocator>,
backend: Arc<dyn CpaProcessBackend>,
control: Arc<dyn CpaControlPlane>,
// Serializes the sole process/auth-file owner, never execution admission.
pub(crate) inner: Mutex<RuntimeInner>,
admission: admission::Admission,
observation: Mutex<admission::Observation>,
health_invalidated: AtomicBool,
pub(crate) epochs: Arc<RuntimeEpochState>,
/// Lifecycle diagnostics; a no-op port keeps library and test hosts unchanged.
diagnostics: DiagnosticsPort,
proxy_environment: crate::proxy_environment::ProxyEnvironment,
}
/// Read-only discovery boundary exposed to Local Control. The returned values contain only
/// catalog-validated source facts and opaque credential references; OAuth material never crosses
/// this interface.
pub trait CpaRegisteredSourcePort: Send + Sync {
/// Best-effort aggregate facts. Exact selections must use provider discovery
/// so a failed sibling cannot turn an unavailable source into a missing one.
fn discover_registered_sources(&self) -> Result<Vec<CpaRegisteredSourceV1>, CpaLifecycleError>;
fn discover_registered_sources_for(
&self,
kind: crate::CpaAccountKind,
) -> Result<Vec<CpaRegisteredSourceV1>, CpaLifecycleError> {
self.discover_registered_sources().map(|sources| {
sources
.into_iter()
.filter(|source| source.source.connector_id == kind.connector_id())
.collect()
})
}
fn begin_routing_batch(&self) -> Result<CpaRoutingBatch<'_>, CpaLifecycleError>;
}
pub(crate) struct RuntimeEpochState {
runtime: AtomicU64,
target: AtomicU64,
}
impl RuntimeEpochState {
fn new() -> Self {
Self {
runtime: AtomicU64::new(0),
target: AtomicU64::new(0),
}
}
pub(crate) fn current(&self) -> (u64, u64) {
(
self.runtime.load(Ordering::Acquire),
self.target.load(Ordering::Acquire),
)
}
pub(crate) fn matches(&self, runtime: u64, target: u64) -> bool {
runtime != 0 && target != 0 && self.current() == (runtime, target)
}
fn advance_runtime(&self) {
self.runtime.fetch_add(1, Ordering::AcqRel);
self.target.fetch_add(1, Ordering::AcqRel);
}
fn advance_target(&self) {
self.target.fetch_add(1, Ordering::AcqRel);
}
}
#[derive(Default)]
pub(crate) struct RuntimeInner {
pub(crate) live: Option<LiveRuntime>,
last_exit: Option<CpaExit>,
crashes: VecDeque<Instant>,
restart_count: u64,
oauth_state: Option<String>,
oauth_callback_submitted: bool,
}
pub(crate) struct LiveRuntime {
lease: OwnerLease,
auth_lease: Option<ManagedAuthLease>,
owner_record: OwnerRecord,
process: Option<Box<dyn CpaProcessHandle>>,
artifact: VerifiedCpaBinary,
pub(crate) address: SocketAddr,
pub(crate) secrets: InstanceSecrets,
pub(crate) accounts: Vec<AccountSnapshotRecord>,
// Persisted account facts cannot establish that this process has finished
// asynchronously registering its account/model inventory.
accounts_validated_for_process: bool,
last_health: Option<Instant>,
published_auth_generation: Option<u64>,
adopted: bool,
// A failed initial start keeps ownership solely to finish cleanup. It cannot
// become execution-ready until that child stops and a normal start commits.
cleanup_only: bool,
}
struct InstanceLayout {
work_dir: PathBuf,
config_path: PathBuf,
capability_path: PathBuf,
accounts_path: PathBuf,
lock_dir: PathBuf,
auth_dir: PathBuf,
}
mod credential_diagnostics;
impl ManagedCpaRuntime {
pub fn managed_kind(&self) -> Option<crate::CpaAccountKind> {
if let Some(kind) = self.spec.managed_oauth {
Some(kind)
} else if self.spec.borrowed_claude_auth.is_some() {
Some(crate::CpaAccountKind::Claude)
} else if self.spec.borrowed_codex_auth.is_some() {
Some(crate::CpaAccountKind::Codex)
} else {
None
}
}
pub fn inspect_subscription_for_check(
&self,
) -> Result<BorrowedSubscriptionEvidence, CpaLifecycleError> {
self.inspect_subscription_with_diagnostics(true)
}
pub fn inspect_subscription(&self) -> Result<BorrowedSubscriptionEvidence, CpaLifecycleError> {
self.inspect_subscription_with_diagnostics(false)
}
pub fn new(
spec: CpaRuntimeSpec,
catalog: Arc<TrustedReleaseCatalog>,
locator: Arc<dyn CpaBinaryLocator>,
) -> Result<Self, CpaLifecycleError> {
Self::with_components(
spec,
catalog,
locator,
Arc::new(StdCpaProcessBackend),
Arc::new(StockCpaControlPlane),
)
}
fn with_components(
spec: CpaRuntimeSpec,
catalog: Arc<TrustedReleaseCatalog>,
locator: Arc<dyn CpaBinaryLocator>,
backend: Arc<dyn CpaProcessBackend>,
control: Arc<dyn CpaControlPlane>,
) -> Result<Self, CpaLifecycleError> {
validate_spec(&spec, &catalog)?;
Ok(Self {
spec,
catalog,
locator,
backend,
control,
inner: Mutex::new(RuntimeInner::default()),
admission: admission::Admission::default(),
observation: Mutex::new(admission::Observation::default()),
health_invalidated: AtomicBool::new(false),
epochs: Arc::new(RuntimeEpochState::new()),
diagnostics: DiagnosticsPort::default(),
proxy_environment: crate::proxy_environment::ProxyEnvironment::capture(
std::env::vars_os(),
),
})
}
/// Record the CPA lifecycle without CPA stdout, command paths or credential locators.
pub fn with_diagnostics(mut self, diagnostics: DiagnosticsPort) -> Self {
self.diagnostics = diagnostics;
self
}
/// Override only the CPA child's proxy environment with a host-owned snapshot.
/// This must be configured before the runtime starts.
pub fn with_proxy_environment(
mut self,
values: impl IntoIterator<Item = (std::ffi::OsString, std::ffi::OsString)>,
) -> Self {
self.proxy_environment = crate::proxy_environment::ProxyEnvironment::capture(values);
if let Some(spec) = self.spec.borrowed_claude_auth.as_mut() {
spec.set_proxy_environment(self.proxy_environment.clone());
}
self
}
fn emit(&self, event: DiagnosticEvent) {
self.diagnostics.handle().try_emit(event);
}
/// One CPA start step: entered before the step blocks, then completed or failed with a
/// stable code. No path, credential or adapter output enters the record.
fn emit_stage(
&self,
stage: CpaStageKind,
outcome: CpaStageOutcome,
elapsed_ms: u64,
restart_generation: u64,
) {
self.emit(DiagnosticEvent::CpaStage(CpaStage {
stage,
elapsed_ms,
restart_generation,
outcome,
}));
}
pub fn start(&self) -> Result<CpaHealth, CpaLifecycleError> {
self.start_expected(None)
}
pub(crate) fn start_expected(
&self,
expected: Option<&BorrowedSubscriptionEvidence>,
) -> Result<CpaHealth, CpaLifecycleError> {
let _scope = self.operation_context().enter();
self.prepare_native_subscription()?;
let mut inner = self.lock_lifecycle()?;
self.start_expected_locked(&mut inner, expected)
}
fn start_expected_locked(
&self,
inner: &mut RuntimeInner,
expected: Option<&BorrowedSubscriptionEvidence>,
) -> Result<CpaHealth, CpaLifecycleError> {
if inner.live.as_ref().is_some_and(|live| live.cleanup_only) {
let live = inner.live.as_mut().expect("retained failed start");
let exit = self.shutdown_live_process(live)?;
live.lease
.release()
.map_err(|_| CpaLifecycleError::OwnerState)?;
inner.live = None;
inner.last_exit = Some(exit);
} else if inner.live.is_some() {
return self.health_locked(inner);
}
let generation = inner.restart_count;
let started = std::time::Instant::now();
self.emit(DiagnosticEvent::CpaLifecycle(CpaLifecycle {
phase: CpaPhase::Start,
elapsed_ms: 0,
exit_code: None,
signal: None,
}));
let artifact_started = std::time::Instant::now();
self.emit_stage(
CpaStageKind::ArtifactLocate,
CpaStageOutcome::Entered,
0,
generation,
);
let artifact = match self.locator.locate() {
Ok(artifact) => {
self.emit_stage(
CpaStageKind::ArtifactLocate,
CpaStageOutcome::Completed,
artifact_started.elapsed().as_millis() as u64,
generation,
);
artifact
}
Err(error) => {
self.emit_stage(
CpaStageKind::ArtifactLocate,
CpaStageOutcome::Failed {
code: locate_failure_code(&error),
},
artifact_started.elapsed().as_millis() as u64,
generation,
);
return Err(error.into());
}
};
self.emit_stage(
CpaStageKind::ArtifactValidate,
CpaStageOutcome::Entered,
0,
generation,
);
if let Err(error) = ensure_supported_artifact(&artifact) {
self.emit_stage(
CpaStageKind::ArtifactValidate,
CpaStageOutcome::Failed {
code: artifact_failure_code(&error),
},
artifact_started.elapsed().as_millis() as u64,
generation,
);
return Err(error);
}
self.emit_stage(
CpaStageKind::ArtifactValidate,
CpaStageOutcome::Completed,
artifact_started.elapsed().as_millis() as u64,
generation,
);
let layout = self.prepare_layout()?;
let nonce = SecretText::generate()?.expose().to_owned();
let mut fresh_record = OwnerRecord::claim(
std::process::id(),
nonce.clone(),
artifact.version().to_string(),
artifact.sha256_hex().to_owned(),
)
.map_err(|_| CpaLifecycleError::OwnerState)?;
fresh_record.proxy_environment_sha256 = Some(self.proxy_environment.digest());
let claim = OwnerLease::acquire(&layout.lock_dir, &fresh_record, nonce.clone())
.map_err(|_| CpaLifecycleError::OwnerState)?;
let mut live = match claim {
OwnerClaim::Acquired(lease) => {
let record = fresh_record.clone();
// The step reports its own real terminal at its own boundary; nothing here
// adds a second one for a step that already ended.
self.start_fresh(inner, lease, record, artifact, &layout, expected)?
}
OwnerClaim::Existing(record) => {
if self.backend.pid_is_running(record.owner_pid)? {
return Err(CpaLifecycleError::AlreadyOwned);
}
if record.cpa_pid != 0 && self.backend.pid_is_running(record.cpa_pid)? {
let transferred = record
.clone()
.transfer(std::process::id(), nonce.clone())
.map_err(|_| CpaLifecycleError::OwnerState)?;
let lease =
OwnerLease::reclaim_stale(&layout.lock_dir, &record, &transferred, nonce)
.map_err(|_| CpaLifecycleError::OwnerState)?;
// Authenticate the orphan before adopting or replacing its launch policy.
self.adopt_orphan(
inner,
lease,
record,
transferred,
artifact,
&layout,
expected,
)?
} else {
let replacement = fresh_record;
let lease =
OwnerLease::reclaim_stale(&layout.lock_dir, &record, &replacement, nonce)
.map_err(|_| CpaLifecycleError::OwnerState)?;
self.start_fresh(inner, lease, replacement, artifact, &layout, expected)?
}
}
};
// Application may have replayed a durable disabled/removed decision before this runtime
// existed. Apply that projection before publishing loaded account state to Attempt.
subscriptions::apply_management_projection(
&mut live.accounts,
&self.admission.snapshot().source_management,
);
let health = ready_health(&live, inner.restart_count);
let mut pending = Some(live);
if let Err(error) = self.commit_lifecycle(|| {
if let Some(live) = pending.as_mut() {
live.cleanup_only = false;
}
inner.live = pending.take();
self.epochs.advance_runtime();
self.health_invalidated.store(false, Ordering::Release);
}) {
if let Some(live) = pending {
self.cleanup_failed_start(inner, live);
}
return Err(error);
}
self.emit(DiagnosticEvent::CpaLifecycle(CpaLifecycle {
phase: CpaPhase::Ready,
elapsed_ms: started.elapsed().as_millis() as u64,
exit_code: None,
signal: None,
}));
Ok(health)
}
pub fn health(&self) -> Result<CpaHealth, CpaLifecycleError> {
// A status read never joins a slow lifecycle operation. Reap a known
// process cheaply when its owner is idle; health probes belong to owner
// maintenance/recovery, not the observation lock.
if let Some(mut inner) = self.inner.try_lock() {
if let Some(live) = inner.live.as_mut()
&& let Some(process) = live.process.as_mut()
&& let Some(exit) = process.try_exit()?
{
live.process = None;
live.accounts_validated_for_process = false;
live.last_health = None;
inner.last_exit = Some(exit);
}
self.publish_observation(&inner);
}
let (observed, sampled_at, sampled_epochs) = {
let snapshot = self.observation.lock();
(
snapshot.health,
snapshot.health_sample,
snapshot.live.as_ref().map(|live| live.epochs),
)
};
if (self.admission.snapshot().subscription_execution_suspended
|| self.health_invalidated.load(Ordering::Acquire)
|| sampled_at.is_none_or(|sample| sample.elapsed() >= Duration::from_secs(5))
|| sampled_epochs.is_none_or(|(runtime, target)| !self.epochs.matches(runtime, target)))
&& let CpaHealth::Ready {
pid, restart_count, ..
} = observed
{
return Ok(CpaHealth::Unhealthy { pid, restart_count });
}
Ok(observed)
}
pub fn shutdown(&self) -> Result<CpaExit, CpaLifecycleError> {
let started = std::time::Instant::now();
self.suspend_subscription_execution();
let mut inner = self.lock_lifecycle()?;
let live = inner.live.as_mut().ok_or(CpaLifecycleError::NotStarted)?;
let exit = self.shutdown_live_process(live)?;
live.lease
.release()
.map_err(|_| CpaLifecycleError::OwnerState)?;
inner.live.take();
self.epochs.advance_runtime();
inner.last_exit = Some(exit);
let generation = inner.restart_count;
drop(inner);
self.emit(DiagnosticEvent::CpaLifecycle(CpaLifecycle {
phase: CpaPhase::Exit,
elapsed_ms: started.elapsed().as_millis() as u64,
exit_code: exit.code,
signal: None,
}));
self.emit_stage(
CpaStageKind::ControlCall,
CpaStageOutcome::Completed,
started.elapsed().as_millis() as u64,
generation,
);
Ok(exit)
}
pub fn discover_registered_sources(
&self,
) -> Result<Vec<CpaRegisteredSourceV1>, CpaLifecycleError> {
let materials = self.discover_materializations(None)?;
materials
.iter()
.map(|material| {
register_cpa_account(&self.catalog, material)
.map_err(|_| CpaLifecycleError::InvalidMaterialization)
})
.collect()
}
pub fn last_exit(&self) -> Option<CpaExit> {
self.observation.lock().last_exit
}
fn invalidate_live_accounts(
&self,
live: &mut LiveRuntime,
kind: Option<crate::CpaAccountKind>,
) -> bool {
self.commit_lifecycle(|| {
let changed = subscriptions::deactivate_accounts(&mut live.accounts, kind);
if changed {
self.epochs.advance_target();
}
live.last_health = None;
changed
})
.unwrap_or(false)
}
fn invalidate_runtime_accounts(
&self,
inner: &mut RuntimeInner,
kind: Option<crate::CpaAccountKind>,
) {
let Some(live) = inner.live.as_mut() else {
return;
};
if self.invalidate_live_accounts(live, kind)
&& let Ok(layout) = self.prepare_layout()
{
let _ = save_account_state(&layout.accounts_path, &live.accounts);
}
}
fn prepare_layout(&self) -> Result<InstanceLayout, CpaLifecycleError> {
let root = ensure_private_dir(&self.spec.state_root)?;
let instance_dir = ensure_private_dir(&root.join(&self.spec.instance_id))?;
let work_dir = ensure_private_dir(&instance_dir.join("work"))?;
let auth_dir = ensure_private_dir(&self.spec.auth_dir)?;
Ok(InstanceLayout {
config_path: instance_dir.join("config.yaml"),
capability_path: instance_dir.join("capability.private"),
accounts_path: instance_dir.join("accounts.private.json"),
lock_dir: instance_dir.join("owner.lock"),
work_dir,
auth_dir,
})
}
fn start_fresh(
&self,
inner: &mut RuntimeInner,
lease: OwnerLease,
record: OwnerRecord,
artifact: VerifiedCpaBinary,
layout: &InstanceLayout,
expected: Option<&BorrowedSubscriptionEvidence>,
) -> Result<LiveRuntime, CpaLifecycleError> {
let generation = inner.restart_count;
let attempt = (|| {
let auth_lease = ManagedAuthLease::acquire_subscription(
&layout.auth_dir,
self.spec.borrowed_codex_auth.as_ref(),
self.spec.borrowed_claude_auth.as_ref(),
self.spec.managed_oauth,
expected,
)?;
let secrets = InstanceSecrets::generate()?;
secrets.write(&layout.capability_path)?;
let address = reserve_loopback()?;
let spawn_started = std::time::Instant::now();
self.emit_stage(
CpaStageKind::ProcessSpawn,
CpaStageOutcome::Entered,
0,
generation,
);
let process = match self.spawn_process(&artifact, layout, address, &secrets) {
Ok(process) => {
self.emit_stage(
CpaStageKind::ProcessSpawn,
CpaStageOutcome::Completed,
spawn_started.elapsed().as_millis() as u64,
generation,
);
process
}
Err(error) => {
self.emit_stage(
CpaStageKind::ProcessSpawn,
CpaStageOutcome::Failed {
code: spawn_failure_code(&error),
},
spawn_started.elapsed().as_millis() as u64,
generation,
);
return Err(error);
}
};
Ok((secrets, address, process, auth_lease))
})();
let mut live = match attempt {
Ok((secrets, address, process, auth_lease)) => LiveRuntime {
lease,
auth_lease: Some(auth_lease),
owner_record: record,
process: Some(process),
artifact,
address,
secrets,
accounts: Vec::new(),
accounts_validated_for_process: false,
last_health: None,
published_auth_generation: None,
adopted: false,
cleanup_only: true,
},
Err(error) => {
let _ = lease.abandon();
return Err(error);
}
};
// Once spawn succeeds, every fallible step retains both the process and
// credential lease. A failed stop must remain retryable by this owner.
let ready = (|| {
live.owner_record = live
.owner_record
.clone()
.running(
live.process.as_ref().expect("spawned process").pid(),
live.address,
)
.map_err(|_| CpaLifecycleError::OwnerState)?;
live.lease
.write(&live.owner_record)
.map_err(|_| CpaLifecycleError::OwnerState)?;
self.wait_ready(
live.process.as_mut().expect("spawned process").as_mut(),
&live.artifact,
live.address,
&live.secrets,
generation,
)?;
validate_runtime_files(layout)?;
live.accounts = load_account_state(&layout.accounts_path)?;
live.last_health = Some(Instant::now());
Ok(())
})();
if let Err(error) = ready {
self.cleanup_failed_start(inner, live);
return Err(error);
}
Ok(live)
}
fn cleanup_failed_start(&self, inner: &mut RuntimeInner, mut live: LiveRuntime) {
live.cleanup_only = true;
crate::request_context::cleanup(self.spec.shutdown_timeout, || {
// Cancellation may have prevented the first owner-record write.
// Record the already-created child for recovery; this grants no work.
let _ = live.lease.write(&live.owner_record);
if self.shutdown_live_process(&mut live).is_ok() {
let _ = live.lease.release();
} else {
live.last_health = None;
inner.live = Some(live);
}
});
}
#[allow(clippy::too_many_arguments)]
fn adopt_orphan(
&self,
inner: &mut RuntimeInner,
lease: OwnerLease,
stale_record: OwnerRecord,
transferred: OwnerRecord,
artifact: VerifiedCpaBinary,
layout: &InstanceLayout,
expected: Option<&BorrowedSubscriptionEvidence>,
) -> Result<LiveRuntime, CpaLifecycleError> {
let generation = inner.restart_count;
let ready_started = std::time::Instant::now();
self.emit_stage(
CpaStageKind::ReadyWait,
CpaStageOutcome::Entered,
0,
generation,
);
let attempt = (|| {
if transferred.binary_version != artifact.version().to_string()
|| transferred.binary_sha256_hex != artifact.sha256_hex()
|| !transferred.address.ip().is_loopback()
{
return Err(CpaLifecycleError::UntrustedOrphan);
}
let auth_lease = ManagedAuthLease::acquire_subscription(
&layout.auth_dir,
self.spec.borrowed_codex_auth.as_ref(),
self.spec.borrowed_claude_auth.as_ref(),
self.spec.managed_oauth,
expected,
)?;
let secrets = InstanceSecrets::read(&layout.capability_path)?;
validate_runtime_files(layout)?;
self.control
.probe_ready(
transferred.address,
&secrets,
&transferred.binary_version,
self.spec.control_timeout,
)
.map_err(|error| self.map_control_error(error))?;
let mut process = crate::process::attach_recovered_process(
self.backend.as_ref(),
transferred.cpa_pid,
)?;
self.control
.probe_ready(
transferred.address,
&secrets,
&transferred.binary_version,
self.spec.control_timeout,
)
.map_err(|error| self.map_control_error(error))?;
if process.pid() != transferred.cpa_pid || process.try_exit()?.is_some() {
return Err(CpaLifecycleError::UntrustedOrphan);
}
let accounts = load_account_state(&layout.accounts_path)?;
Ok((transferred, secrets, process, accounts, auth_lease))
})();
match attempt {
Ok((mut transferred, secrets, mut process, accounts, auth_lease)) => {
self.emit_stage(
CpaStageKind::ReadyWait,
CpaStageOutcome::Completed,
ready_started.elapsed().as_millis() as u64,
generation,
);
let digest = self.proxy_environment.digest();
if transferred.proxy_environment_sha256.as_deref() != Some(digest.as_str()) {
// The management capability has authenticated this exact orphan twice.
// Never serve through an old/unknown proxy policy after owner recovery.
if let Err(error) =
crate::request_context::cleanup(self.spec.shutdown_timeout, || {
process.shutdown(self.spec.shutdown_timeout)
})
{
let _ = lease.restore_stale(&stale_record, self.spec.shutdown_timeout);
return Err(error.into());
}
drop(auth_lease);
transferred.proxy_environment_sha256 = Some(digest);
return self.start_fresh(inner, lease, transferred, artifact, layout, expected);
}
Ok(LiveRuntime {
lease,
auth_lease: Some(auth_lease),
owner_record: transferred.clone(),
process: Some(process),
artifact,
address: transferred.address,
secrets,
accounts,
accounts_validated_for_process: false,
last_health: Some(Instant::now()),
published_auth_generation: None,
adopted: true,
cleanup_only: true,
})
}
Err(error) => {
self.emit_stage(
CpaStageKind::ReadyWait,
CpaStageOutcome::Failed {
code: ready_failure_code(&error),
},
ready_started.elapsed().as_millis() as u64,
generation,
);
let _ = lease.restore_stale(&stale_record, self.spec.shutdown_timeout);
Err(error)
}
}
}
fn spawn_process(
&self,
artifact: &VerifiedCpaBinary,
layout: &InstanceLayout,
address: SocketAddr,
secrets: &InstanceSecrets,
) -> Result<Box<dyn CpaProcessHandle>, CpaLifecycleError> {
crate::request_context::check()?;
let config = render_managed_config(address.port(), &layout.auth_dir, secrets)?;
private_atomic_write(&layout.config_path, &config)?;
crate::request_context::check()?;
self.backend
.spawn(&CpaLaunch {
binary: artifact.clone(),
config_path: layout.config_path.clone(),
work_dir: layout.work_dir.clone(),
management_password: secrets.management.clone(),
proxy_environment: self.proxy_environment.clone(),
})
.map_err(CpaLifecycleError::from)
}
fn wait_ready(
&self,
process: &mut dyn CpaProcessHandle,
artifact: &VerifiedCpaBinary,
address: SocketAddr,
secrets: &InstanceSecrets,
generation: u64,
) -> Result<(), CpaLifecycleError> {
self.emit_stage(
CpaStageKind::ReadyWait,
CpaStageOutcome::Entered,
0,
generation,
);
let ready_started = Instant::now();
let deadline = ready_started + self.spec.startup_timeout;
let outcome = {
let _scope = crate::CpaRequestContext::new(deadline).enter();
(|| loop {
crate::request_context::check()?;
match process.try_exit() {
Ok(Some(exit)) => break Err(CpaLifecycleError::ExitedDuringStartup(exit)),
Ok(None) => {}
Err(error) => break Err(error.into()),
}
let ready = self.control.probe_ready(
address,
secrets,
&artifact.version().to_string(),
crate::request_context::remaining(self.spec.control_timeout)?,
);
crate::request_context::check()?;
match ready {
Ok(()) => {
break Ok(());
}
Err(AccountDiscoveryError::RunningVersionMismatch)
| Err(AccountDiscoveryError::SecretBearingResponse) => {
break Err(CpaLifecycleError::UnsafeControlResponse);
}
Err(_) if Instant::now() < deadline => {
crate::request_context::sleep(Duration::from_millis(20))?
}
Err(_) => break Err(CpaLifecycleError::StartupTimeout),
}
})()
};
// The startup budget is a business timeout. Only the caller's enclosing
// lifetime can classify it as cancellation; both paths emit a terminal.
let outcome = match outcome {
Err(CpaLifecycleError::OperationCancelled)
if Instant::now() >= deadline && crate::request_context::check().is_ok() =>
{
Err(CpaLifecycleError::StartupTimeout)
}
outcome => outcome,
};
let elapsed_ms = ready_started.elapsed().as_millis() as u64;
match &outcome {
Ok(()) => self.emit_stage(
CpaStageKind::ReadyWait,
CpaStageOutcome::Completed,
elapsed_ms,
generation,
),
Err(error) => self.emit_stage(
CpaStageKind::ReadyWait,
CpaStageOutcome::Failed {
code: ready_failure_code(error),
},
elapsed_ms,
generation,
),
}
outcome
}
fn health_locked(&self, inner: &mut RuntimeInner) -> Result<CpaHealth, CpaLifecycleError> {
let Some(live) = inner.live.as_mut() else {
return Ok(CpaHealth::Stopped {
last_exit: inner.last_exit,
});
};
let Some(process) = live.process.as_mut() else {
return Ok(CpaHealth::Crashed {
exit: inner.last_exit.unwrap_or(CpaExit { code: None }),
restart_count: inner.restart_count,
});
};
if live.cleanup_only {
return Ok(CpaHealth::Unhealthy {
pid: process.pid(),
restart_count: inner.restart_count,
});
}
validate_runtime_files(&self.prepare_layout()?)?;
if let Some(exit) = process.try_exit()? {
live.process = None;
live.accounts_validated_for_process = false;
inner.last_exit = Some(exit);
return Ok(CpaHealth::Crashed {
exit,
restart_count: inner.restart_count,
});
}
if self
.control
.probe_ready(
live.address,
&live.secrets,
&live.artifact.version().to_string(),
self.spec.control_timeout,
)
.is_err()
{
live.last_health = None;
return Ok(CpaHealth::Unhealthy {
pid: process.pid(),
restart_count: inner.restart_count,
});
}
live.last_health = Some(Instant::now());
self.health_invalidated.store(false, Ordering::Release);
Ok(ready_health(live, inner.restart_count))
}
pub(crate) fn ensure_ready_locked(
&self,
inner: &mut RuntimeInner,
expected: Option<&BorrowedSubscriptionEvidence>,
) -> Result<(), CpaLifecycleError> {
crate::request_context::check()?;
if inner.live.as_ref().is_some_and(|live| live.cleanup_only) {
if expected.is_none()
&& !self
.admission
.snapshot()
.passive_source_is_admitted(self.is_managed_oauth())
{
return Err(CpaLifecycleError::StaleSourceManagement);
}
self.start_expected_locked(inner, expected)?;
return Ok(());
}
let live = inner.live.as_mut().ok_or(CpaLifecycleError::NotStarted)?;
validate_runtime_files(&self.prepare_layout()?)?;
if let Some(process) = live.process.as_mut() {
if let Some(exit) = process.try_exit()? {
live.process = None;
live.accounts_validated_for_process = false;
live.last_health = None;
self.health_invalidated.store(true, Ordering::Release);
inner.last_exit = Some(exit);
} else {
if !self.health_invalidated.load(Ordering::Acquire)
&& live
.last_health
.is_some_and(|sample| sample.elapsed() < Duration::from_secs(5))
{
return Ok(());
}
live.last_health = None;
self.control
.probe_ready(
live.address,
&live.secrets,
&live.artifact.version().to_string(),
self.spec.control_timeout,
)
.map_err(|error| self.map_control_error(error))?;
crate::request_context::check()?;
live.last_health = Some(Instant::now());
self.health_invalidated.store(false, Ordering::Release);
return Ok(());
}
}
// Native initial discovery may recover without a saved projection. A
// managed OAuth writer needs either saved Enabled authority or explicit
// Check evidence; pending/passive reads never start its refresh worker.
if expected.is_none()
&& !self
.admission
.snapshot()
.passive_source_is_admitted(self.is_managed_oauth())
{
return Err(CpaLifecycleError::StaleSourceManagement);
}
let exit = inner.last_exit.unwrap_or(CpaExit { code: None });
let now = Instant::now();
while inner
.crashes
.front()
.is_some_and(|instant| now.duration_since(*instant) > self.spec.restart_policy.window)
{
inner.crashes.pop_front();
}
inner.crashes.push_back(now);
if inner.crashes.len() > usize::from(self.spec.restart_policy.max_restarts) {
return Err(CpaLifecycleError::CrashLoop(exit));
}
let exponent = u32::try_from(inner.crashes.len().saturating_sub(1)).unwrap_or(u32::MAX);
let multiplier = 2_u32.checked_pow(exponent).unwrap_or(u32::MAX);
let backoff = self
.spec
.restart_policy
.base_backoff
.saturating_mul(multiplier)
.min(self.spec.restart_policy.max_backoff);
if !backoff.is_zero() {
crate::request_context::sleep(backoff)?;
}
let verified = self.locator.locate()?;
ensure_supported_artifact(&verified)?;
let generation = inner.restart_count;
let live = inner.live.as_mut().ok_or(CpaLifecycleError::NotStarted)?;
live.accounts_validated_for_process = false;
if verified.version() != live.artifact.version()
|| verified.sha256_hex() != live.artifact.sha256_hex()
{
return Err(CpaLifecycleError::ArtifactChanged);
}
let layout = self.prepare_layout()?;
live.auth_lease
.as_mut()
.ok_or(CpaLifecycleError::OwnerState)?
.refresh_subscription(expected, None)?;
let spawn_started = Instant::now();
self.emit_stage(
CpaStageKind::ProcessSpawn,
CpaStageOutcome::Entered,
0,
generation,
);
let process = match self.spawn_process(&verified, &layout, live.address, &live.secrets) {
Ok(process) => {
self.emit_stage(
CpaStageKind::ProcessSpawn,
CpaStageOutcome::Completed,
spawn_started.elapsed().as_millis() as u64,
generation,
);
process
}
Err(error) => {
self.emit_stage(
CpaStageKind::ProcessSpawn,
CpaStageOutcome::Failed {
code: spawn_failure_code(&error),
},
spawn_started.elapsed().as_millis() as u64,
generation,
);
return Err(error);
}
};
let running_record = live
.owner_record
.clone()
.running(process.pid(), live.address)
.map_err(|_| CpaLifecycleError::OwnerState)?;
// Track the child before any further fallible I/O. Cancellation may
// revoke publication, but it must never lose the sole cleanup owner.
live.process = Some(process);
live.owner_record = running_record.clone();
let ready = (|| {
live.lease
.write(&running_record)
.map_err(|_| CpaLifecycleError::OwnerState)?;
self.wait_ready(
live.process
.as_mut()
.ok_or(CpaLifecycleError::OwnerState)?
.as_mut(),
&verified,
live.address,
&live.secrets,
generation,
)?;
validate_runtime_files(&layout)?;
self.commit_lifecycle(|| {
live.artifact = verified;
live.adopted = false;
live.last_health = Some(Instant::now());
inner.restart_count = inner.restart_count.saturating_add(1);
self.health_invalidated.store(false, Ordering::Release);
})
})();
if let Err(error) = ready {
live.last_health = None;
crate::request_context::cleanup(self.spec.shutdown_timeout, || {
if let Some(process) = live.process.as_mut()
&& process.shutdown(self.spec.shutdown_timeout).is_ok()
{
live.process = None;
}
});
return Err(error);
}
Ok(())
}
fn shutdown_live_process(&self, live: &mut LiveRuntime) -> Result<CpaExit, CpaLifecycleError> {
let Some(process) = live.process.as_mut() else {
return Ok(CpaExit { code: None });
};
if let Some(exit) = process.try_exit()? {
return Ok(exit);
}
if live.adopted {
self.control
.probe_ready(
live.address,
&live.secrets,
&live.artifact.version().to_string(),
self.spec.control_timeout,
)
.map_err(|error| self.map_control_error(error))?;
}
process
.shutdown(crate::request_context::remaining(
self.spec.shutdown_timeout,
)?)
.map_err(Into::into)
}
}
impl CpaRegisteredSourcePort for ManagedCpaRuntime {
fn begin_routing_batch(&self) -> Result<CpaRoutingBatch<'_>, CpaLifecycleError> {
CpaRoutingBatch::begin(self)
}
fn discover_registered_sources(&self) -> Result<Vec<CpaRegisteredSourceV1>, CpaLifecycleError> {
ManagedCpaRuntime::discover_registered_sources(self)
}
}
impl Drop for ManagedCpaRuntime {
fn drop(&mut self) {
self.epochs.advance_runtime();
if let Some(mut live) = self.inner.get_mut().live.take()
&& self.shutdown_live_process(&mut live).is_ok()
{
let _ = live.lease.release();
}
}
}
fn account_epoch_facts(
accounts: &[AccountSnapshotRecord],
) -> Vec<(crate::CpaAccountKind, String, u64, bool, Vec<String>)> {
accounts
.iter()
.map(|account| {
(
account.account_kind,
account.account_digest.clone(),
account.generation,
account.active,
account.observed_model_ids.iter().cloned().collect(),
)
})
.collect()
}
impl CpaSupervisorPort for ManagedCpaRuntime {
fn materialize_account(
&self,
connector_id: &str,
endpoint_profile_id: &str,
) -> Result<CpaAccountMaterializationV1, CpaSupervisorError> {
let mut matches = self
.discover_materializations(None)
.map_err(|_| CpaSupervisorError::Unavailable)?
.into_iter()
.filter(|value| {
value.connector_id == connector_id
&& value.endpoint_profile_id == endpoint_profile_id
});
let first = matches.next().ok_or(CpaSupervisorError::ActionRequired)?;
if matches.next().is_some() {
return Err(CpaSupervisorError::ActionRequired);
}
Ok(first)
}
}
fn reserve_loopback() -> Result<SocketAddr, CpaLifecycleError> {
let listener =
TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).map_err(CpaLifecycleError::LoopbackBind)?;
let address = listener
.local_addr()
.map_err(CpaLifecycleError::LoopbackBind)?;
if address.ip() != IpAddr::V4(Ipv4Addr::LOCALHOST) || address.port() == 0 {
return Err(CpaLifecycleError::LoopbackBind(std::io::Error::other(
"non-loopback reservation",
)));
}
drop(listener);
Ok(address)
}
fn ensure_supported_artifact(artifact: &VerifiedCpaBinary) -> Result<(), CpaLifecycleError> {
if artifact.version().to_string() != MANAGED_CPA_ARTIFACT_VERSION {
return Err(CpaLifecycleError::UnsupportedArtifactVersion);
}
Ok(())
}
/// Why locating or validating the trusted artifact failed. A refusal by the bridge contract
/// is a rejection; anything else is reported as the artifact being unavailable.
fn artifact_failure_code(error: &CpaLifecycleError) -> CpaFailureCode {
match error {
CpaLifecycleError::UnsupportedArtifactVersion
| CpaLifecycleError::InvalidBinding
| CpaLifecycleError::ArtifactChanged
| CpaLifecycleError::Artifact(_) => CpaFailureCode::ArtifactRejected,
_ => CpaFailureCode::ArtifactUnavailable,
}
}
/// Why the trusted artifact could not be located: the manifest refusing a version or digest
/// is a rejection, everything else means the verified artifact was not available.
fn locate_failure_code(error: &CpaArtifactError) -> CpaFailureCode {
match error {
CpaArtifactError::VersionMismatch { .. }
| CpaArtifactError::DigestMismatch
| CpaArtifactError::InvalidDigest => CpaFailureCode::ArtifactRejected,
_ => CpaFailureCode::ArtifactUnavailable,
}
}
/// Why a spawn, adopt or ready-wait step failed. Causes that are not part of this vocabulary
/// stay `Unknown` instead of being attributed to a step that did not fail.
fn spawn_failure_code(error: &CpaLifecycleError) -> CpaFailureCode {
match error {
CpaLifecycleError::AlreadyOwned => CpaFailureCode::AlreadyOwned,
CpaLifecycleError::UntrustedOrphan | CpaLifecycleError::ArtifactChanged => {
CpaFailureCode::ArtifactRejected
}
CpaLifecycleError::StartupTimeout => CpaFailureCode::ReadyTimeout,
CpaLifecycleError::ExitedDuringStartup(_)
| CpaLifecycleError::CrashLoop(_)
| CpaLifecycleError::UnsafeControlResponse
| CpaLifecycleError::ControlUnavailable => CpaFailureCode::ReadyRejected,
CpaLifecycleError::Process(_)
| CpaLifecycleError::Config(_)
| CpaLifecycleError::LoopbackBind(_) => CpaFailureCode::SpawnFailed,
_ => CpaFailureCode::Unknown,
}
}
/// Why the bounded ready wait failed.
fn ready_failure_code(error: &CpaLifecycleError) -> CpaFailureCode {
match error {
CpaLifecycleError::StartupTimeout => CpaFailureCode::ReadyTimeout,
CpaLifecycleError::ExitedDuringStartup(_)
| CpaLifecycleError::CrashLoop(_)
| CpaLifecycleError::UnsafeControlResponse
| CpaLifecycleError::ControlUnavailable => CpaFailureCode::ReadyRejected,
_ => CpaFailureCode::Unknown,
}
}
fn validate_runtime_files(layout: &InstanceLayout) -> Result<(), CpaLifecycleError> {
validate_private_file(&layout.config_path)?;
validate_private_file(&layout.capability_path)?;
if layout.accounts_path.exists() {
validate_private_file(&layout.accounts_path)?;
}
Ok(())
}
fn ready_health(live: &LiveRuntime, restart_count: u64) -> CpaHealth {
CpaHealth::Ready {
pid: live.process.as_ref().map_or(0, |process| process.pid()),
address: live.address,
restart_count,
adopted: live.adopted,
}
}
impl ManagedCpaRuntime {
fn map_control_error(&self, error: AccountDiscoveryError) -> CpaLifecycleError {
match error {
AccountDiscoveryError::AuthenticationRequired
| AccountDiscoveryError::AccountDisappeared => {
if self.is_managed_oauth() {
CpaLifecycleError::ManagedOAuthAuthenticationRequired
} else if self.spec.borrowed_claude_auth.is_some() {
CpaLifecycleError::InvalidBorrowedClaudeAuth
} else {
CpaLifecycleError::BorrowedCodexAuthUnavailable
}
}
AccountDiscoveryError::SecretBearingResponse
| AccountDiscoveryError::NonSubscriptionAccount
| AccountDiscoveryError::RunningVersionMismatch => {
CpaLifecycleError::UnsafeControlResponse
}
_ => CpaLifecycleError::ControlUnavailable,
}
}
}
#[cfg(test)]
#[path = "runtime/tests.rs"]
mod tests;