Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.
Rust ai-agentsai-gatewayclaude-codecodexcoding-agentcost-optimizationdeepseekllm-gatewayllm-routerlocal-firstmodel-routingrusttask-routingtauri
use std::sync::Arc;
use std::time::Instant;
use async_trait::async_trait;
use hiroute_gateway_core::core::execution_plan::{
AcceptedResponseExecutionBinding, AttemptExecutionBinding, ConfigEventSnapshot, TransportTarget,
};
use hiroute_gateway_core::core::filter::LocalReply;
use hiroute_gateway_core::runtime::attempt::{
PrecommitEvent, PreparedAttemptHttpRequest, PublishedDisposition,
};
use hiroute_gateway_core::runtime::driver::{
AcceptedBodyFrame, AttemptFailureFacts, AttemptMaterializationContext,
AttemptMaterializationFailure, ClassifiedAttemptResult, DecodedSseToken,
LogicalRequestBodyFrame, LogicalRequestContext, NormalizedAttemptLocalReply,
PinnedConfigContext, PrecommitClassification, ProviderAcceptedEvent, ProviderAttemptCompletion,
ProviderClassificationFacts, ProviderRuntimePort, UpstreamSideEffectSnapshot,
};
use hiroute_gateway_core::transport::{GatewayRequestHead, GatewayResponseHead};
use tokio_util::sync::CancellationToken;
use crate::runtime::{
ProductionAttemptState, ProductionDecodedSse, ProductionLogicalRequest, ProductionProvider,
ProductionReadiness, ProductionRouteContext,
};
use super::{RequestObservation, active_request};
#[path = "provider/capture.rs"]
mod capture;
#[path = "provider/cpa_execution.rs"]
pub(super) mod cpa_execution;
#[path = "provider/wire_diagnostic.rs"]
pub(in crate::server::core_runtime) mod wire_diagnostic;
pub(super) use capture::{CanonicalCaptureHandle, CanonicalCaptureProducer};
/// Observation-only decorator around the production provider. The inner
/// provider remains the sole execution owner. Only after authoritative
/// classification succeeds does this decorator enqueue a byte-budgeted copy
/// for off-path canonical capture; every observation error degrades capture
/// and is deliberately excluded from the provider result.
pub struct ObservedProductionProvider {
inner: ProductionProvider,
}
pub struct ObservedAttemptState {
inner: ProductionAttemptState,
tracker: Option<CanonicalCaptureHandle>,
observation: Option<RequestObservation>,
}
pub struct ObservedReadiness {
inner: ProductionReadiness,
tracker: Option<CanonicalCaptureHandle>,
observation: Option<RequestObservation>,
}
pub struct ObservedDecodedSse {
inner: ProductionDecodedSse,
}
impl ObservedProductionProvider {
pub fn new(inner: ProductionProvider) -> Self {
Self { inner }
}
}
#[async_trait]
impl ProviderRuntimePort for ObservedProductionProvider {
type LogicalRequest = ProductionLogicalRequest;
type RouteRequestContext = ProductionRouteContext;
type AttemptState = ObservedAttemptState;
type Readiness = ObservedReadiness;
type DecodedSseEvent = ObservedDecodedSse;
async fn begin_request(
&self,
head: GatewayRequestHead,
context: LogicalRequestContext<'_>,
) -> Result<Self::LogicalRequest, Arc<str>> {
self.inner.begin_request(head, context).await
}
async fn consume_request_body(
&self,
logical: &mut Self::LogicalRequest,
frame: LogicalRequestBodyFrame,
) -> Result<(), Arc<str>> {
self.inner.consume_request_body(logical, frame).await
}
fn finalize_route_request_context(
&self,
logical: &mut Self::LogicalRequest,
) -> Result<Self::RouteRequestContext, Arc<str>> {
self.inner.finalize_route_request_context(logical)
}
async fn materialize_attempt(
&self,
logical: &mut Self::LogicalRequest,
context: AttemptMaterializationContext<'_>,
) -> Result<(PreparedAttemptHttpRequest, Self::AttemptState), Arc<str>> {
let stable_binding_id = context.plan().stable_target_key.as_str().to_owned();
let no_credential_ref = context
.credential_ref()
.as_str()
.starts_with("credential/none/")
.then(|| context.credential_ref().as_str().to_owned());
let (request, inner) = self.inner.materialize_attempt(logical, context).await?;
let observation = active_request().filter(RequestObservation::tracks_attempts);
if let Some(observation) = &observation {
observation.lock_state().managed_cpa = inner.trusts_cpa_execution_metadata();
}
if let (Some(observation), Some(credential_ref)) = (&observation, no_credential_ref) {
observation.no_credential_materialized(&stable_binding_id, &credential_ref);
}
// The unauthenticated profile has no credential-read staging hook.
// Bind only after authoritative materialization and its None staging.
#[cfg(all(unix, debug_assertions))]
if let Some(observation) = &observation {
inner.register_private_capture(observation);
}
let tracker = observation.as_ref().and_then(|observation| {
observation.begin_response_capture(
&stable_binding_id,
inner.chat_tool_projection_for_observation(),
)
});
Ok((
request,
ObservedAttemptState {
inner,
tracker,
observation,
},
))
}
fn resolved_transport_target(
&self,
state: &Self::AttemptState,
binding: &AttemptExecutionBinding,
) -> Result<Option<TransportTarget>, Arc<str>> {
self.inner.resolved_transport_target(&state.inner, binding)
}
fn classify_materialization_failure(&self, error: &Arc<str>) -> AttemptMaterializationFailure {
self.inner.classify_materialization_failure(error)
}
fn classify_precommit(
&self,
state: &mut Self::AttemptState,
event: PrecommitEvent,
pinned_configs: &PinnedConfigContext<'_>,
event_configs: &ConfigEventSnapshot,
) -> Result<PrecommitClassification<Self::Readiness, Self::DecodedSseEvent>, Arc<str>> {
if let (Some(observation), PrecommitEvent::ResponseHead(head)) =
(&state.observation, &event)
{
observation.response_head_diagnostic(head.headers(), head.status().as_u16());
}
let prepared = state
.tracker
.as_ref()
.and_then(|tracker| tracker.prepare_native(&event));
let result =
self.inner
.classify_precommit(&mut state.inner, event, pinned_configs, event_configs);
let result = match result {
Ok(result) => {
if let Some(prepared) = prepared {
prepared.commit();
}
result
}
Err(error) => {
if let Some(prepared) = prepared {
prepared.cancel("canonical_capture_authoritative_classification_failed");
} else if let Some(tracker) = &state.tracker {
tracker.cancel_unless_terminal(
"canonical_capture_authoritative_classification_failed",
);
}
return Err(error);
}
};
Ok(PrecommitClassification {
classified: result.classified.map(|classified| ClassifiedAttemptResult {
facts: classified.facts,
readiness: ObservedReadiness {
inner: classified.readiness,
tracker: state.tracker.take(),
observation: state.observation.clone(),
},
}),
decoded_sse: result.decoded_sse.map(|decoded| DecodedSseToken {
sequence: decoded.sequence,
decoded: ObservedDecodedSse {
inner: decoded.decoded,
},
}),
})
}
async fn confirm_precommit(
&self,
state: &mut Self::AttemptState,
facts: &ProviderClassificationFacts,
deadline: Instant,
cancellation: &CancellationToken,
) -> Result<(), Arc<str>> {
self.inner
.confirm_precommit(&mut state.inner, facts, deadline, cancellation)
.await
}
async fn confirm_attempt_failure(
&self,
state: &mut Self::AttemptState,
failure: &AttemptFailureFacts,
deadline: Instant,
cancellation: &CancellationToken,
) -> Result<Option<ProviderClassificationFacts>, Arc<str>> {
self.inner
.confirm_attempt_failure(&mut state.inner, failure, deadline, cancellation)
.await
}
fn normalize_attempt_local_reply(
&self,
state: &mut Self::AttemptState,
reply: LocalReply,
upstream_side_effects: UpstreamSideEffectSnapshot,
) -> Result<NormalizedAttemptLocalReply<Self::Readiness>, Arc<str>> {
let normalized = self.inner.normalize_attempt_local_reply(
&mut state.inner,
reply,
upstream_side_effects,
)?;
Ok(NormalizedAttemptLocalReply {
classified: ClassifiedAttemptResult {
facts: normalized.classified.facts,
readiness: ObservedReadiness {
inner: normalized.classified.readiness,
tracker: state.tracker.take(),
observation: state.observation.clone(),
},
},
reply: normalized.reply,
upstream_side_effects: normalized.upstream_side_effects,
})
}
fn finalize_attempt_facts(
&self,
state: &mut Self::AttemptState,
readiness: Option<&mut Self::Readiness>,
published_facts: Option<&ProviderClassificationFacts>,
completion: &ProviderAttemptCompletion,
) -> Result<ProviderClassificationFacts, Arc<str>> {
self.inner.finalize_attempt_facts(
&mut state.inner,
readiness.map(|readiness| &mut readiness.inner),
published_facts,
completion,
)
}
fn accepted_response_head(
&self,
readiness: &mut Self::Readiness,
published: &PublishedDisposition,
accepted: &AcceptedResponseExecutionBinding,
configs: &PinnedConfigContext<'_>,
) -> Result<GatewayResponseHead, Arc<str>> {
let head = self.inner.accepted_response_head(
&mut readiness.inner,
published,
accepted,
configs,
)?;
if let (Some(observation), Some(tracker)) = (&readiness.observation, &readiness.tracker) {
observation.bind_response_capture(tracker.clone());
}
Ok(head)
}
fn take_accepted_prefix(
&self,
readiness: &mut Self::Readiness,
) -> Result<Option<ProviderAcceptedEvent<Self::DecodedSseEvent>>, Arc<str>> {
self.inner
.take_accepted_prefix(&mut readiness.inner)
.map(|event| event.map(wrap_accepted_event))
}
fn encode_accepted_event(
&self,
readiness: &mut Self::Readiness,
event: ProviderAcceptedEvent<Self::DecodedSseEvent>,
published: &PublishedDisposition,
accepted: &AcceptedResponseExecutionBinding,
pinned_configs: &PinnedConfigContext<'_>,
event_configs: &ConfigEventSnapshot,
) -> Result<Option<AcceptedBodyFrame>, Arc<str>> {
let prepared = match (&readiness.tracker, &event) {
(Some(tracker), ProviderAcceptedEvent::Raw(raw)) => tracker.prepare_native(raw),
_ => None,
};
let frame = self.inner.encode_accepted_event(
&mut readiness.inner,
unwrap_accepted_event(event),
published,
accepted,
pinned_configs,
event_configs,
);
match frame {
Ok(frame) => {
if let Some(prepared) = prepared {
prepared.commit();
}
Ok(frame)
}
Err(error) => {
if let Some(prepared) = prepared {
prepared.cancel("canonical_capture_authoritative_encoding_failed");
} else if let Some(tracker) = &readiness.tracker {
tracker
.cancel_unless_terminal("canonical_capture_authoritative_encoding_failed");
}
Err(error)
}
}
}
fn release_terminal_request(
&self,
logical: Self::LogicalRequest,
published: &PublishedDisposition,
) -> Result<(), Arc<str>> {
self.inner.release_terminal_request(logical, published)
}
}
impl Drop for ObservedAttemptState {
fn drop(&mut self) {
if let Some(tracker) = &self.tracker {
tracker.cancel_unless_terminal("canonical_capture_attempt_released");
}
}
}
impl Drop for ObservedReadiness {
fn drop(&mut self) {
if let Some(tracker) = &self.tracker {
tracker.cancel_unless_terminal("canonical_capture_readiness_released");
}
}
}
fn wrap_accepted_event(
event: ProviderAcceptedEvent<ProductionDecodedSse>,
) -> ProviderAcceptedEvent<ObservedDecodedSse> {
match event {
ProviderAcceptedEvent::Raw(event) => ProviderAcceptedEvent::Raw(event),
ProviderAcceptedEvent::DecodedSse {
sequence,
decoded,
provenance,
} => ProviderAcceptedEvent::DecodedSse {
sequence,
decoded: ObservedDecodedSse { inner: decoded },
provenance,
},
ProviderAcceptedEvent::Terminal {
body,
provenance,
upstream_side_effects,
} => ProviderAcceptedEvent::Terminal {
body,
provenance,
upstream_side_effects,
},
}
}
fn unwrap_accepted_event(
event: ProviderAcceptedEvent<ObservedDecodedSse>,
) -> ProviderAcceptedEvent<ProductionDecodedSse> {
match event {
ProviderAcceptedEvent::Raw(event) => ProviderAcceptedEvent::Raw(event),
ProviderAcceptedEvent::DecodedSse {
sequence,
decoded,
provenance,
} => ProviderAcceptedEvent::DecodedSse {
sequence,
decoded: decoded.inner,
provenance,
},
ProviderAcceptedEvent::Terminal {
body,
provenance,
upstream_side_effects,
} => ProviderAcceptedEvent::Terminal {
body,
provenance,
upstream_side_effects,
},
}
}