Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.
Rust ai-agentsai-gatewayclaude-codecodexcoding-agentcost-optimizationdeepseekllm-gatewayllm-routerlocal-firstmodel-routingrusttask-routingtauri
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::{Duration, Instant};
use async_trait::async_trait;
use hiroute_gateway_core::core::execution_plan::{
AttemptExecutionBinding, CompiledAttemptPlan, TransportScheme, TransportTarget,
};
use hiroute_gateway_core::runtime::attempt::{
AttemptTimeoutKind, PreparedAttemptBody, PreparedAttemptHttpRequest, PreparedRequestHead,
};
use hiroute_gateway_core::runtime::body::{ChargedBodyQueue, MemoryRole};
use hiroute_gateway_core::runtime::driver::{
AttemptFailureClass as CoreAttemptFailureClass, AttemptFailureFacts,
AttemptMaterializationContext, AttemptMaterializationFailure,
AttemptMaterializationFailureClass, LogicalRequestBodyFrame, LogicalRequestContext,
ProviderClassificationFacts, RetryabilityFact,
};
use hiroute_gateway_core::transport::GatewayRequestHead;
use http::header::{CONTENT_LENGTH, CONTENT_TYPE, HOST};
use http::{HeaderMap, HeaderValue, Method};
use crate::attempt_outcome::{AttemptFailure, RawAttemptFailure, classify_failure};
use crate::ports::{
CredentialLeaseRequest, ExecutionScope, RuntimeStateEntry, RuntimeStateKey, RuntimeStateStore,
};
use crate::runtime::native_endpoint_state_key;
use crate::server::composition::{
PortError, ProductionPorts, RuntimeStateStore as ProductionStore,
};
use crate::server::core_runtime::adapters;
use crate::server::core_runtime::profiles::{
AuthenticationSemantics, CandidateProtocolProfile, ClientProtocolProfile, CostClassV1,
PlannerCandidateFactsV1,
};
use super::response::{connector_error_profile, failure_facts};
use crate::server::publication::lease_target;
use super::{
MATERIALIZATION_AUTHORITY_FAILED, MATERIALIZATION_BINDING_COOLING_PREFIX,
MATERIALIZATION_BINDING_DISABLED, MATERIALIZATION_BINDING_PROBE_BUSY,
MATERIALIZATION_BINDING_UNAVAILABLE, MATERIALIZATION_CREDENTIAL_COOLING_PREFIX,
MATERIALIZATION_CREDENTIAL_DISABLED, MATERIALIZATION_CREDENTIAL_EXHAUSTED,
MATERIALIZATION_CREDENTIAL_PROBE_BUSY, MATERIALIZATION_DNS_FAILED,
MATERIALIZATION_KEY_DISABLED, MATERIALIZATION_PROTOCOL_FAILED, MATERIALIZATION_SOURCE_DISABLED,
MechanicalFailureSignal, PermitAvailability, ProductionAttemptState, ProductionLogicalRequest,
ProductionProvider, ProductionReplaySeedEnvelope, RUNTIME_STATE_AUTHORITY_FAILED,
RuntimeCooldownPolicy, StatePermit, acquire_target_permit_status, confirm_success, label,
record_failure, safe_error, validate_attempt_permits,
};
pub(super) struct ProductionStateAdapter {
pub(super) inner: Arc<dyn ProductionStore>,
}
#[async_trait]
impl RuntimeStateStore for ProductionStateAdapter {
async fn read(
&self,
key: &RuntimeStateKey,
scope: &ExecutionScope,
) -> Result<RuntimeStateEntry, crate::ports::RuntimeStateError> {
self.inner
.read_exact(key, scope)
.await
.map_err(map_state_port_error)
}
async fn compare_and_swap(
&self,
key: &RuntimeStateKey,
expected_generation: u64,
next: RuntimeStateEntry,
scope: &ExecutionScope,
) -> Result<crate::ports::CasOutcome, crate::ports::RuntimeStateError> {
self.inner
.compare_and_swap_exact(key, expected_generation, next, scope)
.await
.map_err(map_state_port_error)
}
async fn acquire_probe_lease(
&self,
key: &RuntimeStateKey,
expected_generation: u64,
now: Instant,
lease_duration: std::time::Duration,
scope: &ExecutionScope,
) -> Result<crate::ports::ProbeLeaseOutcome, crate::ports::RuntimeStateError> {
self.inner
.acquire_probe_lease_exact(key, expected_generation, now, lease_duration, scope)
.await
.map_err(map_state_port_error)
}
}
fn map_state_port_error(_error: PortError) -> crate::ports::RuntimeStateError {
crate::ports::RuntimeStateError::Unavailable
}
impl ProductionProvider {
pub fn new(ports: &ProductionPorts) -> Self {
Self {
credentials: Arc::clone(&ports.credentials),
state: Arc::new(ProductionStateAdapter {
inner: Arc::clone(&ports.runtime_state),
}),
cooldowns: RuntimeCooldownPolicy::default(),
target_resolver: TargetResolver::new(),
}
}
}
pub(super) fn begin_request(
head: GatewayRequestHead,
context: LogicalRequestContext<'_>,
) -> Result<ProductionLogicalRequest, Arc<str>> {
let ingress = crate::server::request_plan::IngressProtocol::from_path(
head.path_and_query
.split_once('?')
.map_or(head.path_and_query.as_ref(), |(path, _)| path),
)
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
let seed = context
.provider_context
.as_ref()
.and_then(|context| context.downcast_ref::<ProductionReplaySeedEnvelope>())
.and_then(ProductionReplaySeedEnvelope::take)
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
let frozen_candidates = context
.frozen_candidates
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
if frozen_candidates.is_empty()
|| frozen_candidates.iter().any(|candidate| {
candidate.candidate_id.is_none()
|| candidate.profile_digest.is_none()
|| candidate.reason_ledger_identity.is_none()
|| candidate.provider_profile.is_none()
})
{
return Err(Arc::from(MATERIALIZATION_PROTOCOL_FAILED));
}
if seed.ingress != ingress || seed.ir.ingress_protocol != ingress {
return Err(Arc::from(MATERIALIZATION_PROTOCOL_FAILED));
}
Ok(ProductionLogicalRequest {
reasoning_cleanup: Arc::clone(&seed.route_context.reasoning_cleanup),
ingress,
replay: seed.replay,
raw_body: seed.raw_body,
observed_body_bytes: 0,
body_eos: false,
ir: Some(seed.ir),
frozen_candidates,
runtime_state_authority: seed.runtime_state_authority,
route_context: Some(seed.route_context),
})
}
pub(super) fn consume_request_body(
logical: &mut ProductionLogicalRequest,
frame: LogicalRequestBodyFrame,
) -> Result<(), Arc<str>> {
if logical.body_eos {
return Err(Arc::from(MATERIALIZATION_PROTOCOL_FAILED));
}
if let Some(bytes) = frame.bytes {
logical.observed_body_bytes = logical
.observed_body_bytes
.checked_add(bytes.bytes().len() as u64)
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
}
if frame.end_stream {
if logical.observed_body_bytes != logical.raw_body.byte_len() {
return Err(Arc::from(MATERIALIZATION_PROTOCOL_FAILED));
}
logical
.replay
.release_stream(&logical.raw_body)
.map_err(safe_error)?;
logical.body_eos = true;
}
Ok(())
}
pub(super) fn finalize_route_request_context(
logical: &mut ProductionLogicalRequest,
) -> Result<super::ProductionRouteContext, Arc<str>> {
if !logical.body_eos || logical.ir.is_none() {
return Err(Arc::from(MATERIALIZATION_PROTOCOL_FAILED));
}
logical
.route_context
.take()
.filter(|context| context.ingress == logical.ingress)
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))
}
pub(super) async fn materialize_attempt(
provider: &ProductionProvider,
logical: &mut ProductionLogicalRequest,
context: AttemptMaterializationContext<'_>,
) -> Result<(PreparedAttemptHttpRequest, ProductionAttemptState), Arc<str>> {
let runtime_state_authority = logical.runtime_state_authority.clone();
let scope = ExecutionScope::new(
context.attempt_deadline(),
context.cancellation_token().clone(),
);
scope
.ensure_active()
.map_err(|_| Arc::from(MATERIALIZATION_AUTHORITY_FAILED))?;
let plan = context.plan();
let stable_target = plan.stable_target_key.as_str();
let candidate = logical
.frozen_candidates
.iter()
.find(|candidate| candidate.binding == context.binding_id())
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
if candidate.stable_target.as_str() != stable_target {
return Err(Arc::from(MATERIALIZATION_PROTOCOL_FAILED));
}
let execution: crate::server::publication::SealedCandidateExecutionV1 = serde_json::from_slice(
candidate
.provider_profile
.as_deref()
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?,
)
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
let profile = &execution.protocol_profile;
let expected_profile_digest = candidate
.profile_digest
.as_ref()
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?
.as_str();
if execution.schema_version != crate::server::publication::SEALED_CANDIDATE_EXECUTION_SCHEMA_V1
|| execution.stable_target_key != stable_target
|| execution.credential_destination_ref.trim().is_empty()
|| execution.logical_endpoint.trim().is_empty()
|| execution.upstream_model_id.trim().is_empty()
|| execution.native_transport_model.trim().is_empty()
|| profile.capability.native_model != execution.native_transport_model
|| (execution.connector_runtime == hiroute_domain::ConnectorRuntimeKind::BuiltinNative
&& execution.upstream_model_id != execution.native_transport_model)
|| profile_digest(profile)? != expected_profile_digest
|| execution.profile_digest != expected_profile_digest
|| profile.ingress_protocol != logical.ingress
|| (execution.connector_runtime != hiroute_domain::ConnectorRuntimeKind::CpaBridge
&& profile.native_target.is_none()
&& execution
.operational_target
.for_protocol_path(&profile.connector.request_path)
.is_none())
|| !execution
.operational_target
.validate_for(execution.connector_runtime, &execution.logical_endpoint)
|| !matches!(
hiroute_domain::CanonicalDigest::of(&execution.operational_target),
Ok(digest) if digest == execution.operational_target_digest
)
|| hiroute_domain::CanonicalDigest::parse(execution.protocol_set_digest.as_str()).is_err()
|| !operational_target_matches_plan(&execution.operational_target, plan)
|| profile.connector.authentication.exact().is_none()
{
return Err(Arc::from(MATERIALIZATION_PROTOCOL_FAILED));
}
let ir = logical
.ir
.as_ref()
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
let cleanup_prefix = logical.reasoning_cleanup.prefix_for(context.binding_id());
let mut prepared =
adapters::project_candidate_request_template_with_cleanup(ir, profile, cleanup_prefix)
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
let streaming = ir.stream;
let served_model_alias = ir.served_model_id.clone();
let client_profile = client_profile_for_candidate(profile)?;
let native_output = profile.ingress_protocol == profile.capability.upstream_protocol;
let renderer = (streaming && !native_output)
.then(|| {
adapters::IncrementalClientSseRenderer::new(
client_profile.clone(),
served_model_alias.clone(),
)
.map(|renderer| renderer.with_budget(context.budget.clone()))
})
.transpose()
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
let prefix = ChargedBodyQueue::new(
context.budget,
MemoryRole::ResponsePrefix,
&plan.body_plans.attempt_response_precommit,
usize::MAX,
plan.precommit_event_capacity,
)
.map_err(safe_error)?;
let lease_target = lease_target::for_request(&execution, profile)?;
let lease_target_digest = hiroute_domain::CanonicalDigest::of(&lease_target)
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
let endpoint_state_key = if profile.native_target.is_some() {
native_endpoint_state_key(stable_target, expected_profile_digest)
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?
} else {
stable_target.to_owned()
};
let binding_key = RuntimeStateKey::binding(endpoint_state_key.clone());
let binding_permit =
acquire_target_permit_status(provider.state.as_ref(), &binding_key, &scope)
.await
.map_err(|_| {
runtime_state_authority.fail();
Arc::from(RUNTIME_STATE_AUTHORITY_FAILED)
})?;
let binding_permit = permit_or_materialization_error(binding_permit, true)?;
let authentication = profile
.connector
.authentication
.exact()
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
let no_credential = matches!(authentication, AuthenticationSemantics::None);
if no_credential
!= context
.credential_ref()
.as_str()
.starts_with("credential/none/")
{
return Err(Arc::from(MATERIALIZATION_PROTOCOL_FAILED));
}
let credential_destination_ref = profile
.native_target
.as_ref()
.map_or(execution.credential_destination_ref.as_str(), |target| {
target.credential_destination_ref.as_str()
});
let lease_logical_endpoint =
if execution.connector_runtime == hiroute_domain::ConnectorRuntimeKind::BuiltinNative {
lease_target.uri()
} else {
&execution.logical_endpoint
};
let credential = if no_credential {
None
} else {
Some(
scope
.run(provider.credentials.lease_exact(
CredentialLeaseRequest {
execution_binding_id: stable_target,
stable_binding_id: execution.credential_binding_id(),
credential_ref: context.credential_ref().as_str(),
credential_destination_ref,
excluded_key_ids: &[],
connector_runtime: execution.connector_runtime,
connector_id: &profile.connector.connector_id,
upstream_protocol: profile.capability.upstream_protocol,
upstream_model_id: &execution.upstream_model_id,
native_transport_model: &execution.native_transport_model,
logical_endpoint: lease_logical_endpoint,
operational_target: lease_target.uri(),
operational_target_digest: lease_target_digest.as_str(),
runtime_epoch: lease_target.runtime_epoch(),
target_epoch: lease_target.target_epoch(),
protocol_profile_digest: expected_profile_digest,
request_path: &profile.connector.request_path,
authentication,
},
&scope,
))
.await
.map_err(|_| Arc::from(MATERIALIZATION_AUTHORITY_FAILED))?
.map_err(|error| Arc::from(match error {
crate::server::composition::PortError::CredentialDenied(reason) => match reason {
hiroute_domain::NativeCredentialAccessDeniedV1::SourceDisabled => MATERIALIZATION_SOURCE_DISABLED,
hiroute_domain::NativeCredentialAccessDeniedV1::CredentialDisabled => MATERIALIZATION_KEY_DISABLED,
},
_ => MATERIALIZATION_AUTHORITY_FAILED,
}))?
.ok_or_else(|| Arc::from(MATERIALIZATION_CREDENTIAL_EXHAUSTED))?,
)
};
let permits = if let Some(credential) = credential.as_ref() {
if credential.credential_ref() != context.credential_ref().as_str() {
return Err(Arc::from(MATERIALIZATION_AUTHORITY_FAILED));
}
let credential_key = RuntimeStateKey::credential(
endpoint_state_key.clone(),
credential.credential_ref(),
credential.key_id(),
credential.generation(),
);
let credential_permit =
acquire_target_permit_status(provider.state.as_ref(), &credential_key, &scope)
.await
.map_err(|_| {
runtime_state_authority.fail();
Arc::from(RUNTIME_STATE_AUTHORITY_FAILED)
})?;
let credential_permit = permit_or_materialization_error(credential_permit, false)?;
super::OwnedAttemptStatePermits::new(
binding_key,
binding_permit,
credential_key,
credential_permit,
)
} else {
super::OwnedAttemptStatePermits::without_credential(binding_key, binding_permit)
};
let resolved_target = match credential
.as_ref()
.and_then(|credential| credential.transport_override())
{
Some(target) => {
if execution.connector_runtime != hiroute_domain::ConnectorRuntimeKind::CpaBridge
|| execution.operational_target.runtime_epoch().is_none()
|| target.request_path() != profile.connector.request_path
{
return Err(Arc::from(MATERIALIZATION_AUTHORITY_FAILED));
}
let mut resolved = plan.transport_target.clone();
resolved.scheme = TransportScheme::Http;
resolved.authority = target.address().to_string().into();
resolved.addresses = Arc::from([target.address()]);
resolved.sni = None;
resolved.connection_fingerprint = resolved.derive_connection_fingerprint();
resolved
.validate()
.map_err(|_| Arc::from(MATERIALIZATION_AUTHORITY_FAILED))?;
resolved
}
None => {
if execution.connector_runtime == hiroute_domain::ConnectorRuntimeKind::CpaBridge {
return Err(Arc::from(MATERIALIZATION_AUTHORITY_FAILED));
}
let target = transport_target_for_profile(plan, &lease_target)?;
provider.target_resolver.resolve(target, &scope).await?
}
};
validate_attempt_permits(provider.state.as_ref(), permits.borrowed(), &scope)
.await
.map_err(|_| {
runtime_state_authority.fail();
Arc::from(RUNTIME_STATE_AUTHORITY_FAILED)
})?;
let quantum = context
.write_quantum
.min(plan.body_plans.attempt_request.max_chunk_bytes());
let chat_tool_projection_bytes = prepared
.chat_tool_projection
.as_ref()
.map(|projection| {
projection
.retained_bytes()
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))
})
.transpose()?;
let chat_tool_projection_budget = match chat_tool_projection_bytes {
Some(bytes) if bytes != 0 => Some(
context
.budget
.reserve(MemoryRole::AttemptWire, bytes)
.map_err(safe_error)?,
),
Some(_) | None => None,
};
#[cfg(all(unix, debug_assertions))]
let capture = crate::runtime::stream_capture::Capture::begin(
profile,
prepared.chat_tool_projection.as_ref(),
prepared.wire_len,
streaming,
);
let chat_tool_projection = prepared.chat_tool_projection.take();
let body = adapters::sequential_attempt_body(
prepared.clone(),
logical.replay.clone(),
context.budget,
quantum,
)
.map_err(safe_error)?;
#[cfg(all(unix, debug_assertions))]
let body = match capture.as_ref() {
Some(capture) => capture.wrap(body),
None => body,
};
let lease = context.leases.acquire().map_err(safe_error)?;
let mut headers = HeaderMap::new();
let authority = resolved_target.authority.as_ref();
headers.insert(
HOST,
HeaderValue::from_str(authority).map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?,
);
let connector_headers = profile
.connector
.headers
.exact()
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
headers.insert(
CONTENT_TYPE,
HeaderValue::from_str(&connector_headers.content_type)
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?,
);
for (name, value) in &connector_headers.required_headers {
let name = http::header::HeaderName::from_bytes(name.as_bytes())
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
let value =
HeaderValue::from_str(value).map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
headers.insert(name, value);
}
headers.insert(CONTENT_LENGTH, HeaderValue::from(prepared.wire_len as u64));
if let Some(credential) = credential.as_ref() {
credential
.apply_authorization(&mut headers)
.map_err(|_| Arc::from(MATERIALIZATION_AUTHORITY_FAILED))?;
}
headers.insert(
http::header::HeaderName::from_static("x-hiroute-profile-digest"),
HeaderValue::from_str(expected_profile_digest)
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?,
);
headers.insert(
http::header::HeaderName::from_static("x-hiroute-reason-ledger-id"),
HeaderValue::from_str(
candidate
.reason_ledger_identity
.as_ref()
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?
.as_str(),
)
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?,
);
let body = PreparedAttemptBody::from_reader(body, lease).map_err(safe_error)?;
if let Some(observation) = crate::server::core_runtime::observation::active_request() {
observation.prepared_request_diagnostic(&headers, &prepared.bytes);
use hiroute_diagnostics::event::ReasoningCleanupReason;
let reason = if cleanup_prefix.is_none() {
ReasoningCleanupReason::RequestProtocolProjection
} else if logical
.reasoning_cleanup
.used
.load(std::sync::atomic::Ordering::Acquire)
{
ReasoningCleanupReason::ContextBreakRetry
} else {
ReasoningCleanupReason::SuccessfulPrefixReuse
};
observation.reasoning_cleanup(
reason,
adapters::reasoning_loss::request_loss(
ir,
profile.capability.upstream_protocol,
cleanup_prefix,
),
cleanup_prefix,
);
}
Ok((
PreparedAttemptHttpRequest {
head: PreparedRequestHead {
method: Method::POST,
path_and_query: prepared.path.into(),
headers,
},
body,
},
ProductionAttemptState {
managed_cpa: execution.connector_runtime
== hiroute_domain::ConnectorRuntimeKind::CpaBridge
&& !resolved_target.addresses.is_empty()
&& resolved_target
.addresses
.iter()
.all(|address| address.ip().is_loopback()),
#[cfg(all(unix, debug_assertions))]
capture,
response_status: None,
resolved_target,
permits,
classified_failure: None,
profile: profile.clone(),
client_profile,
served_model_alias,
streaming,
native_output,
omit_thinking: ir.requested_reasoning.messages_omit_thinking,
chat_tool_projection,
chat_tool_projection_budget,
decoder: None,
renderer,
projector: None,
prefix: Some(prefix),
prefix_terminal_chunks: None,
budget: context.budget.clone(),
semantic_seen: false,
semantic_terminal: None,
retry_after: None,
runtime_state_authority,
},
))
}
fn client_profile_for_candidate(
candidate: &CandidateProtocolProfile,
) -> Result<ClientProtocolProfile, Arc<str>> {
ClientProtocolProfile::for_candidate(candidate)
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))
}
pub(super) fn resolved_transport_target(
state: &ProductionAttemptState,
binding: &AttemptExecutionBinding,
) -> Option<TransportTarget> {
(state.resolved_target != binding.plan().transport_target)
.then(|| state.resolved_target.clone())
}
pub(super) fn classify_materialization_failure(error: &Arc<str>) -> AttemptMaterializationFailure {
let binding_cooldown = parse_cooldown(error, MATERIALIZATION_BINDING_COOLING_PREFIX);
let credential_cooldown = parse_cooldown(error, MATERIALIZATION_CREDENTIAL_COOLING_PREFIX);
let cooldown = binding_cooldown.or(credential_cooldown);
let probe_busy = matches!(
error.as_ref(),
MATERIALIZATION_BINDING_PROBE_BUSY | MATERIALIZATION_CREDENTIAL_PROBE_BUSY
);
let disabled = matches!(
error.as_ref(),
MATERIALIZATION_BINDING_DISABLED | MATERIALIZATION_CREDENTIAL_DISABLED
);
let retryable = cooldown.is_some()
|| probe_busy
|| disabled
|| matches!(
error.as_ref(),
MATERIALIZATION_BINDING_UNAVAILABLE
| MATERIALIZATION_CREDENTIAL_EXHAUSTED
| MATERIALIZATION_SOURCE_DISABLED
| MATERIALIZATION_KEY_DISABLED
| MATERIALIZATION_DNS_FAILED
);
let class = match error.as_ref() {
MATERIALIZATION_DNS_FAILED => AttemptMaterializationFailureClass::Dns,
MATERIALIZATION_CREDENTIAL_EXHAUSTED
| MATERIALIZATION_SOURCE_DISABLED
| MATERIALIZATION_KEY_DISABLED
| MATERIALIZATION_CREDENTIAL_PROBE_BUSY
| MATERIALIZATION_CREDENTIAL_DISABLED => AttemptMaterializationFailureClass::Credential,
_ if credential_cooldown.is_some() => AttemptMaterializationFailureClass::Credential,
_ => AttemptMaterializationFailureClass::Materialization,
};
AttemptMaterializationFailure {
class,
provider: ProviderClassificationFacts {
error_class: Some(label(if cooldown.is_some() {
"runtime_state_cooling_down"
} else if probe_busy {
"runtime_state_probe_busy"
} else if disabled {
"runtime_state_disabled"
} else if error.as_ref() == MATERIALIZATION_SOURCE_DISABLED {
"source_disabled"
} else if error.as_ref() == MATERIALIZATION_KEY_DISABLED {
"credential_disabled"
} else if error.as_ref() == MATERIALIZATION_CREDENTIAL_EXHAUSTED {
"credential_exhausted"
} else if retryable {
"materialization_retryable"
} else {
"materialization_fail_closed"
})),
retryability: if retryable {
RetryabilityFact::Retryable
} else {
RetryabilityFact::NonRetryable
},
retry_after: cooldown.or_else(|| probe_busy.then(|| Duration::from_secs(1))),
readiness: label("materialization_failed"),
..ProviderClassificationFacts::default()
},
termination_reason: label(match class {
AttemptMaterializationFailureClass::Credential => "credential",
AttemptMaterializationFailureClass::Dns => "dns",
AttemptMaterializationFailureClass::Materialization => "materialization",
}),
}
}
fn permit_or_materialization_error(
availability: PermitAvailability,
binding: bool,
) -> Result<StatePermit, Arc<str>> {
match availability {
PermitAvailability::Permit(permit) => Ok(permit),
PermitAvailability::CoolingDown { until } => {
let remaining = until.saturating_duration_since(Instant::now());
let rounded_millis = remaining
.as_millis()
.saturating_add(u128::from(remaining.subsec_nanos() % 1_000_000 != 0));
let millis = u64::try_from(rounded_millis).unwrap_or(u64::MAX).max(1);
let prefix = if binding {
MATERIALIZATION_BINDING_COOLING_PREFIX
} else {
MATERIALIZATION_CREDENTIAL_COOLING_PREFIX
};
Err(Arc::from(format!("{prefix}{millis}")))
}
PermitAvailability::ProbeBusy => Err(Arc::from(if binding {
MATERIALIZATION_BINDING_PROBE_BUSY
} else {
MATERIALIZATION_CREDENTIAL_PROBE_BUSY
})),
PermitAvailability::Disabled => Err(Arc::from(if binding {
MATERIALIZATION_BINDING_DISABLED
} else {
MATERIALIZATION_CREDENTIAL_DISABLED
})),
}
}
fn parse_cooldown(error: &str, prefix: &str) -> Option<Duration> {
error
.strip_prefix(prefix)?
.parse::<u64>()
.ok()
.filter(|millis| *millis != 0)
.map(Duration::from_millis)
}
pub(super) async fn confirm_precommit(
provider: &ProductionProvider,
state: &mut ProductionAttemptState,
deadline: Instant,
cancellation: &tokio_util::sync::CancellationToken,
) -> Result<(), Arc<str>> {
let scope = ExecutionScope::new(deadline, cancellation.clone());
let authority = state.runtime_state_authority.clone();
match &state.classified_failure {
Some(failure) => record_failure(
provider.state.as_ref(),
state.permits.borrowed(),
failure,
&provider.cooldowns,
&scope,
)
.await
.map_err(|_| {
authority.fail();
Arc::from(RUNTIME_STATE_AUTHORITY_FAILED)
}),
None => confirm_success(provider.state.as_ref(), state.permits.borrowed(), &scope)
.await
.map_err(|_| {
authority.fail();
Arc::from(RUNTIME_STATE_AUTHORITY_FAILED)
}),
}
}
pub(super) async fn confirm_attempt_failure(
provider: &ProductionProvider,
state: &mut ProductionAttemptState,
failure: &AttemptFailureFacts,
deadline: Instant,
cancellation: &tokio_util::sync::CancellationToken,
) -> Result<Option<ProviderClassificationFacts>, Arc<str>> {
let Some((classified, facts)) = confirm_mechanical_state(
provider.state.as_ref(),
&state.permits,
&state.profile,
&provider.cooldowns,
MechanicalFailureSignal {
class: failure.class,
timeout: failure.transport.timeout,
runtime_state_authority: state.runtime_state_authority.clone(),
},
deadline,
cancellation,
)
.await?
else {
return Ok(failure.provider.clone());
};
state.classified_failure = Some(classified);
Ok(Some(facts))
}
pub(super) fn release_terminal_request(mut logical: ProductionLogicalRequest) {
let _ = logical.replay.release_stream(&logical.raw_body);
logical.replay.mark_terminal();
logical.ir.take();
}
async fn confirm_mechanical_state(
store: &dyn RuntimeStateStore,
permits: &super::OwnedAttemptStatePermits,
profile: &CandidateProtocolProfile,
cooldowns: &RuntimeCooldownPolicy,
failure: MechanicalFailureSignal,
deadline: Instant,
cancellation: &tokio_util::sync::CancellationToken,
) -> Result<Option<(AttemptFailure, ProviderClassificationFacts)>, Arc<str>> {
let raw = match failure.class {
CoreAttemptFailureClass::Connect
if failure.timeout == Some(AttemptTimeoutKind::Connect) =>
{
RawAttemptFailure::Timeout {
phase: crate::attempt_outcome::AttemptTimeoutPhase::Connect,
}
}
CoreAttemptFailureClass::Connect => RawAttemptFailure::Connect,
CoreAttemptFailureClass::RequestWrite => RawAttemptFailure::Timeout {
phase: crate::attempt_outcome::AttemptTimeoutPhase::RequestWrite,
},
CoreAttemptFailureClass::FirstByte => RawAttemptFailure::Timeout {
phase: crate::attempt_outcome::AttemptTimeoutPhase::FirstByte,
},
CoreAttemptFailureClass::StreamIdle => RawAttemptFailure::Timeout {
phase: crate::attempt_outcome::AttemptTimeoutPhase::StreamIdle,
},
CoreAttemptFailureClass::AttemptDeadline => RawAttemptFailure::Timeout {
phase: crate::attempt_outcome::AttemptTimeoutPhase::Overall,
},
CoreAttemptFailureClass::Transport => RawAttemptFailure::Disconnect,
CoreAttemptFailureClass::Credential
| CoreAttemptFailureClass::Dns
| CoreAttemptFailureClass::Materialization
| CoreAttemptFailureClass::AttemptRequestFilter => return Ok(None),
};
let classified = classify_failure(&raw, connector_error_profile(profile)?);
let facts = failure_facts(&classified);
let scope = ExecutionScope::new(deadline, cancellation.clone());
record_failure(store, permits.borrowed(), &classified, cooldowns, &scope)
.await
.map_err(|_| {
failure.runtime_state_authority.fail();
Arc::from(RUNTIME_STATE_AUTHORITY_FAILED)
})?;
Ok(Some((classified, facts)))
}
/// DNS resolution and the optional controlled dial mapping. Constructed at the
/// caller's original configuration boundary, independently of model credentials
/// and runtime-state ownership.
pub(crate) struct TargetResolver {
#[cfg(feature = "e2e-test-control")]
test_dial: Result<Option<crate::server::test_control::E2eDialMap>, ()>,
}
impl TargetResolver {
pub(crate) fn new() -> Self {
Self {
#[cfg(feature = "e2e-test-control")]
test_dial: crate::server::test_control::E2eDialMap::from_environment(),
}
}
pub(crate) async fn resolve(
&self,
target: TransportTarget,
scope: &ExecutionScope,
) -> Result<TransportTarget, Arc<str>> {
#[cfg(feature = "e2e-test-control")]
{
let test_dial = self
.test_dial
.as_ref()
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
if test_dial
.as_ref()
.is_some_and(|dial| dial.fails_dns(&target))
{
return Err(Arc::from(MATERIALIZATION_DNS_FAILED));
}
if let Some(mapped) = test_dial
.as_ref()
.map(|dial| dial.apply(target.clone()))
.transpose()
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?
.flatten()
{
return Ok(mapped);
}
}
let Some(authority) = target.unresolved_authority().map(ToOwned::to_owned) else {
return Ok(target);
};
let (host, port) = dns_lookup_target(&authority, target.scheme)?;
let addresses = scope
.run(tokio::net::lookup_host((host.as_str(), port)))
.await
.map_err(|_| Arc::from(MATERIALIZATION_DNS_FAILED))?
.map_err(|_| Arc::from(MATERIALIZATION_DNS_FAILED))?
.collect::<Vec<SocketAddr>>();
if addresses.is_empty() {
return Err(Arc::from(MATERIALIZATION_DNS_FAILED));
}
target
.with_resolved_addresses(addresses.into())
.map_err(|_| Arc::from(MATERIALIZATION_DNS_FAILED))
}
}
fn transport_target_for_profile(
plan: &CompiledAttemptPlan,
operational: &hiroute_domain::GatewayOperationalTargetV1,
) -> Result<TransportTarget, Arc<str>> {
let uri = operational
.uri()
.parse::<http::Uri>()
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
let scheme = match uri.scheme_str() {
Some("http") => TransportScheme::Http,
Some("https") => TransportScheme::Https,
_ => return Err(Arc::from(MATERIALIZATION_PROTOCOL_FAILED)),
};
let authority = uri
.authority()
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
let mut matching = std::iter::once(&plan.transport_target)
.chain(plan.authorized_native_targets.iter())
.filter(|target| {
target.scheme == scheme
&& target.unresolved_authority().unwrap_or(&target.authority) == authority.as_str()
});
let exact = matching
.next()
.ok_or_else(|| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
if matching.next().is_some() {
return Err(Arc::from(MATERIALIZATION_PROTOCOL_FAILED));
}
Ok(exact.clone())
}
fn dns_lookup_target(authority: &str, scheme: TransportScheme) -> Result<(String, u16), Arc<str>> {
let authority = authority
.parse::<http::uri::Authority>()
.map_err(|_| Arc::from(MATERIALIZATION_DNS_FAILED))?;
let port = authority.port_u16().unwrap_or(match scheme {
TransportScheme::Http => 80,
TransportScheme::Https => 443,
});
Ok((authority.host().to_owned(), port))
}
fn operational_target_matches_plan(
operational: &hiroute_domain::GatewayOperationalTargetV1,
plan: &CompiledAttemptPlan,
) -> bool {
let Ok(uri) = operational.uri().parse::<http::Uri>() else {
return false;
};
let Some(authority) = uri.authority() else {
return false;
};
let scheme_matches = matches!(
(uri.scheme_str(), plan.transport_target.scheme),
(Some("http"), TransportScheme::Http) | (Some("https"), TransportScheme::Https)
);
let plan_authority = plan
.transport_target
.unresolved_authority()
.unwrap_or(&plan.transport_target.authority);
if !scheme_matches || authority.as_str() != plan_authority {
return false;
}
match operational {
hiroute_domain::GatewayOperationalTargetV1::RegisteredHttps { .. } => {
plan.transport_target.requires_resolution()
}
hiroute_domain::GatewayOperationalTargetV1::UserConfiguredNative { .. } => {
let numeric_host = authority
.host()
.strip_prefix('[')
.and_then(|host| host.strip_suffix(']'))
.unwrap_or_else(|| authority.host());
match numeric_host.parse::<std::net::IpAddr>() {
Ok(ip) => {
!plan.transport_target.requires_resolution()
&& plan.transport_target.addresses.len() == 1
&& plan.transport_target.addresses[0]
== SocketAddr::new(
ip,
authority.port_u16().unwrap_or(
if uri.scheme_str() == Some("https") {
443
} else {
80
},
),
)
}
Err(_) => plan.transport_target.requires_resolution(),
}
}
hiroute_domain::GatewayOperationalTargetV1::ManagedCpaLoopback { .. } => {
let numeric_host = authority
.host()
.strip_prefix('[')
.and_then(|host| host.strip_suffix(']'))
.unwrap_or_else(|| authority.host());
!plan.transport_target.requires_resolution()
&& plan.transport_target.addresses.len() == 1
&& numeric_host
.parse::<std::net::IpAddr>()
.ok()
.zip(authority.port_u16())
.is_some_and(|(ip, port)| {
plan.transport_target.addresses[0] == SocketAddr::new(ip, port)
})
}
}
}
fn profile_digest(profile: &CandidateProtocolProfile) -> Result<String, Arc<str>> {
PlannerCandidateFactsV1::seal(
"digest-check",
"digest-check",
profile.clone(),
1,
None,
CostClassV1::Free,
None,
)
.map(|candidate| candidate.profile_digest)
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))
}
#[cfg(test)]
#[path = "materialization_tests.rs"]
mod tests;