Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

fix(gateway): return structured candidate preparation errors (#110)

澄潭ac012a3cec94
diff --git a/crates/daemon/src/control/runtime.rs b/crates/daemon/src/control/runtime.rs
index 444e51e6..f198421f 100644
--- a/crates/daemon/src/control/runtime.rs
+++ b/crates/daemon/src/control/runtime.rs
@@ -829,7 +829,7 @@ impl NativeCredentialAuthorityV1 for ControlNativeCredentialAuthority {
fn lease_native_credential(
&self,
request: &NativeCredentialLeaseRequestV1,
- ) -> PortResult<Option<NativeCredentialLeaseV1>> {
+ ) -> Result<Option<NativeCredentialLeaseV1>, hiroute_domain::NativeCredentialLeaseErrorV1> {
self.adapter
.stores
.lock()
diff --git a/crates/daemon/src/gateway_ports/credential.rs b/crates/daemon/src/gateway_ports/credential.rs
index 4b1ee040..23fb1f80 100644
--- a/crates/daemon/src/gateway_ports/credential.rs
+++ b/crates/daemon/src/gateway_ports/credential.rs
@@ -99,11 +99,18 @@ where
let Some(lease) = self
.native
.lease_native_credential(&native_request)
- .map_err(|error| match error.code {
- hiroute_domain::PortErrorCode::Unavailable => {
- PortError::Unavailable("NativeCredentialAuthorityV1")
+ .map_err(|error| match error {
+ hiroute_domain::NativeCredentialLeaseErrorV1::Denied(reason) => {
+ PortError::CredentialDenied(reason)
+ }
+ hiroute_domain::NativeCredentialLeaseErrorV1::Authority(error) => {
+ match error.code {
+ hiroute_domain::PortErrorCode::Unavailable => {
+ PortError::Unavailable("NativeCredentialAuthorityV1")
+ }
+ _ => PortError::Rejected,
+ }
}
- _ => PortError::Rejected,
})?
else {
return Ok(None);
diff --git a/crates/daemon/src/gateway_ports/tests.rs b/crates/daemon/src/gateway_ports/tests.rs
index 8d38e96e..cff0f0f9 100644
--- a/crates/daemon/src/gateway_ports/tests.rs
+++ b/crates/daemon/src/gateway_ports/tests.rs
@@ -1,4 +1,5 @@
mod cpa_async;
+mod credential_denial;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
@@ -410,17 +411,21 @@ async fn gateway_ports_runtime_state_preserves_durable_cas_and_typed_probe_outco
struct FakeNativeAuthority {
requests: Mutex<Vec<NativeCredentialLeaseRequestV1>>,
header_secret_requests: Mutex<Vec<HeaderSecretLeaseRequestV1>>,
+ denial: Option<hiroute_domain::NativeCredentialAccessDeniedV1>,
}
impl NativeCredentialAuthorityV1 for FakeNativeAuthority {
fn lease_native_credential(
&self,
request: &NativeCredentialLeaseRequestV1,
- ) -> PortResult<Option<NativeCredentialLeaseV1>> {
+ ) -> Result<Option<NativeCredentialLeaseV1>, hiroute_domain::NativeCredentialLeaseErrorV1> {
request.validate().map_err(|_| {
ProductPortError::new(PortErrorCode::InvalidData, "test.native.request")
})?;
self.requests.lock().unwrap().push(request.clone());
+ if let Some(reason) = self.denial {
+ return Err(hiroute_domain::NativeCredentialLeaseErrorV1::Denied(reason));
+ }
NativeCredentialLeaseV1::issue(
request.credential_id.clone(),
"native-key",
@@ -430,7 +435,7 @@ impl NativeCredentialAuthorityV1 for FakeNativeAuthority {
}),
)
.map(Some)
- .map_err(|_| ProductPortError::new(PortErrorCode::Corrupt, "test.native.lease"))
+ .map_err(|_| ProductPortError::new(PortErrorCode::Corrupt, "test.native.lease").into())
}
fn lease_header_secret(
diff --git a/crates/daemon/src/gateway_ports/tests/credential_denial.rs b/crates/daemon/src/gateway_ports/tests/credential_denial.rs
new file mode 100644
index 00000000..02a24404
--- /dev/null
+++ b/crates/daemon/src/gateway_ports/tests/credential_denial.rs
@@ -0,0 +1,51 @@
+use super::*;
+use hiroute_domain::NativeCredentialAccessDeniedV1;
+use hiroute_gateway::server::composition::PortError;
+
+#[tokio::test]
+async fn exact_native_lease_keeps_source_and_key_denials_distinct_from_authority_errors() {
+ for reason in [
+ NativeCredentialAccessDeniedV1::SourceDisabled,
+ NativeCredentialAccessDeniedV1::CredentialDisabled,
+ ] {
+ let native = Arc::new(FakeNativeAuthority {
+ denial: Some(reason),
+ ..FakeNativeAuthority::default()
+ });
+ let resolver = GatewayCredentialResolver::new(
+ Arc::clone(&native),
+ Arc::new(FakeCpaAuthority::default()),
+ );
+ let target = GatewayOperationalTargetV1::RegisteredHttps {
+ uri: "https://api.anthropic.com/v1/messages".into(),
+ };
+ let digest = CanonicalDigest::of(&target).unwrap();
+ let profile = CanonicalDigest::of_bytes(b"native-profile");
+ let result = resolver
+ .lease_exact(
+ CredentialLeaseRequest {
+ stable_binding_id: "binding/claude",
+ credential_ref: "credential/claude",
+ credential_destination_ref: "connection-option/claude.custom.v1",
+ excluded_key_ids: &[],
+ connector_runtime: ConnectorRuntimeKind::BuiltinNative,
+ connector_id: "builtin-anthropic",
+ upstream_protocol: IngressProtocol::Messages,
+ upstream_model_id: "claude-sonnet",
+ native_transport_model: "claude-sonnet",
+ logical_endpoint: target.uri(),
+ operational_target: target.uri(),
+ operational_target_digest: digest.as_str(),
+ runtime_epoch: None,
+ target_epoch: None,
+ protocol_profile_digest: profile.as_str(),
+ request_path: "/v1/messages",
+ authentication: &AuthenticationSemantics::Bearer,
+ },
+ &scope(),
+ )
+ .await;
+ assert_eq!(result.err(), Some(PortError::CredentialDenied(reason)));
+ assert_eq!(native.requests.lock().unwrap().len(), 1);
+ }
+}
diff --git a/crates/daemon/src/gateway_ports/tests/observation_collection.rs b/crates/daemon/src/gateway_ports/tests/observation_collection.rs
index 4e34127a..f5517925 100644
--- a/crates/daemon/src/gateway_ports/tests/observation_collection.rs
+++ b/crates/daemon/src/gateway_ports/tests/observation_collection.rs
@@ -1,5 +1,8 @@
use std::sync::Arc;
+use hiroute_domain::{
+ FactsCompleteness, ObservationQueryPort, ReceiptId, RequestOutcome, WorkspaceId,
+};
use hiroute_gateway::server::core_runtime::observation::{
EXECUTION_FACT_PORT_DIGEST, EXECUTION_FACT_SCHEMA,
};
@@ -11,6 +14,116 @@ use super::super::GatewayExecutionFactSink;
const DIGEST: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+#[test]
+fn current_credential_denials_reach_real_product_ingestion_with_released_outcomes() {
+ let directory = tempfile::tempdir().unwrap();
+ let store = Arc::new(
+ LocalObservationStore::open(directory.path(), DigestAuthority::new([32; 32])).unwrap(),
+ );
+ let sink = GatewayExecutionFactSink::new(store);
+ for (index, outcome) in [
+ "leased",
+ "exhausted",
+ "authority_error",
+ "source_disabled",
+ "credential_disabled",
+ ]
+ .into_iter()
+ .enumerate()
+ {
+ let sequence = u64::try_from(index).unwrap() + 1;
+ let mut payload = execution_payload(sequence, &format!("request-{sequence}"), None);
+ payload.as_object_mut().unwrap().remove("attempt_id");
+ payload["fact"] = json!({
+ "kind": "credential_lease", "stable_binding_id": "binding-main", "credential_ref": "credential-main",
+ "key_id": if outcome == "leased" { json!("key-main") } else { Value::Null },
+ "credential_generation": if outcome == "leased" { json!(1) } else { Value::Null },
+ "excluded_key_count": 0, "outcome": outcome
+ });
+ let result = sink
+ .project_and_ingest_payload_for_test(&serde_json::to_vec(&payload).unwrap())
+ .unwrap();
+ assert!(matches!(result, IngestOutcome::Ack(_)), "outcome {outcome}");
+ }
+}
+
+#[test]
+fn current_credential_denials_close_complete_zero_attempt_receipts_and_reopen() {
+ let directory = tempfile::tempdir().unwrap();
+ let store = Arc::new(
+ LocalObservationStore::open(directory.path(), DigestAuthority::new([33; 32])).unwrap(),
+ );
+ let sink = GatewayExecutionFactSink::new(store.clone());
+ let mut sequence = 0;
+ let mut receipts = Vec::new();
+ for outcome in ["source_disabled", "credential_disabled"] {
+ for stream in [false, true] {
+ let request_id = format!("request-{outcome}-{stream}");
+ let lease = json!({
+ "kind": "credential_lease", "stable_binding_id": "binding-main",
+ "credential_ref": "credential-main", "key_id": null,
+ "credential_generation": null, "excluded_key_count": 0, "outcome": outcome
+ });
+ let finished = json!({
+ "kind": "request_finished", "outcome": "failed", "attempts_started": 0,
+ "attempts_finished": 0, "accepted_attempt_ordinal": null,
+ "facts_completeness": "complete"
+ });
+ for fact in [route_decision(stream), lease.clone(), finished.clone()] {
+ sequence += 1;
+ let mut payload = execution_payload(sequence, &request_id, None);
+ payload.as_object_mut().unwrap().remove("attempt_id");
+ payload["fact"] = fact;
+ let result = sink
+ .project_and_ingest_payload_for_test(&serde_json::to_vec(&payload).unwrap())
+ .unwrap();
+ assert!(matches!(result, IngestOutcome::Ack(_)));
+ }
+ let receipt_id = ReceiptId::parse(request_id.clone()).unwrap();
+ let receipt = store
+ .get_receipt(&WorkspaceId::default(), &receipt_id)
+ .unwrap();
+ receipt.validate().unwrap();
+ assert_eq!(receipt.request_id.as_str(), request_id);
+ assert_eq!(receipt.outcome, RequestOutcome::Failed);
+ assert_eq!(receipt.facts_completeness, FactsCompleteness::Complete);
+ assert!(receipt.final_attempt_id.is_none());
+ assert_eq!(receipt.ordered_facts.len(), 3);
+ assert!(
+ receipt
+ .ordered_facts
+ .iter()
+ .all(|event| event.attempt_id.is_none())
+ );
+ assert_eq!(
+ serde_json::to_value(&receipt.ordered_facts[0].fact).unwrap(),
+ route_decision(stream)
+ );
+ assert_eq!(
+ serde_json::to_value(&receipt.ordered_facts[1].fact).unwrap(),
+ lease
+ );
+ assert_eq!(
+ serde_json::to_value(&receipt.ordered_facts[2].fact).unwrap(),
+ finished
+ );
+ let digest = receipt.digest().unwrap();
+ receipts.push((receipt_id, receipt, digest));
+ }
+ }
+ drop(sink);
+ drop(store);
+ let reopened =
+ LocalObservationStore::open(directory.path(), DigestAuthority::new([33; 32])).unwrap();
+ for (receipt_id, receipt, digest) in receipts {
+ let read = reopened
+ .get_receipt(&WorkspaceId::default(), &receipt_id)
+ .unwrap();
+ assert_eq!(read, receipt);
+ assert_eq!(read.digest().unwrap(), digest);
+ }
+}
+
#[test]
fn current_gateway_payloads_collect_priced_and_unpriced_attempts_in_one_product_table() {
let directory = tempfile::tempdir().unwrap();
@@ -111,6 +224,28 @@ fn execution_payload(sequence: u64, request_id: &str, pricing: Option<Value>) ->
})
}
+fn route_decision(stream: bool) -> Value {
+ json!({
+ "kind": "route_decision", "planner_version": "hiroute-deterministic-planner/v1",
+ "plan_id": "plan/coding",
+ "route": {"kind": "plan", "revision": 7, "semantic_digest": DIGEST},
+ "input_digest": DIGEST, "policy_digest": DIGEST, "output_digest": DIGEST,
+ "branch": "custom_exact_order", "complexity": null, "groups": [], "reason_ledger": [],
+ "requirements": {
+ "text": true, "image_url": false, "image_base64": false, "image_media_types": [],
+ "function_tools": false, "strict_tools": false, "parallel_tools": false,
+ "tool_choice": {"kind": "auto"}, "tool_result_text": false, "tool_result_json": false,
+ "tool_roundtrip": false, "logical_tool_id_mapping": false, "provider_state": false,
+ "initial_instructions": false, "mid_conversation_instructions": false,
+ "streaming": stream, "stream_text": stream, "stream_reasoning": false,
+ "stream_tool_arguments": false, "stream_usage": false, "ingress_protocol": "responses"
+ },
+ "requested_reasoning_disposition": "absent", "requested_reasoning_value": null,
+ "requested_max_output_tokens": null, "stream": stream, "outcome": "ready",
+ "outcome_code": null, "max_attempts": 1
+ })
+}
+
fn pricing() -> Value {
json!({
"schema_version": "hiroute.observation.execution-pricing/v1",
diff --git a/crates/daemon/tests/support/model_connections_product.py b/crates/daemon/tests/support/model_connections_product.py
index 11e3135f..6fd62f50 100644
--- a/crates/daemon/tests/support/model_connections_product.py
+++ b/crates/daemon/tests/support/model_connections_product.py
@@ -458,13 +458,13 @@ def publish_plan_and_agent(product, binding_id, native_model_mode='preserve_avai
assert [model['id'] for model in catalog['data']] == expected_models, catalog
-def gateway_request(product, allowed=True):
+def gateway_request(product, allowed=True, stream=False):
client = http.client.HTTPConnection('127.0.0.1', product.port, timeout=30)
try:
client.request('POST', '/v1/responses', body=encoded({
'model': product.model_alias,
'input': 'native product request',
- 'stream': False,
+ 'stream': stream,
}), headers={
'X-HiRoute-Token': product.bearer(product.agent_connection),
'Content-Type': 'application/json',
@@ -475,7 +475,13 @@ def gateway_request(product, allowed=True):
if allowed:
assert response.status == 200 and b'native product answer' in body, (response.status, body)
else:
- assert response.status >= 400 and b'native product answer' not in body, (response.status, body)
+ assert response.status == 502, (response.status, body)
+ assert response.getheader('Content-Type') == 'application/json', response.getheaders()
+ assert response.getheader('Retry-After') is None, response.getheaders()
+ assert json.loads(body) == {
+ 'schema_version': 'hiroute.gateway.error/v1',
+ 'code': 'SOURCE_DISABLED', 'phase': 'credential_lease',
+ }, body
finally:
client.close()
@@ -515,7 +521,8 @@ def assert_live_credential_switches(product, upstream, saved):
def rejected_without_upstream():
with upstream.lock:
before = len(upstream.requests)
- gateway_request(product, allowed=False)
+ for stream in (False, True):
+ gateway_request(product, allowed=False, stream=stream)
with upstream.lock:
assert len(upstream.requests) == before, 'disabled credential reached the upstream'
@@ -643,7 +650,9 @@ def run(repository, expected_sha=None):
'saved_binding': saved['binding_id'],
'saved_source_rechecks': 2,
'gateway_attempts': 3,
- 'disabled_requests_without_upstream': 3,
+ 'disabled_requests_without_upstream': 6,
+ 'preparation_error_code': 'SOURCE_DISABLED',
+ 'preparation_error_modes': ['nonstream', 'prebody_stream'],
'credential_resolver_mode': 'saved_bearer',
'session_record': session_id,
'session_model': observed_model,
diff --git a/crates/domain/src/compute/credential_runtime.rs b/crates/domain/src/compute/credential_runtime.rs
index 69374dca..20ed174a 100644
--- a/crates/domain/src/compute/credential_runtime.rs
+++ b/crates/domain/src/compute/credential_runtime.rs
@@ -244,10 +244,12 @@ impl NativeCredentialLeaseV1 {
/// Replaceable Product authority. Local storage, a Vault client, or a remote authority can
/// implement this same port; Gateway adapters never depend on SQLite types.
pub trait NativeCredentialAuthorityV1: Send + Sync {
+ /// An empty lease means exact exhaustion. Current managed source/Key permission
+ /// denials retain their closed reason before any secret is decrypted.
fn lease_native_credential(
&self,
request: &NativeCredentialLeaseRequestV1,
- ) -> PortResult<Option<NativeCredentialLeaseV1>>;
+ ) -> Result<Option<NativeCredentialLeaseV1>, NativeCredentialLeaseErrorV1>;
fn lease_header_secret(
&self,
@@ -255,6 +257,21 @@ pub trait NativeCredentialAuthorityV1: Send + Sync {
) -> PortResult<Option<NativeCredentialLeaseV1>>;
}
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum NativeCredentialAccessDeniedV1 {
+ SourceDisabled,
+ CredentialDisabled,
+}
+
+/// In-process lease failure, separate from persisted credential and request schemas.
+#[derive(Clone, Debug, Error, Eq, PartialEq)]
+pub enum NativeCredentialLeaseErrorV1 {
+ #[error("current managed credential permission was denied: {0:?}")]
+ Denied(NativeCredentialAccessDeniedV1),
+ #[error(transparent)]
+ Authority(#[from] crate::PortError),
+}
+
#[derive(Clone, Copy, Debug, Error, Eq, PartialEq)]
pub enum NativeCredentialContractErrorV1 {
#[error("native credential request is invalid")]
diff --git a/crates/domain/src/observation/execution_attempt.rs b/crates/domain/src/observation/execution_attempt.rs
index 4bf0478c..791f5596 100644
--- a/crates/domain/src/observation/execution_attempt.rs
+++ b/crates/domain/src/observation/execution_attempt.rs
@@ -28,6 +28,8 @@ pub enum CredentialLeaseOutcomeV1 {
Leased,
Exhausted,
AuthorityError,
+ SourceDisabled,
+ CredentialDisabled,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
diff --git a/crates/domain/src/observation/execution_tests.rs b/crates/domain/src/observation/execution_tests.rs
index 5e570925..84eae7fe 100644
--- a/crates/domain/src/observation/execution_tests.rs
+++ b/crates/domain/src/observation/execution_tests.rs
@@ -4,6 +4,31 @@ use crate::{AgentPlanId, CanonicalDigest, WorkspaceId};
use super::*;
+#[test]
+fn credential_denial_extension_keeps_released_outcomes_and_rejects_unknown_values() {
+ // The three original outcomes were verified at supported v0.2.0/v0.3.0/v0.3.1 sources.
+ for (wire, value) in [
+ ("leased", CredentialLeaseOutcomeV1::Leased),
+ ("exhausted", CredentialLeaseOutcomeV1::Exhausted),
+ ("authority_error", CredentialLeaseOutcomeV1::AuthorityError),
+ ("source_disabled", CredentialLeaseOutcomeV1::SourceDisabled),
+ (
+ "credential_disabled",
+ CredentialLeaseOutcomeV1::CredentialDisabled,
+ ),
+ ] {
+ let bytes = format!("\"{wire}\"");
+ assert_eq!(
+ serde_json::from_str::<CredentialLeaseOutcomeV1>(&bytes).unwrap(),
+ value
+ );
+ assert_eq!(serde_json::to_string(&value).unwrap(), bytes);
+ }
+ assert!(
+ serde_json::from_str::<CredentialLeaseOutcomeV1>("\"provider_private_detail\"").is_err()
+ );
+}
+
#[test]
fn adapter_fact_contract_roundtrips_every_gateway_variant_and_identity_field() {
let envelopes = gateway_fact_envelopes();
diff --git a/crates/gateway-core/src/runtime/driver/decision.rs b/crates/gateway-core/src/runtime/driver/decision.rs
index 03c84a70..ec60ccf4 100644
--- a/crates/gateway-core/src/runtime/driver/decision.rs
+++ b/crates/gateway-core/src/runtime/driver/decision.rs
@@ -456,9 +456,10 @@ pub trait DecisionSessionPort: Send {
request: SelectionRequest<'_>,
) -> Result<Option<SelectedGatewayAttempt>, Arc<str>>;
- /// Optional product-owned response when selection ended before any
- /// upstream attempt. Core keeps its generic 502 when no exact response is
- /// available.
+ /// Optional product-owned response when selection is exhausted or a provisional
+ /// candidate terminates before an upstream attempt. Core keeps its generic 502
+ /// when no exact response is available. This does not change failure disposition
+ /// or publish a formal attempt for a provisional candidate.
fn exhaustion_local_reply(&mut self, _now: Instant) -> Result<Option<LocalReply>, Arc<str>> {
Ok(None)
}
diff --git a/crates/gateway-core/src/runtime/driver/lifecycle/preexchange/failure.rs b/crates/gateway-core/src/runtime/driver/lifecycle/preexchange/failure.rs
index 5cfbec1c..c6bb3856 100644
--- a/crates/gateway-core/src/runtime/driver/lifecycle/preexchange/failure.rs
+++ b/crates/gateway-core/src/runtime/driver/lifecycle/preexchange/failure.rs
@@ -170,6 +170,15 @@ where
)
.map_err(GatewayExecutionError::ProviderTerminalRelease)?;
let _release_ready = leases.mark_terminal_and_release_ready()?;
+ let local_reply = decision_session
+ .exhaustion_local_reply(Instant::now())
+ .map_err(GatewayExecutionError::Selection)?
+ .unwrap_or_else(|| LocalReply {
+ status: StatusCode::BAD_GATEWAY,
+ headers: HeaderMap::new(),
+ body: Bytes::from_static(b"upstream candidate preparation failed"),
+ provenance: SemanticProvenance::NonSemantic,
+ });
let result = emit_local_response(
session,
final_writer,
@@ -178,12 +187,7 @@ where
binding,
request_configs,
&mut request_filters.filters,
- LocalReply {
- status: StatusCode::BAD_GATEWAY,
- headers: HeaderMap::new(),
- body: Bytes::from_static(b"upstream candidate preparation failed"),
- provenance: SemanticProvenance::NonSemantic,
- },
+ local_reply,
budget,
request_id,
Some(route_binding),
diff --git a/crates/gateway-core/tests/gateway_lifecycle/fixture/selection.rs b/crates/gateway-core/tests/gateway_lifecycle/fixture/selection.rs
index 90d19c65..cd9c7cea 100644
--- a/crates/gateway-core/tests/gateway_lifecycle/fixture/selection.rs
+++ b/crates/gateway-core/tests/gateway_lifecycle/fixture/selection.rs
@@ -19,6 +19,8 @@ pub(crate) struct SelectionState {
pub(crate) fail_observation_sink: bool,
pub(crate) routing_snapshots: VecDeque<RealtimeRoutingFacts>,
pub(crate) preexchange_completion_trace: Option<Arc<Mutex<Vec<PreexchangeCompletionStep>>>>,
+ pub(crate) provisional_candidates: bool,
+ pub(crate) exhaustion_reply: Option<LocalReply>,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -61,6 +63,8 @@ impl TestSelection {
fail_observation_sink: false,
routing_snapshots: VecDeque::new(),
preexchange_completion_trace: None,
+ provisional_candidates: false,
+ exhaustion_reply: None,
})),
}
}
@@ -70,6 +74,14 @@ impl TestSelection {
self
}
+ pub(crate) fn with_provisional_reply(self, reply: Option<LocalReply>) -> Self {
+ let mut state = self.state.lock().expect("selection state");
+ state.provisional_candidates = true;
+ state.exhaustion_reply = reply;
+ drop(state);
+ self
+ }
+
pub(crate) fn with_attempt_budgets(
self,
attempt_budgets: impl IntoIterator<Item = Duration>,
@@ -268,7 +280,18 @@ impl DecisionSessionPort for TestDecisionSession {
.unwrap_or(request.generation);
let selected = SelectedGatewayAttempt {
request_id: request.request_id,
- attempt_id: hiroute_gateway_core::runtime::attempt::AttemptId(self.next_attempt_id),
+ attempt_id: hiroute_gateway_core::runtime::attempt::AttemptId(
+ if self
+ .state
+ .lock()
+ .map_err(|_| Arc::from("poisoned selector"))?
+ .provisional_candidates
+ {
+ 0
+ } else {
+ self.next_attempt_id
+ },
+ ),
generation,
binding,
credential_ref: CredentialRef::new(format!(
@@ -302,6 +325,15 @@ impl DecisionSessionPort for TestDecisionSession {
Ok(Some(selected))
}
+ fn exhaustion_local_reply(&mut self, _now: Instant) -> Result<Option<LocalReply>, Arc<str>> {
+ Ok(self
+ .state
+ .lock()
+ .map_err(|_| Arc::from("poisoned selector"))?
+ .exhaustion_reply
+ .clone())
+ }
+
fn decide(
&mut self,
_selected: &SelectedGatewayAttempt,
diff --git a/crates/gateway-core/tests/gateway_lifecycle/preexchange.rs b/crates/gateway-core/tests/gateway_lifecycle/preexchange.rs
index 49e8d904..90430963 100644
--- a/crates/gateway-core/tests/gateway_lifecycle/preexchange.rs
+++ b/crates/gateway-core/tests/gateway_lifecycle/preexchange.rs
@@ -1,5 +1,91 @@
use crate::fixture::*;
+#[tokio::test]
+async fn provisional_termination_uses_product_reply_without_publishing_an_attempt()
+-> Result<(), TestError> {
+ let _network_guard = NETWORK_TEST_LOCK.lock().await;
+ for product_reply in [true, false] {
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let plan = PlanRevision(7591);
+ let binding = ResolvedTargetBindingId::new(plan, 1);
+ let publications =
+ install_publication(BootstrapPublicationBuilder::new(plan.0, 59).route(
+ "gateway.test",
+ "/preparation-denied",
+ 1,
+ plain_target(listener.local_addr()?, 59),
+ )?)?;
+ let mut headers = HeaderMap::new();
+ headers.insert(
+ http::header::CONTENT_TYPE,
+ http::HeaderValue::from_static("application/json"),
+ );
+ let reply = LocalReply {
+ status: StatusCode::BAD_GATEWAY,
+ headers,
+ body: Bytes::from_static(br#"{"schema_version":"hiroute.gateway.error/v1","code":"SOURCE_DISABLED","phase":"credential_lease"}"#),
+ provenance: SemanticProvenance::NonSemantic,
+ };
+ let selection = TestSelection::new([binding])
+ .with_provisional_reply(product_reply.then_some(reply.clone()));
+ let provider = PassthroughProvider {
+ materialization_failure: Some(AttemptMaterializationFailure {
+ class: AttemptMaterializationFailureClass::Credential,
+ provider: ProviderClassificationFacts {
+ error_class: Some(ObservationLabel::new("source_disabled").unwrap()),
+ retryability: RetryabilityFact::Retryable,
+ readiness: ObservationLabel::new("materialization_failed").unwrap(),
+ ..ProviderClassificationFacts::default()
+ },
+ termination_reason: ObservationLabel::new("credential").unwrap(),
+ }),
+ materialization_failures_remaining: Arc::new(AtomicUsize::new(1)),
+ materialization_raw_error: Some(Arc::from("private provider credential detail")),
+ ..PassthroughProvider::default()
+ };
+ let finalized = Arc::clone(&provider.finalized_attempts);
+ let gateway = lifecycle_with_provider(
+ publications,
+ selection.clone(),
+ provider,
+ TrackingFilters::default(),
+ )?;
+ let mut session = RecordingSession::new(
+ "gateway.test",
+ "/preparation-denied",
+ Bytes::from_static(b"prompt"),
+ HttpProtocol::Http1,
+ );
+ assert_eq!(gateway.process(&mut session).await?, SessionReuse::Reusable);
+ assert_eq!(
+ session.response_head.as_ref().unwrap().status,
+ StatusCode::BAD_GATEWAY
+ );
+ if product_reply {
+ assert_eq!(session.response_body, reply.body);
+ assert_eq!(
+ session.response_head.as_ref().unwrap().headers[http::header::CONTENT_TYPE],
+ "application/json"
+ );
+ } else {
+ assert_eq!(
+ session.response_body,
+ b"upstream candidate preparation failed"
+ );
+ }
+ assert_eq!(selection.failures(), [AttemptFailureClass::Credential]);
+ assert!(selection.completed().is_empty());
+ assert!(selection.published().is_empty());
+ assert_eq!(finalized.load(Ordering::Relaxed), 0);
+ assert!(
+ tokio::time::timeout(Duration::from_millis(50), listener.accept())
+ .await
+ .is_err()
+ );
+ }
+ Ok(())
+}
+
#[tokio::test]
async fn materialized_attempt_filter_failure_finalizes_before_fallback_selection()
-> Result<(), TestError> {
diff --git a/crates/gateway/README.md b/crates/gateway/README.md
index c20b96fc..ab0fe797 100644
--- a/crates/gateway/README.md
+++ b/crates/gateway/README.md
@@ -161,6 +161,41 @@ error message or a substitute for evidence of credential failure.
## Response failure evidence
+### Candidate preparation errors
+
+A published plan freezes candidate selection, while the exact credential authority
+still checks current source and Key permission before decryption. A managed source
+or Key denial retains its closed cause through the daemon adapter; an empty lease
+means only that the exact credential is exhausted. Neither outcome starts an upstream
+attempt or broadens the frozen fallback policy.
+
+When preparation terminates with no upstream attempt, the core asks the existing
+decision-session local-reply port for the product response. HiRoute returns HTTP 502
+with `Content-Type: application/json`, `schema_version: hiroute.gateway.error/v1`,
+and these closed fields:
+
+| Preparation evidence | `code` | `phase` |
+| --- | --- | --- |
+| Every rejected candidate has a managed source disabled | `SOURCE_DISABLED` | `credential_lease` |
+| Every rejected candidate has a managed Key disabled | `CREDENTIAL_DISABLED` | `credential_lease` |
+| Every exact credential lease is exhausted | `CREDENTIAL_UNAVAILABLE` | `credential_lease` |
+| Other or mixed preparation failures | `CANDIDATE_PREPARATION_FAILED` | `candidate_preparation` |
+
+These replies have no `Retry-After`. Pure cooldown/probe exhaustion retains its
+existing HTTP 503 runtime-state response and bounded retry hint. A failure after a
+formal upstream attempt keeps the existing attempt response; this port cannot rewrite
+committed bytes. No provider text, credential identity or authorization material is
+included in these local replies.
+
+Request-level `credential_lease` execution facts retain the denial as `source_disabled`
+or `credential_disabled`; exhausted and authority errors remain distinct. Their
+existing envelope is unchanged; the Domain ingestion enum accepts these two new values
+alongside the original leased, exhausted and authority-error outcomes. With observation enabled and a
+healthy sink, the request finishes as failed with zero started/finished attempts,
+no accepted attempt and no fabricated usage, and can produce a zero-attempt receipt.
+Follow the [live source/Key regression](../daemon/tests/support/model_connections_product.py)
+for disable, restart, independent-source isolation and re-enable evidence.
+
`runtime/driver/response_diagnostics.rs` owns the closed, payload-free failure
classification; `adapters/response/decoder.rs` reports SSE event ordinal and
received-byte upper bound without per-token logs. Unix Debug-only
diff --git a/crates/gateway/src/composition/ports.rs b/crates/gateway/src/composition/ports.rs
index 69d7915b..3c7ffcc2 100644
--- a/crates/gateway/src/composition/ports.rs
+++ b/crates/gateway/src/composition/ports.rs
@@ -707,6 +707,8 @@ pub enum PortError {
Unavailable(&'static str),
#[error("production port rejected the operation")]
Rejected,
+ #[error("current managed credential permission was denied: {0:?}")]
+ CredentialDenied(hiroute_domain::NativeCredentialAccessDeniedV1),
#[error("requested fixed-model reasoning cannot be represented")]
InvalidReasoningControl,
}
diff --git a/crates/gateway/src/observation/ports.rs b/crates/gateway/src/observation/ports.rs
index 08293daf..1d296506 100644
--- a/crates/gateway/src/observation/ports.rs
+++ b/crates/gateway/src/observation/ports.rs
@@ -57,6 +57,20 @@ impl CredentialResolver for ObservedCredentialResolver {
None,
"exhausted",
),
+ Err(PortError::CredentialDenied(reason)) => observation.credential_lease(
+ &stable_binding_id,
+ &credential_ref,
+ excluded_key_count,
+ None,
+ match reason {
+ hiroute_domain::NativeCredentialAccessDeniedV1::SourceDisabled => {
+ "source_disabled"
+ }
+ hiroute_domain::NativeCredentialAccessDeniedV1::CredentialDisabled => {
+ "credential_disabled"
+ }
+ },
+ ),
Err(_) => observation.credential_lease(
&stable_binding_id,
&credential_ref,
diff --git a/crates/gateway/src/runtime/driver.rs b/crates/gateway/src/runtime/driver.rs
index fa710cc0..5a69e036 100644
--- a/crates/gateway/src/runtime/driver.rs
+++ b/crates/gateway/src/runtime/driver.rs
@@ -48,6 +48,8 @@ pub use selection::{ProductionDecisionSession, ProductionSelection};
const MATERIALIZATION_BINDING_UNAVAILABLE: &str = "binding_unavailable";
const MATERIALIZATION_CREDENTIAL_EXHAUSTED: &str = "credential_exhausted";
+const MATERIALIZATION_SOURCE_DISABLED: &str = "managed_source_disabled";
+const MATERIALIZATION_KEY_DISABLED: &str = "managed_key_disabled";
const MATERIALIZATION_BINDING_COOLING_PREFIX: &str = "binding_cooling_down:";
const MATERIALIZATION_CREDENTIAL_COOLING_PREFIX: &str = "credential_cooling_down:";
const MATERIALIZATION_BINDING_PROBE_BUSY: &str = "binding_probe_busy";
diff --git a/crates/gateway/src/runtime/driver/materialization.rs b/crates/gateway/src/runtime/driver/materialization.rs
index b6b34236..4fa65951 100644
--- a/crates/gateway/src/runtime/driver/materialization.rs
+++ b/crates/gateway/src/runtime/driver/materialization.rs
@@ -44,11 +44,11 @@ use super::{
MATERIALIZATION_BINDING_UNAVAILABLE, MATERIALIZATION_CREDENTIAL_COOLING_PREFIX,
MATERIALIZATION_CREDENTIAL_DISABLED, MATERIALIZATION_CREDENTIAL_EXHAUSTED,
MATERIALIZATION_CREDENTIAL_PROBE_BUSY, MATERIALIZATION_DNS_FAILED,
- MATERIALIZATION_PROTOCOL_FAILED, MechanicalFailureSignal, PermitAvailability,
- ProductionAttemptState, ProductionLogicalRequest, ProductionProvider,
- ProductionReplaySeedEnvelope, RUNTIME_STATE_AUTHORITY_FAILED, RuntimeCooldownPolicy,
- StatePermit, acquire_target_permit_status, confirm_success, label, record_failure, safe_error,
- validate_attempt_permits,
+ MATERIALIZATION_KEY_DISABLED, MATERIALIZATION_PROTOCOL_FAILED, MATERIALIZATION_SOURCE_DISABLED,
+ MechanicalFailureSignal, PermitAvailability, ProductionAttemptState, ProductionLogicalRequest,
+ ProductionProvider, ProductionReplaySeedEnvelope, RUNTIME_STATE_AUTHORITY_FAILED,
+ RuntimeCooldownPolicy, StatePermit, acquire_target_permit_status, confirm_success, label,
+ record_failure, safe_error, validate_attempt_permits,
};
pub(super) struct ProductionStateAdapter {
@@ -369,7 +369,13 @@ pub(super) async fn materialize_attempt(
))
.await
.map_err(|_| Arc::from(MATERIALIZATION_AUTHORITY_FAILED))?
- .map_err(|_| Arc::from(MATERIALIZATION_AUTHORITY_FAILED))?
+ .map_err(|error| Arc::from(match error {
+ crate::server::composition::PortError::CredentialDenied(reason) => match reason {
+ hiroute_domain::NativeCredentialAccessDeniedV1::SourceDisabled => MATERIALIZATION_SOURCE_DISABLED,
+ hiroute_domain::NativeCredentialAccessDeniedV1::CredentialDisabled => MATERIALIZATION_KEY_DISABLED,
+ },
+ _ => MATERIALIZATION_AUTHORITY_FAILED,
+ }))?
.ok_or_else(|| Arc::from(MATERIALIZATION_CREDENTIAL_EXHAUSTED))?,
)
};
@@ -629,11 +635,15 @@ pub(super) fn classify_materialization_failure(error: &Arc<str>) -> AttemptMater
error.as_ref(),
MATERIALIZATION_BINDING_UNAVAILABLE
| MATERIALIZATION_CREDENTIAL_EXHAUSTED
+ | MATERIALIZATION_SOURCE_DISABLED
+ | MATERIALIZATION_KEY_DISABLED
| MATERIALIZATION_DNS_FAILED
);
let class = match error.as_ref() {
MATERIALIZATION_DNS_FAILED => AttemptMaterializationFailureClass::Dns,
MATERIALIZATION_CREDENTIAL_EXHAUSTED
+ | MATERIALIZATION_SOURCE_DISABLED
+ | MATERIALIZATION_KEY_DISABLED
| MATERIALIZATION_CREDENTIAL_PROBE_BUSY
| MATERIALIZATION_CREDENTIAL_DISABLED => AttemptMaterializationFailureClass::Credential,
_ if credential_cooldown.is_some() => AttemptMaterializationFailureClass::Credential,
@@ -648,6 +658,12 @@ pub(super) fn classify_materialization_failure(error: &Arc<str>) -> AttemptMater
"runtime_state_probe_busy"
} else if disabled {
"runtime_state_disabled"
+ } else if error.as_ref() == MATERIALIZATION_SOURCE_DISABLED {
+ "source_disabled"
+ } else if error.as_ref() == MATERIALIZATION_KEY_DISABLED {
+ "credential_disabled"
+ } else if error.as_ref() == MATERIALIZATION_CREDENTIAL_EXHAUSTED {
+ "credential_exhausted"
} else if retryable {
"materialization_retryable"
} else {
diff --git a/crates/gateway/src/runtime/driver/selection.rs b/crates/gateway/src/runtime/driver/selection.rs
index d49772b6..f84f034a 100644
--- a/crates/gateway/src/runtime/driver/selection.rs
+++ b/crates/gateway/src/runtime/driver/selection.rs
@@ -22,6 +22,9 @@ use crate::context_hold::{ContextHoldStore, HoldCompletion};
use super::ProductionRouteContext;
+mod preparation;
+use preparation::PreparationFailure;
+
#[derive(Clone, Default)]
pub struct ProductionSelection;
@@ -40,6 +43,7 @@ pub struct ProductionDecisionSession {
earliest_retry_at: Option<Instant>,
non_temporary_block_seen: bool,
upstream_attempted: bool,
+ preparation_failure: Option<PreparationFailure>,
}
impl DecisionSessionPort for ProductionDecisionSession {
@@ -92,9 +96,14 @@ impl DecisionSessionPort for ProductionDecisionSession {
}
fn exhaustion_local_reply(&mut self, now: Instant) -> Result<Option<LocalReply>, Arc<str>> {
- if !self.temporary_block_seen || self.non_temporary_block_seen || self.upstream_attempted {
+ if self.upstream_attempted {
return Ok(None);
}
+ if !self.temporary_block_seen || self.non_temporary_block_seen {
+ return Ok(self
+ .preparation_failure
+ .map(PreparationFailure::local_reply));
+ }
let (code, retry_after_seconds) = if self.probe_busy_seen {
("RECOVERY_PROBE_IN_PROGRESS", 1)
} else {
@@ -195,6 +204,13 @@ impl DecisionSessionPort for ProductionDecisionSession {
error_class,
Some("runtime_state_cooling_down" | "runtime_state_probe_busy")
);
+ if selected.attempt_id.0 == 0 {
+ let cause = PreparationFailure::from_error_class(error_class);
+ self.preparation_failure = Some(
+ self.preparation_failure
+ .map_or(cause, |previous| previous.merge(cause)),
+ );
+ }
if temporary && selected.attempt_id.0 == 0 {
self.temporary_block_seen = true;
if error_class == Some("runtime_state_probe_busy") {
@@ -351,6 +367,7 @@ impl SelectionPublicationPort<ProductionRouteContext> for ProductionSelection {
earliest_retry_at: None,
non_temporary_block_seen: false,
upstream_attempted: false,
+ preparation_failure: None,
})
}
}
@@ -411,686 +428,4 @@ impl ProductionDecisionSession {
}
#[cfg(test)]
-mod tests {
- use std::time::Duration;
-
- use hiroute_gateway_core::core::execution_plan::{
- CredentialRef, PlanRevision, ResolvedTargetBindingId,
- };
- use hiroute_gateway_core::runtime::attempt::{
- AttemptGeneration, AttemptTransportFacts, CommitFence, RequestId,
- };
- use hiroute_gateway_core::runtime::driver::{
- AttemptCommitFacts, AttemptFailureClass, AttemptFailureFacts,
- };
-
- use super::*;
-
- fn candidate(binding_local_id: u32, credential_refs: &[&str]) -> DecisionCandidateAuthority {
- DecisionCandidateAuthority {
- binding: ResolvedTargetBindingId::new(PlanRevision(1), binding_local_id),
- stable_target: ObservationLabel::new(format!("binding-{binding_local_id}")).unwrap(),
- credential_refs: credential_refs
- .iter()
- .map(|value| CredentialRef::new(*value).unwrap())
- .collect::<Vec<_>>()
- .into(),
- candidate_id: None,
- profile_digest: None,
- reason_ledger_identity: None,
- provider_profile: None,
- }
- }
-
- fn selection_request<'a>(
- _now: Instant,
- generation: u64,
- remaining_attempts: u32,
- realtime_facts: &'a RealtimeRoutingFacts,
- ) -> SelectionRequest<'a> {
- SelectionRequest {
- request_id: RequestId(1),
- generation: AttemptGeneration(generation),
- route_binding: ResolvedTargetBindingId::new(PlanRevision(1), 1),
- remaining_total: Duration::from_secs(1),
- remaining_attempts,
- realtime_facts,
- previous_attempts: &[],
- }
- }
-
- fn session(
- now: Instant,
- candidates: Vec<DecisionCandidateAuthority>,
- ) -> ProductionDecisionSession {
- ProductionDecisionSession {
- request_id: RequestId(1),
- route_decision_id: RouteDecisionId(1),
- candidates: candidates.into(),
- candidate_index: 0,
- credential_index: 0,
- overall_deadline: now + Duration::from_secs(10),
- hold: None,
- switch_fallback: None,
- reasoning_cleanup: Arc::new(super::super::ReasoningCleanup::new(None, None)),
- temporary_block_seen: false,
- probe_busy_seen: false,
- earliest_retry_at: None,
- non_temporary_block_seen: false,
- upstream_attempted: false,
- }
- }
-
- fn materialization_failure(
- now: Instant,
- error_class: &str,
- retry_after: Option<Duration>,
- ) -> AttemptFailureFacts {
- AttemptFailureFacts {
- class: AttemptFailureClass::Materialization,
- provider: Some(ProviderClassificationFacts {
- error_class: Some(ObservationLabel::new(error_class).unwrap()),
- retryability: RetryabilityFact::Retryable,
- retry_after,
- readiness: ObservationLabel::new("materialization_failed").unwrap(),
- ..ProviderClassificationFacts::default()
- }),
- transport: AttemptTransportFacts {
- started_at: now,
- upstream_protocol: None,
- connect_elapsed: None,
- request_write_elapsed: None,
- upstream_ttfb: None,
- last_upstream_progress_at: None,
- local_read_suppressed: Duration::ZERO,
- upstream_body_bytes: 0,
- timeout: None,
- },
- request_committed: false,
- termination_reason: ObservationLabel::new("materialization").unwrap(),
- }
- }
-
- #[test]
- fn retryable_quota_advances_exact_credential_before_binding() {
- let now = Instant::now();
- let realtime_facts = RealtimeRoutingFacts::default();
- let transport = AttemptTransportFacts {
- started_at: now,
- upstream_protocol: None,
- connect_elapsed: None,
- request_write_elapsed: None,
- upstream_ttfb: None,
- last_upstream_progress_at: None,
- local_read_suppressed: Duration::ZERO,
- upstream_body_bytes: 0,
- timeout: None,
- };
- let mut session = ProductionDecisionSession {
- request_id: RequestId(1),
- route_decision_id: RouteDecisionId(1),
- candidates: vec![
- candidate(1, &["credential-a", "credential-b"]),
- candidate(2, &["credential-c"]),
- ]
- .into(),
- candidate_index: 0,
- credential_index: 0,
- overall_deadline: now + Duration::from_secs(1),
- hold: None,
- switch_fallback: None,
- reasoning_cleanup: Arc::new(super::super::ReasoningCleanup::new(None, None)),
- temporary_block_seen: false,
- probe_busy_seen: false,
- earliest_retry_at: None,
- non_temporary_block_seen: false,
- upstream_attempted: false,
- };
- let first = session
- .select_next(selection_request(now, 1, 3, &realtime_facts))
- .unwrap()
- .unwrap();
- assert_eq!(first.binding.local_id(), 1);
- assert_eq!(first.credential_ref.as_str(), "credential-a");
-
- let quota = ProviderClassificationFacts {
- error_class: Some(ObservationLabel::new("quota").unwrap()),
- retryability: RetryabilityFact::Retryable,
- readiness: ObservationLabel::new("rejected").unwrap(),
- ..ProviderClassificationFacts::default()
- };
- assert_eq!(
- session.decide(&first, &quota, &transport).unwrap(),
- Disposition::Continue
- );
- let second = session
- .select_next(selection_request(now, 2, 2, &realtime_facts))
- .unwrap()
- .unwrap();
- assert_eq!(second.binding.local_id(), 1);
- assert_eq!(second.credential_ref.as_str(), "credential-b");
-
- assert_eq!(
- session.decide(&second, &quota, &transport).unwrap(),
- Disposition::Continue
- );
- let third = session
- .select_next(selection_request(now, 3, 1, &realtime_facts))
- .unwrap()
- .unwrap();
- assert_eq!(third.binding.local_id(), 2);
- assert_eq!(third.credential_ref.as_str(), "credential-c");
- }
-
- #[test]
- fn switched_candidate_precommit_400_falls_back_to_previous_once() {
- let now = Instant::now();
- let realtime_facts = RealtimeRoutingFacts::default();
- let mut session = session(
- now,
- vec![candidate(1, &["new-key"]), candidate(2, &["original-key"])],
- );
- session.switch_fallback = Some(ResolvedTargetBindingId::new(PlanRevision(1), 2));
- let mut selected = session
- .select_next(selection_request(now, 1, 2, &realtime_facts))
- .unwrap()
- .unwrap();
- // The core promotes the candidate after pre-exchange materialization.
- selected.attempt_id = AttemptId(1);
- let rejected = ProviderClassificationFacts {
- error_class: Some(ObservationLabel::new("permanent_client").unwrap()),
- retryability: RetryabilityFact::NonRetryable,
- http_status: Some(StatusCode::BAD_REQUEST),
- ..ProviderClassificationFacts::default()
- };
- assert_eq!(
- session
- .decide(
- &selected,
- &rejected,
- &AttemptTransportFacts {
- started_at: now,
- upstream_protocol: None,
- connect_elapsed: None,
- request_write_elapsed: None,
- upstream_ttfb: None,
- last_upstream_progress_at: None,
- local_read_suppressed: Duration::ZERO,
- upstream_body_bytes: 0,
- timeout: None,
- }
- )
- .unwrap(),
- Disposition::Continue
- );
- let original = session
- .select_next(selection_request(now, 2, 1, &realtime_facts))
- .unwrap()
- .unwrap();
- assert_eq!(original.binding.local_id(), 2);
- assert_eq!(
- session
- .decide(
- &original,
- &rejected,
- &AttemptTransportFacts {
- started_at: now,
- upstream_protocol: None,
- connect_elapsed: None,
- request_write_elapsed: None,
- upstream_ttfb: None,
- last_upstream_progress_at: None,
- local_read_suppressed: Duration::ZERO,
- upstream_body_bytes: 0,
- timeout: None,
- }
- )
- .unwrap(),
- Disposition::Terminate
- );
- }
-
- #[test]
- fn explicit_history_rejection_retries_only_eligible_first_candidate_once() {
- let now = Instant::now();
- let facts = RealtimeRoutingFacts::default();
- let mut session = session(now, vec![candidate(1, &["b"]), candidate(2, &["a"])]);
- let binding = session.candidates[0].binding;
- session.reasoning_cleanup = Arc::new(super::super::ReasoningCleanup::new(
- Some((binding, 3)),
- None,
- ));
- let rejected = ProviderClassificationFacts {
- error_class: Some(ObservationLabel::new("reasoning_history").unwrap()),
- retryability: RetryabilityFact::NonRetryable,
- http_status: Some(StatusCode::BAD_REQUEST),
- ..ProviderClassificationFacts::default()
- };
- let transport = AttemptTransportFacts {
- started_at: now,
- upstream_protocol: None,
- connect_elapsed: None,
- request_write_elapsed: None,
- upstream_ttfb: None,
- last_upstream_progress_at: None,
- local_read_suppressed: Duration::ZERO,
- upstream_body_bytes: 0,
- timeout: None,
- };
- let first = session
- .select_next(selection_request(now, 1, 3, &facts))
- .unwrap()
- .unwrap();
- assert_eq!(session.reasoning_cleanup.prefix_for(binding), None);
- assert_eq!(
- session.decide(&first, &rejected, &transport).unwrap(),
- Disposition::Continue
- );
- let retry = session
- .select_next(selection_request(now, 2, 2, &facts))
- .unwrap()
- .unwrap();
- assert_eq!(retry.binding, binding);
- assert_eq!(session.reasoning_cleanup.prefix_for(binding), Some(3));
- assert_eq!(
- session.decide(&retry, &rejected, &transport).unwrap(),
- Disposition::Continue
- );
- let fallback = session
- .select_next(selection_request(now, 3, 1, &facts))
- .unwrap()
- .unwrap();
- assert_ne!(fallback.binding, binding);
- assert_eq!(session.reasoning_cleanup.prefix_for(fallback.binding), None);
- assert_eq!(
- session.decide(&fallback, &rejected, &transport).unwrap(),
- Disposition::Terminate
- );
- assert!(
- session
- .select_next(selection_request(now, 4, 0, &facts))
- .unwrap()
- .is_none()
- );
- }
-
- #[test]
- fn history_error_without_context_break_authorization_does_not_enable_cleanup() {
- let now = Instant::now();
- let facts = RealtimeRoutingFacts::default();
- let mut session = session(now, vec![candidate(1, &["a"]), candidate(2, &["b"])]);
- let selected = session
- .select_next(selection_request(now, 1, 3, &facts))
- .unwrap()
- .unwrap();
- let mut failure = materialization_failure(now, "reasoning_history", None);
- failure.provider.as_mut().unwrap().retryability = RetryabilityFact::NonRetryable;
- assert!(!session.reasoning_cleanup.consume(selected.binding));
- assert_eq!(session.reasoning_cleanup.prefix_for(selected.binding), None);
- assert_eq!(
- session.decide_failure(&selected, &failure).unwrap(),
- Disposition::Terminate
- );
- }
-
- #[test]
- fn unknown_or_nonretryable_materialization_never_advances_selection() {
- let now = Instant::now();
- let selected = SelectedGatewayAttempt {
- request_id: RequestId(1),
- attempt_id: AttemptId(0),
- generation: AttemptGeneration(1),
- binding: ResolvedTargetBindingId::new(PlanRevision(1), 1),
- credential_ref: CredentialRef::new("credential").unwrap(),
- route_decision_id: RouteDecisionId(1),
- budget: AttemptBudgetGrant {
- issued_at: now,
- allocated: Duration::from_secs(1),
- deadline: now + Duration::from_secs(1),
- },
- };
- let mut session = ProductionDecisionSession {
- request_id: RequestId(1),
- route_decision_id: RouteDecisionId(1),
- candidates: Arc::new([]),
- candidate_index: 0,
- credential_index: 0,
- overall_deadline: now + Duration::from_secs(1),
- hold: None,
- switch_fallback: None,
- reasoning_cleanup: Arc::new(super::super::ReasoningCleanup::new(None, None)),
- temporary_block_seen: false,
- probe_busy_seen: false,
- earliest_retry_at: None,
- non_temporary_block_seen: false,
- upstream_attempted: false,
- };
- for retryability in [RetryabilityFact::Unknown, RetryabilityFact::NonRetryable] {
- let failure = AttemptFailureFacts {
- class: AttemptFailureClass::Materialization,
- provider: Some(ProviderClassificationFacts {
- retryability,
- readiness: ObservationLabel::new("materialization_failed").unwrap(),
- ..ProviderClassificationFacts::default()
- }),
- transport: AttemptTransportFacts {
- started_at: now,
- upstream_protocol: None,
- connect_elapsed: None,
- request_write_elapsed: None,
- upstream_ttfb: None,
- last_upstream_progress_at: None,
- local_read_suppressed: Duration::ZERO,
- upstream_body_bytes: 0,
- timeout: None,
- },
- request_committed: false,
- termination_reason: ObservationLabel::new("materialization").unwrap(),
- };
- assert_eq!(
- session.decide_failure(&selected, &failure).unwrap(),
- Disposition::Terminate
- );
- assert_eq!(session.candidate_index, 0);
- assert_eq!(session.credential_index, 0);
- }
- }
-
- #[test]
- fn switched_provisional_nonretryable_failure_does_not_call_previous_model() {
- let now = Instant::now();
- let realtime_facts = RealtimeRoutingFacts::default();
- let mut session = session(
- now,
- vec![candidate(1, &["new-key"]), candidate(2, &["original-key"])],
- );
- session.switch_fallback = Some(ResolvedTargetBindingId::new(PlanRevision(1), 2));
- let selected = session
- .select_next(selection_request(now, 1, 2, &realtime_facts))
- .unwrap()
- .unwrap();
- assert_eq!(selected.attempt_id, AttemptId(0));
-
- let mut failure = materialization_failure(now, "materialization_fail_closed", None);
- failure.provider.as_mut().unwrap().retryability = RetryabilityFact::NonRetryable;
- assert_eq!(
- session.decide_failure(&selected, &failure).unwrap(),
- Disposition::Terminate
- );
- assert_eq!(session.candidate_index, 0);
- }
-
- #[test]
- fn switched_promoted_preexchange_failure_does_not_call_previous_model() {
- let now = Instant::now();
- let realtime_facts = RealtimeRoutingFacts::default();
- let mut session = session(
- now,
- vec![candidate(1, &["new-key"]), candidate(2, &["original-key"])],
- );
- session.switch_fallback = Some(ResolvedTargetBindingId::new(PlanRevision(1), 2));
- let mut selected = session
- .select_next(selection_request(now, 1, 2, &realtime_facts))
- .unwrap()
- .unwrap();
- selected.attempt_id = AttemptId(1);
-
- let mut failure = materialization_failure(now, "preexchange_filter_failure", None);
- failure.provider.as_mut().unwrap().retryability = RetryabilityFact::NonRetryable;
- assert!(!failure.request_committed);
- assert_eq!(
- session.decide_failure(&selected, &failure).unwrap(),
- Disposition::Terminate
- );
- assert_eq!(session.candidate_index, 0);
- }
-
- #[test]
- fn pure_runtime_state_exhaustion_returns_retryable_503_and_probe_busy_wins() {
- let now = Instant::now();
- let realtime = RealtimeRoutingFacts::default();
- let mut session = session(
- now,
- vec![
- candidate(1, &["credential-a"]),
- candidate(2, &["credential-b"]),
- ],
- );
- let first = session
- .select_next(selection_request(now, 1, 3, &realtime))
- .unwrap()
- .unwrap();
- assert_eq!(
- session
- .decide_failure(
- &first,
- &materialization_failure(
- now,
- "runtime_state_cooling_down",
- Some(Duration::from_millis(1_500)),
- ),
- )
- .unwrap(),
- Disposition::Continue
- );
- let second = session
- .select_next(selection_request(now, 1, 3, &realtime))
- .unwrap()
- .unwrap();
- assert_eq!(
- session
- .decide_failure(
- &second,
- &materialization_failure(
- now,
- "runtime_state_probe_busy",
- Some(Duration::from_secs(1)),
- ),
- )
- .unwrap(),
- Disposition::Continue
- );
- assert!(
- session
- .select_next(selection_request(now, 1, 3, &realtime))
- .unwrap()
- .is_none()
- );
- let reply = session
- .exhaustion_local_reply(now)
- .unwrap()
- .expect("pure temporary exhaustion has a retry response");
- assert_eq!(reply.status, StatusCode::SERVICE_UNAVAILABLE);
- assert_eq!(reply.headers.get(RETRY_AFTER).unwrap(), "1");
- assert_eq!(reply.headers.get(CONTENT_TYPE).unwrap(), "application/json");
- assert_eq!(
- serde_json::from_slice::<serde_json::Value>(&reply.body).unwrap(),
- serde_json::json!({
- "schema_version": "hiroute.gateway.error/v1",
- "code": "RECOVERY_PROBE_IN_PROGRESS",
- "phase": "runtime_state",
- "retry_after_seconds": 1,
- })
- );
- }
-
- #[test]
- fn cooling_exhaustion_uses_earliest_deadline_and_rounds_up() {
- assert_eq!(retry_after_seconds(Duration::from_nanos(1)), 1);
- assert_eq!(retry_after_seconds(Duration::from_nanos(1_000_000_001)), 2);
-
- let now = Instant::now();
- let realtime = RealtimeRoutingFacts::default();
- let mut session = session(
- now,
- vec![
- candidate(1, &["credential-a"]),
- candidate(2, &["credential-b"]),
- ],
- );
- let first = session
- .select_next(selection_request(now, 1, 3, &realtime))
- .unwrap()
- .unwrap();
- assert_eq!(
- session
- .decide_failure(
- &first,
- &materialization_failure(
- now,
- "runtime_state_cooling_down",
- Some(Duration::from_secs(12)),
- ),
- )
- .unwrap(),
- Disposition::Continue
- );
- let second = session
- .select_next(selection_request(now, 1, 3, &realtime))
- .unwrap()
- .unwrap();
- assert_eq!(
- session
- .decide_failure(
- &second,
- &materialization_failure(
- now,
- "runtime_state_cooling_down",
- Some(Duration::from_millis(3_200)),
- ),
- )
- .unwrap(),
- Disposition::Continue
- );
- assert!(
- session
- .select_next(selection_request(now, 1, 3, &realtime))
- .unwrap()
- .is_none()
- );
- let reply = session
- .exhaustion_local_reply(now)
- .unwrap()
- .expect("pure cooldown exhaustion has a retry response");
- assert_eq!(reply.headers.get(RETRY_AFTER).unwrap(), "4");
- assert_eq!(
- serde_json::from_slice::<serde_json::Value>(&reply.body).unwrap(),
- serde_json::json!({
- "schema_version": "hiroute.gateway.error/v1",
- "code": "CANDIDATES_COOLING_DOWN",
- "phase": "runtime_state",
- "retry_after_seconds": 4,
- })
- );
- }
-
- #[test]
- fn mixed_runtime_state_exhaustion_keeps_generic_failure() {
- let now = Instant::now();
- let realtime = RealtimeRoutingFacts::default();
- let mut session = session(
- now,
- vec![
- candidate(1, &["credential-a"]),
- candidate(2, &["credential-b"]),
- ],
- );
- let first = session
- .select_next(selection_request(now, 1, 3, &realtime))
- .unwrap()
- .unwrap();
- session
- .decide_failure(
- &first,
- &materialization_failure(
- now,
- "runtime_state_cooling_down",
- Some(Duration::from_secs(2)),
- ),
- )
- .unwrap();
- let second = session
- .select_next(selection_request(now, 1, 3, &realtime))
- .unwrap()
- .unwrap();
- assert_eq!(
- session
- .decide_failure(
- &second,
- &materialization_failure(now, "runtime_state_disabled", None),
- )
- .unwrap(),
- Disposition::Terminate
- );
- assert!(session.exhaustion_local_reply(now).unwrap().is_none());
- }
-
- #[test]
- fn hold_success_requires_full_accepted_eos_downstream_and_cleanup() {
- let now = Instant::now();
- let accepted = CompletedAttemptObservation {
- route_decision_id: RouteDecisionId(1),
- attempt_id: AttemptId(1),
- generation: AttemptGeneration(1),
- binding: ResolvedTargetBindingId::new(PlanRevision(1), 1),
- credential_ref: CredentialRef::new("credential").unwrap(),
- budget: AttemptBudgetGrant {
- issued_at: now,
- allocated: Duration::from_secs(1),
- deadline: now + Duration::from_secs(1),
- },
- routing_facts: RealtimeRoutingFacts::default(),
- provider: Some(ProviderClassificationFacts {
- model_event: Some(ObservationLabel::new("response_complete").unwrap()),
- ..ProviderClassificationFacts::default()
- }),
- failure: None,
- transport: AttemptTransportFacts {
- started_at: now,
- upstream_protocol: None,
- connect_elapsed: None,
- request_write_elapsed: None,
- upstream_ttfb: None,
- last_upstream_progress_at: None,
- local_read_suppressed: Duration::ZERO,
- upstream_body_bytes: 1,
- timeout: None,
- },
- disposition: Disposition::Accept,
- commits: AttemptCommitFacts {
- upstream_request: CommitFence::WriteConfirmed,
- downstream_headers: CommitFence::WriteConfirmed,
- downstream_semantic: CommitFence::WriteConfirmed,
- },
- stream: AttemptStreamOutcome::CompletedEos,
- downstream: AttemptDownstreamOutcome::Completed,
- cleanup: AttemptCleanupOutcome::Completed,
- ended_at: now,
- termination_reason: AttemptTerminationReason::AcceptedEos,
- };
- assert!(is_complete_accept(&accepted));
- let mut interrupted = accepted.clone();
- interrupted.stream = AttemptStreamOutcome::StreamStartedNoRetry;
- assert!(!is_complete_accept(&interrupted));
- let mut downstream_failed = accepted.clone();
- downstream_failed.downstream = AttemptDownstreamOutcome::Failed;
- assert!(!is_complete_accept(&downstream_failed));
- let mut cleanup_failed = accepted.clone();
- cleanup_failed.cleanup = AttemptCleanupOutcome::Failed;
- assert!(!is_complete_accept(&cleanup_failed));
- let mut incomplete = accepted.clone();
- incomplete.provider.as_mut().unwrap().model_event =
- Some(ObservationLabel::new("response_incomplete").unwrap());
- assert!(!is_complete_accept(&incomplete));
- let mut failed = accepted.clone();
- failed.provider.as_mut().unwrap().model_event =
- Some(ObservationLabel::new("response_failed").unwrap());
- assert!(!is_complete_accept(&failed));
- let mut missing_terminal = accepted.clone();
- missing_terminal.provider.as_mut().unwrap().model_event = None;
- assert!(!is_complete_accept(&missing_terminal));
- let mut terminated = accepted;
- terminated.disposition = Disposition::Terminate;
- assert!(!is_complete_accept(&terminated));
- }
-}
+mod tests;
diff --git a/crates/gateway/src/runtime/driver/selection/preparation.rs b/crates/gateway/src/runtime/driver/selection/preparation.rs
new file mode 100644
index 00000000..d97cd4de
--- /dev/null
+++ b/crates/gateway/src/runtime/driver/selection/preparation.rs
@@ -0,0 +1,49 @@
+use bytes::Bytes;
+use hiroute_gateway_core::core::filter::LocalReply;
+use hiroute_gateway_core::runtime::sse::SemanticProvenance;
+use http::header::CONTENT_TYPE;
+use http::{HeaderMap, HeaderValue, StatusCode};
+
+/// Request-local summary of the closed preparation facts. Different causes cannot
+/// be projected as if every candidate was rejected for one source or Key permission.
+#[derive(Clone, Copy, Eq, PartialEq)]
+pub(super) enum PreparationFailure {
+ SourceDisabled,
+ CredentialDisabled,
+ CredentialUnavailable,
+ Other,
+}
+
+impl PreparationFailure {
+ pub(super) fn from_error_class(error_class: Option<&str>) -> Self {
+ match error_class {
+ Some("source_disabled") => Self::SourceDisabled,
+ Some("credential_disabled") => Self::CredentialDisabled,
+ Some("credential_exhausted") => Self::CredentialUnavailable,
+ _ => Self::Other,
+ }
+ }
+
+ pub(super) fn merge(self, next: Self) -> Self {
+ if self == next { self } else { Self::Other }
+ }
+
+ pub(super) fn local_reply(self) -> LocalReply {
+ let (code, phase) = match self {
+ Self::SourceDisabled => ("SOURCE_DISABLED", "credential_lease"),
+ Self::CredentialDisabled => ("CREDENTIAL_DISABLED", "credential_lease"),
+ Self::CredentialUnavailable => ("CREDENTIAL_UNAVAILABLE", "credential_lease"),
+ Self::Other => ("CANDIDATE_PREPARATION_FAILED", "candidate_preparation"),
+ };
+ let mut headers = HeaderMap::new();
+ headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
+ LocalReply {
+ status: StatusCode::BAD_GATEWAY,
+ headers,
+ body: Bytes::from(format!(
+ "{{\"schema_version\":\"hiroute.gateway.error/v1\",\"code\":\"{code}\",\"phase\":\"{phase}\"}}"
+ )),
+ provenance: SemanticProvenance::NonSemantic,
+ }
+ }
+}
diff --git a/crates/gateway/src/runtime/driver/selection/tests.rs b/crates/gateway/src/runtime/driver/selection/tests.rs
new file mode 100644
index 00000000..ffae44f3
--- /dev/null
+++ b/crates/gateway/src/runtime/driver/selection/tests.rs
@@ -0,0 +1,774 @@
+use std::time::Duration;
+
+use hiroute_gateway_core::core::execution_plan::{
+ CredentialRef, PlanRevision, ResolvedTargetBindingId,
+};
+use hiroute_gateway_core::runtime::attempt::{
+ AttemptGeneration, AttemptTransportFacts, CommitFence, RequestId,
+};
+use hiroute_gateway_core::runtime::driver::{
+ AttemptCommitFacts, AttemptFailureClass, AttemptFailureFacts,
+};
+
+use super::*;
+
+fn candidate(binding_local_id: u32, credential_refs: &[&str]) -> DecisionCandidateAuthority {
+ DecisionCandidateAuthority {
+ binding: ResolvedTargetBindingId::new(PlanRevision(1), binding_local_id),
+ stable_target: ObservationLabel::new(format!("binding-{binding_local_id}")).unwrap(),
+ credential_refs: credential_refs
+ .iter()
+ .map(|value| CredentialRef::new(*value).unwrap())
+ .collect::<Vec<_>>()
+ .into(),
+ candidate_id: None,
+ profile_digest: None,
+ reason_ledger_identity: None,
+ provider_profile: None,
+ }
+}
+
+fn selection_request<'a>(
+ _now: Instant,
+ generation: u64,
+ remaining_attempts: u32,
+ realtime_facts: &'a RealtimeRoutingFacts,
+) -> SelectionRequest<'a> {
+ SelectionRequest {
+ request_id: RequestId(1),
+ generation: AttemptGeneration(generation),
+ route_binding: ResolvedTargetBindingId::new(PlanRevision(1), 1),
+ remaining_total: Duration::from_secs(1),
+ remaining_attempts,
+ realtime_facts,
+ previous_attempts: &[],
+ }
+}
+
+fn session(now: Instant, candidates: Vec<DecisionCandidateAuthority>) -> ProductionDecisionSession {
+ ProductionDecisionSession {
+ request_id: RequestId(1),
+ route_decision_id: RouteDecisionId(1),
+ candidates: candidates.into(),
+ candidate_index: 0,
+ credential_index: 0,
+ overall_deadline: now + Duration::from_secs(10),
+ hold: None,
+ switch_fallback: None,
+ reasoning_cleanup: Arc::new(super::super::super::ReasoningCleanup::new(None, None)),
+ temporary_block_seen: false,
+ probe_busy_seen: false,
+ earliest_retry_at: None,
+ non_temporary_block_seen: false,
+ upstream_attempted: false,
+ preparation_failure: None,
+ }
+}
+
+fn materialization_failure(
+ now: Instant,
+ error_class: &str,
+ retry_after: Option<Duration>,
+) -> AttemptFailureFacts {
+ AttemptFailureFacts {
+ class: AttemptFailureClass::Materialization,
+ provider: Some(ProviderClassificationFacts {
+ error_class: Some(ObservationLabel::new(error_class).unwrap()),
+ retryability: RetryabilityFact::Retryable,
+ retry_after,
+ readiness: ObservationLabel::new("materialization_failed").unwrap(),
+ ..ProviderClassificationFacts::default()
+ }),
+ transport: AttemptTransportFacts {
+ started_at: now,
+ upstream_protocol: None,
+ connect_elapsed: None,
+ request_write_elapsed: None,
+ upstream_ttfb: None,
+ last_upstream_progress_at: None,
+ local_read_suppressed: Duration::ZERO,
+ upstream_body_bytes: 0,
+ timeout: None,
+ },
+ request_committed: false,
+ termination_reason: ObservationLabel::new("materialization").unwrap(),
+ }
+}
+
+#[test]
+fn uniform_provisional_denials_retain_the_closed_code_without_retry_or_attempts() {
+ for (class, code) in [
+ ("source_disabled", "SOURCE_DISABLED"),
+ ("credential_disabled", "CREDENTIAL_DISABLED"),
+ ("credential_exhausted", "CREDENTIAL_UNAVAILABLE"),
+ (
+ "private_unrecognized_detail",
+ "CANDIDATE_PREPARATION_FAILED",
+ ),
+ ] {
+ let now = Instant::now();
+ let realtime = RealtimeRoutingFacts::default();
+ let mut session = session(
+ now,
+ vec![candidate(1, &["key-a"]), candidate(2, &["key-b"])],
+ );
+ for index in 0..2 {
+ let selected = session
+ .select_next(selection_request(now, 1, 3, &realtime))
+ .unwrap()
+ .unwrap();
+ assert_eq!(selected.attempt_id, AttemptId(0));
+ assert_eq!(
+ session
+ .decide_failure(&selected, &materialization_failure(now, class, None))
+ .unwrap(),
+ if index == 0 {
+ Disposition::Continue
+ } else {
+ Disposition::Terminate
+ }
+ );
+ }
+ let reply = session.exhaustion_local_reply(now).unwrap().unwrap();
+ assert_eq!(reply.status, StatusCode::BAD_GATEWAY);
+ assert_eq!(reply.headers[CONTENT_TYPE], "application/json");
+ assert!(!reply.headers.contains_key(RETRY_AFTER));
+ assert_eq!(
+ serde_json::from_slice::<serde_json::Value>(&reply.body).unwrap(),
+ serde_json::json!({
+ "schema_version": "hiroute.gateway.error/v1", "code": code,
+ "phase": if code == "CANDIDATE_PREPARATION_FAILED" { "candidate_preparation" } else { "credential_lease" }
+ })
+ );
+ }
+}
+
+#[test]
+fn provisional_reply_does_not_override_a_previous_formal_attempt() {
+ let now = Instant::now();
+ let realtime = RealtimeRoutingFacts::default();
+ let mut session = session(
+ now,
+ vec![candidate(1, &["key-a"]), candidate(2, &["key-b"])],
+ );
+ let mut first = session
+ .select_next(selection_request(now, 1, 3, &realtime))
+ .unwrap()
+ .unwrap();
+ first.attempt_id = AttemptId(1);
+ assert_eq!(
+ session
+ .decide_failure(
+ &first,
+ &materialization_failure(now, "materialization_retryable", None)
+ )
+ .unwrap(),
+ Disposition::Continue
+ );
+ let second = session
+ .select_next(selection_request(now, 2, 2, &realtime))
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ session
+ .decide_failure(
+ &second,
+ &materialization_failure(now, "source_disabled", None)
+ )
+ .unwrap(),
+ Disposition::Terminate
+ );
+ assert!(session.exhaustion_local_reply(now).unwrap().is_none());
+}
+
+#[test]
+fn retryable_quota_advances_exact_credential_before_binding() {
+ let now = Instant::now();
+ let realtime_facts = RealtimeRoutingFacts::default();
+ let transport = AttemptTransportFacts {
+ started_at: now,
+ upstream_protocol: None,
+ connect_elapsed: None,
+ request_write_elapsed: None,
+ upstream_ttfb: None,
+ last_upstream_progress_at: None,
+ local_read_suppressed: Duration::ZERO,
+ upstream_body_bytes: 0,
+ timeout: None,
+ };
+ let mut session = ProductionDecisionSession {
+ request_id: RequestId(1),
+ route_decision_id: RouteDecisionId(1),
+ candidates: vec![
+ candidate(1, &["credential-a", "credential-b"]),
+ candidate(2, &["credential-c"]),
+ ]
+ .into(),
+ candidate_index: 0,
+ credential_index: 0,
+ overall_deadline: now + Duration::from_secs(1),
+ hold: None,
+ switch_fallback: None,
+ reasoning_cleanup: Arc::new(super::super::super::ReasoningCleanup::new(None, None)),
+ temporary_block_seen: false,
+ probe_busy_seen: false,
+ earliest_retry_at: None,
+ non_temporary_block_seen: false,
+ upstream_attempted: false,
+ preparation_failure: None,
+ };
+ let first = session
+ .select_next(selection_request(now, 1, 3, &realtime_facts))
+ .unwrap()
+ .unwrap();
+ assert_eq!(first.binding.local_id(), 1);
+ assert_eq!(first.credential_ref.as_str(), "credential-a");
+
+ let quota = ProviderClassificationFacts {
+ error_class: Some(ObservationLabel::new("quota").unwrap()),
+ retryability: RetryabilityFact::Retryable,
+ readiness: ObservationLabel::new("rejected").unwrap(),
+ ..ProviderClassificationFacts::default()
+ };
+ assert_eq!(
+ session.decide(&first, &quota, &transport).unwrap(),
+ Disposition::Continue
+ );
+ let second = session
+ .select_next(selection_request(now, 2, 2, &realtime_facts))
+ .unwrap()
+ .unwrap();
+ assert_eq!(second.binding.local_id(), 1);
+ assert_eq!(second.credential_ref.as_str(), "credential-b");
+
+ assert_eq!(
+ session.decide(&second, &quota, &transport).unwrap(),
+ Disposition::Continue
+ );
+ let third = session
+ .select_next(selection_request(now, 3, 1, &realtime_facts))
+ .unwrap()
+ .unwrap();
+ assert_eq!(third.binding.local_id(), 2);
+ assert_eq!(third.credential_ref.as_str(), "credential-c");
+}
+
+#[test]
+fn switched_candidate_precommit_400_falls_back_to_previous_once() {
+ let now = Instant::now();
+ let realtime_facts = RealtimeRoutingFacts::default();
+ let mut session = session(
+ now,
+ vec![candidate(1, &["new-key"]), candidate(2, &["original-key"])],
+ );
+ session.switch_fallback = Some(ResolvedTargetBindingId::new(PlanRevision(1), 2));
+ let mut selected = session
+ .select_next(selection_request(now, 1, 2, &realtime_facts))
+ .unwrap()
+ .unwrap();
+ // The core promotes the candidate after pre-exchange materialization.
+ selected.attempt_id = AttemptId(1);
+ let rejected = ProviderClassificationFacts {
+ error_class: Some(ObservationLabel::new("permanent_client").unwrap()),
+ retryability: RetryabilityFact::NonRetryable,
+ http_status: Some(StatusCode::BAD_REQUEST),
+ ..ProviderClassificationFacts::default()
+ };
+ assert_eq!(
+ session
+ .decide(
+ &selected,
+ &rejected,
+ &AttemptTransportFacts {
+ started_at: now,
+ upstream_protocol: None,
+ connect_elapsed: None,
+ request_write_elapsed: None,
+ upstream_ttfb: None,
+ last_upstream_progress_at: None,
+ local_read_suppressed: Duration::ZERO,
+ upstream_body_bytes: 0,
+ timeout: None,
+ }
+ )
+ .unwrap(),
+ Disposition::Continue
+ );
+ let original = session
+ .select_next(selection_request(now, 2, 1, &realtime_facts))
+ .unwrap()
+ .unwrap();
+ assert_eq!(original.binding.local_id(), 2);
+ assert_eq!(
+ session
+ .decide(
+ &original,
+ &rejected,
+ &AttemptTransportFacts {
+ started_at: now,
+ upstream_protocol: None,
+ connect_elapsed: None,
+ request_write_elapsed: None,
+ upstream_ttfb: None,
+ last_upstream_progress_at: None,
+ local_read_suppressed: Duration::ZERO,
+ upstream_body_bytes: 0,
+ timeout: None,
+ }
+ )
+ .unwrap(),
+ Disposition::Terminate
+ );
+}
+
+#[test]
+fn explicit_history_rejection_retries_only_eligible_first_candidate_once() {
+ let now = Instant::now();
+ let facts = RealtimeRoutingFacts::default();
+ let mut session = session(now, vec![candidate(1, &["b"]), candidate(2, &["a"])]);
+ let binding = session.candidates[0].binding;
+ session.reasoning_cleanup = Arc::new(super::super::ReasoningCleanup::new(
+ Some((binding, 3)),
+ None,
+ ));
+ let rejected = ProviderClassificationFacts {
+ error_class: Some(ObservationLabel::new("reasoning_history").unwrap()),
+ retryability: RetryabilityFact::NonRetryable,
+ http_status: Some(StatusCode::BAD_REQUEST),
+ ..ProviderClassificationFacts::default()
+ };
+ let transport = AttemptTransportFacts {
+ started_at: now,
+ upstream_protocol: None,
+ connect_elapsed: None,
+ request_write_elapsed: None,
+ upstream_ttfb: None,
+ last_upstream_progress_at: None,
+ local_read_suppressed: Duration::ZERO,
+ upstream_body_bytes: 0,
+ timeout: None,
+ };
+ let first = session
+ .select_next(selection_request(now, 1, 3, &facts))
+ .unwrap()
+ .unwrap();
+ assert_eq!(session.reasoning_cleanup.prefix_for(binding), None);
+ assert_eq!(
+ session.decide(&first, &rejected, &transport).unwrap(),
+ Disposition::Continue
+ );
+ let retry = session
+ .select_next(selection_request(now, 2, 2, &facts))
+ .unwrap()
+ .unwrap();
+ assert_eq!(retry.binding, binding);
+ assert_eq!(session.reasoning_cleanup.prefix_for(binding), Some(3));
+ assert_eq!(
+ session.decide(&retry, &rejected, &transport).unwrap(),
+ Disposition::Continue
+ );
+ let fallback = session
+ .select_next(selection_request(now, 3, 1, &facts))
+ .unwrap()
+ .unwrap();
+ assert_ne!(fallback.binding, binding);
+ assert_eq!(session.reasoning_cleanup.prefix_for(fallback.binding), None);
+ assert_eq!(
+ session.decide(&fallback, &rejected, &transport).unwrap(),
+ Disposition::Terminate
+ );
+ assert!(
+ session
+ .select_next(selection_request(now, 4, 0, &facts))
+ .unwrap()
+ .is_none()
+ );
+}
+
+#[test]
+fn history_error_without_context_break_authorization_does_not_enable_cleanup() {
+ let now = Instant::now();
+ let facts = RealtimeRoutingFacts::default();
+ let mut session = session(now, vec![candidate(1, &["a"]), candidate(2, &["b"])]);
+ let selected = session
+ .select_next(selection_request(now, 1, 3, &facts))
+ .unwrap()
+ .unwrap();
+ let mut failure = materialization_failure(now, "reasoning_history", None);
+ failure.provider.as_mut().unwrap().retryability = RetryabilityFact::NonRetryable;
+ assert!(!session.reasoning_cleanup.consume(selected.binding));
+ assert_eq!(session.reasoning_cleanup.prefix_for(selected.binding), None);
+ assert_eq!(
+ session.decide_failure(&selected, &failure).unwrap(),
+ Disposition::Terminate
+ );
+}
+
+#[test]
+fn unknown_or_nonretryable_materialization_never_advances_selection() {
+ let now = Instant::now();
+ let selected = SelectedGatewayAttempt {
+ request_id: RequestId(1),
+ attempt_id: AttemptId(0),
+ generation: AttemptGeneration(1),
+ binding: ResolvedTargetBindingId::new(PlanRevision(1), 1),
+ credential_ref: CredentialRef::new("credential").unwrap(),
+ route_decision_id: RouteDecisionId(1),
+ budget: AttemptBudgetGrant {
+ issued_at: now,
+ allocated: Duration::from_secs(1),
+ deadline: now + Duration::from_secs(1),
+ },
+ };
+ let mut session = ProductionDecisionSession {
+ request_id: RequestId(1),
+ route_decision_id: RouteDecisionId(1),
+ candidates: Arc::new([]),
+ candidate_index: 0,
+ credential_index: 0,
+ overall_deadline: now + Duration::from_secs(1),
+ hold: None,
+ switch_fallback: None,
+ reasoning_cleanup: Arc::new(super::super::super::ReasoningCleanup::new(None, None)),
+ temporary_block_seen: false,
+ probe_busy_seen: false,
+ earliest_retry_at: None,
+ non_temporary_block_seen: false,
+ upstream_attempted: false,
+ preparation_failure: None,
+ };
+ for retryability in [RetryabilityFact::Unknown, RetryabilityFact::NonRetryable] {
+ let failure = AttemptFailureFacts {
+ class: AttemptFailureClass::Materialization,
+ provider: Some(ProviderClassificationFacts {
+ retryability,
+ readiness: ObservationLabel::new("materialization_failed").unwrap(),
+ ..ProviderClassificationFacts::default()
+ }),
+ transport: AttemptTransportFacts {
+ started_at: now,
+ upstream_protocol: None,
+ connect_elapsed: None,
+ request_write_elapsed: None,
+ upstream_ttfb: None,
+ last_upstream_progress_at: None,
+ local_read_suppressed: Duration::ZERO,
+ upstream_body_bytes: 0,
+ timeout: None,
+ },
+ request_committed: false,
+ termination_reason: ObservationLabel::new("materialization").unwrap(),
+ };
+ assert_eq!(
+ session.decide_failure(&selected, &failure).unwrap(),
+ Disposition::Terminate
+ );
+ assert_eq!(session.candidate_index, 0);
+ assert_eq!(session.credential_index, 0);
+ }
+}
+
+#[test]
+fn switched_provisional_nonretryable_failure_does_not_call_previous_model() {
+ let now = Instant::now();
+ let realtime_facts = RealtimeRoutingFacts::default();
+ let mut session = session(
+ now,
+ vec![candidate(1, &["new-key"]), candidate(2, &["original-key"])],
+ );
+ session.switch_fallback = Some(ResolvedTargetBindingId::new(PlanRevision(1), 2));
+ let selected = session
+ .select_next(selection_request(now, 1, 2, &realtime_facts))
+ .unwrap()
+ .unwrap();
+ assert_eq!(selected.attempt_id, AttemptId(0));
+
+ let mut failure = materialization_failure(now, "materialization_fail_closed", None);
+ failure.provider.as_mut().unwrap().retryability = RetryabilityFact::NonRetryable;
+ assert_eq!(
+ session.decide_failure(&selected, &failure).unwrap(),
+ Disposition::Terminate
+ );
+ assert_eq!(session.candidate_index, 0);
+}
+
+#[test]
+fn switched_promoted_preexchange_failure_does_not_call_previous_model() {
+ let now = Instant::now();
+ let realtime_facts = RealtimeRoutingFacts::default();
+ let mut session = session(
+ now,
+ vec![candidate(1, &["new-key"]), candidate(2, &["original-key"])],
+ );
+ session.switch_fallback = Some(ResolvedTargetBindingId::new(PlanRevision(1), 2));
+ let mut selected = session
+ .select_next(selection_request(now, 1, 2, &realtime_facts))
+ .unwrap()
+ .unwrap();
+ selected.attempt_id = AttemptId(1);
+
+ let mut failure = materialization_failure(now, "preexchange_filter_failure", None);
+ failure.provider.as_mut().unwrap().retryability = RetryabilityFact::NonRetryable;
+ assert!(!failure.request_committed);
+ assert_eq!(
+ session.decide_failure(&selected, &failure).unwrap(),
+ Disposition::Terminate
+ );
+ assert_eq!(session.candidate_index, 0);
+}
+
+#[test]
+fn pure_runtime_state_exhaustion_returns_retryable_503_and_probe_busy_wins() {
+ let now = Instant::now();
+ let realtime = RealtimeRoutingFacts::default();
+ let mut session = session(
+ now,
+ vec![
+ candidate(1, &["credential-a"]),
+ candidate(2, &["credential-b"]),
+ ],
+ );
+ let first = session
+ .select_next(selection_request(now, 1, 3, &realtime))
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ session
+ .decide_failure(
+ &first,
+ &materialization_failure(
+ now,
+ "runtime_state_cooling_down",
+ Some(Duration::from_millis(1_500)),
+ ),
+ )
+ .unwrap(),
+ Disposition::Continue
+ );
+ let second = session
+ .select_next(selection_request(now, 1, 3, &realtime))
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ session
+ .decide_failure(
+ &second,
+ &materialization_failure(
+ now,
+ "runtime_state_probe_busy",
+ Some(Duration::from_secs(1)),
+ ),
+ )
+ .unwrap(),
+ Disposition::Continue
+ );
+ assert!(
+ session
+ .select_next(selection_request(now, 1, 3, &realtime))
+ .unwrap()
+ .is_none()
+ );
+ let reply = session
+ .exhaustion_local_reply(now)
+ .unwrap()
+ .expect("pure temporary exhaustion has a retry response");
+ assert_eq!(reply.status, StatusCode::SERVICE_UNAVAILABLE);
+ assert_eq!(reply.headers.get(RETRY_AFTER).unwrap(), "1");
+ assert_eq!(reply.headers.get(CONTENT_TYPE).unwrap(), "application/json");
+ assert_eq!(
+ serde_json::from_slice::<serde_json::Value>(&reply.body).unwrap(),
+ serde_json::json!({
+ "schema_version": "hiroute.gateway.error/v1",
+ "code": "RECOVERY_PROBE_IN_PROGRESS",
+ "phase": "runtime_state",
+ "retry_after_seconds": 1,
+ })
+ );
+}
+
+#[test]
+fn cooling_exhaustion_uses_earliest_deadline_and_rounds_up() {
+ assert_eq!(retry_after_seconds(Duration::from_nanos(1)), 1);
+ assert_eq!(retry_after_seconds(Duration::from_nanos(1_000_000_001)), 2);
+
+ let now = Instant::now();
+ let realtime = RealtimeRoutingFacts::default();
+ let mut session = session(
+ now,
+ vec![
+ candidate(1, &["credential-a"]),
+ candidate(2, &["credential-b"]),
+ ],
+ );
+ let first = session
+ .select_next(selection_request(now, 1, 3, &realtime))
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ session
+ .decide_failure(
+ &first,
+ &materialization_failure(
+ now,
+ "runtime_state_cooling_down",
+ Some(Duration::from_secs(12)),
+ ),
+ )
+ .unwrap(),
+ Disposition::Continue
+ );
+ let second = session
+ .select_next(selection_request(now, 1, 3, &realtime))
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ session
+ .decide_failure(
+ &second,
+ &materialization_failure(
+ now,
+ "runtime_state_cooling_down",
+ Some(Duration::from_millis(3_200)),
+ ),
+ )
+ .unwrap(),
+ Disposition::Continue
+ );
+ assert!(
+ session
+ .select_next(selection_request(now, 1, 3, &realtime))
+ .unwrap()
+ .is_none()
+ );
+ let reply = session
+ .exhaustion_local_reply(now)
+ .unwrap()
+ .expect("pure cooldown exhaustion has a retry response");
+ assert_eq!(reply.headers.get(RETRY_AFTER).unwrap(), "4");
+ assert_eq!(
+ serde_json::from_slice::<serde_json::Value>(&reply.body).unwrap(),
+ serde_json::json!({
+ "schema_version": "hiroute.gateway.error/v1",
+ "code": "CANDIDATES_COOLING_DOWN",
+ "phase": "runtime_state",
+ "retry_after_seconds": 4,
+ })
+ );
+}
+
+#[test]
+fn mixed_runtime_state_exhaustion_returns_a_generic_typed_preparation_failure() {
+ let now = Instant::now();
+ let realtime = RealtimeRoutingFacts::default();
+ let mut session = session(
+ now,
+ vec![
+ candidate(1, &["credential-a"]),
+ candidate(2, &["credential-b"]),
+ ],
+ );
+ let first = session
+ .select_next(selection_request(now, 1, 3, &realtime))
+ .unwrap()
+ .unwrap();
+ session
+ .decide_failure(
+ &first,
+ &materialization_failure(
+ now,
+ "runtime_state_cooling_down",
+ Some(Duration::from_secs(2)),
+ ),
+ )
+ .unwrap();
+ let second = session
+ .select_next(selection_request(now, 1, 3, &realtime))
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ session
+ .decide_failure(
+ &second,
+ &materialization_failure(now, "runtime_state_disabled", None),
+ )
+ .unwrap(),
+ Disposition::Terminate
+ );
+ let reply = session.exhaustion_local_reply(now).unwrap().unwrap();
+ assert_eq!(reply.status, StatusCode::BAD_GATEWAY);
+ assert!(!reply.headers.contains_key(RETRY_AFTER));
+ assert_eq!(
+ serde_json::from_slice::<serde_json::Value>(&reply.body).unwrap(),
+ serde_json::json!({"schema_version": "hiroute.gateway.error/v1",
+ "code": "CANDIDATE_PREPARATION_FAILED", "phase": "candidate_preparation"})
+ );
+}
+
+#[test]
+fn hold_success_requires_full_accepted_eos_downstream_and_cleanup() {
+ let now = Instant::now();
+ let accepted = CompletedAttemptObservation {
+ route_decision_id: RouteDecisionId(1),
+ attempt_id: AttemptId(1),
+ generation: AttemptGeneration(1),
+ binding: ResolvedTargetBindingId::new(PlanRevision(1), 1),
+ credential_ref: CredentialRef::new("credential").unwrap(),
+ budget: AttemptBudgetGrant {
+ issued_at: now,
+ allocated: Duration::from_secs(1),
+ deadline: now + Duration::from_secs(1),
+ },
+ routing_facts: RealtimeRoutingFacts::default(),
+ provider: Some(ProviderClassificationFacts {
+ model_event: Some(ObservationLabel::new("response_complete").unwrap()),
+ ..ProviderClassificationFacts::default()
+ }),
+ failure: None,
+ transport: AttemptTransportFacts {
+ started_at: now,
+ upstream_protocol: None,
+ connect_elapsed: None,
+ request_write_elapsed: None,
+ upstream_ttfb: None,
+ last_upstream_progress_at: None,
+ local_read_suppressed: Duration::ZERO,
+ upstream_body_bytes: 1,
+ timeout: None,
+ },
+ disposition: Disposition::Accept,
+ commits: AttemptCommitFacts {
+ upstream_request: CommitFence::WriteConfirmed,
+ downstream_headers: CommitFence::WriteConfirmed,
+ downstream_semantic: CommitFence::WriteConfirmed,
+ },
+ stream: AttemptStreamOutcome::CompletedEos,
+ downstream: AttemptDownstreamOutcome::Completed,
+ cleanup: AttemptCleanupOutcome::Completed,
+ ended_at: now,
+ termination_reason: AttemptTerminationReason::AcceptedEos,
+ };
+ assert!(is_complete_accept(&accepted));
+ let mut interrupted = accepted.clone();
+ interrupted.stream = AttemptStreamOutcome::StreamStartedNoRetry;
+ assert!(!is_complete_accept(&interrupted));
+ let mut downstream_failed = accepted.clone();
+ downstream_failed.downstream = AttemptDownstreamOutcome::Failed;
+ assert!(!is_complete_accept(&downstream_failed));
+ let mut cleanup_failed = accepted.clone();
+ cleanup_failed.cleanup = AttemptCleanupOutcome::Failed;
+ assert!(!is_complete_accept(&cleanup_failed));
+ let mut incomplete = accepted.clone();
+ incomplete.provider.as_mut().unwrap().model_event =
+ Some(ObservationLabel::new("response_incomplete").unwrap());
+ assert!(!is_complete_accept(&incomplete));
+ let mut failed = accepted.clone();
+ failed.provider.as_mut().unwrap().model_event =
+ Some(ObservationLabel::new("response_failed").unwrap());
+ assert!(!is_complete_accept(&failed));
+ let mut missing_terminal = accepted.clone();
+ missing_terminal.provider.as_mut().unwrap().model_event = None;
+ assert!(!is_complete_accept(&missing_terminal));
+ let mut terminated = accepted;
+ terminated.disposition = Disposition::Terminate;
+ assert!(!is_complete_accept(&terminated));
+}
diff --git a/crates/local-storage/src/control/compute/management/tests/runtime_leases.rs b/crates/local-storage/src/control/compute/management/tests/runtime_leases.rs
index 66ba7a0a..6d3099be 100644
--- a/crates/local-storage/src/control/compute/management/tests/runtime_leases.rs
+++ b/crates/local-storage/src/control/compute/management/tests/runtime_leases.rs
@@ -2,7 +2,7 @@ use super::*;
use hiroute_domain::{
ComputeManagementRepositoryPort, ComputeManagementSourceV2, ConnectorRuntimeKind,
GatewayOperationalTargetV1, NATIVE_CREDENTIAL_LEASE_REQUEST_SCHEMA_V1,
- NativeCredentialLeaseRequestV1,
+ NativeCredentialAccessDeniedV1, NativeCredentialLeaseErrorV1, NativeCredentialLeaseRequestV1,
};
fn save(
@@ -147,10 +147,10 @@ fn frozen_native_requests_obey_current_source_and_key_switches_after_restart() {
);
for request in &requests[..2] {
assert!(
- stores
- .lease_native_credential_exact(request)
- .unwrap()
- .is_none(),
+ stores.lease_native_credential_exact(request).err()
+ == Some(NativeCredentialLeaseErrorV1::Denied(
+ NativeCredentialAccessDeniedV1::CredentialDisabled
+ )),
"the same disabled key must be unavailable to both frozen model bindings"
);
assert!(
@@ -178,20 +178,20 @@ fn frozen_native_requests_obey_current_source_and_key_switches_after_restart() {
);
for request in &requests {
assert!(
- stores
- .lease_native_credential_exact(request)
- .unwrap()
- .is_none()
+ stores.lease_native_credential_exact(request).err()
+ == Some(NativeCredentialLeaseErrorV1::Denied(
+ NativeCredentialAccessDeniedV1::SourceDisabled
+ ))
);
}
drop(stores);
let stores = LocalStorageSet::open_for_daemon_startup(root.path()).unwrap();
for request in &requests {
assert!(
- stores
- .lease_native_credential_exact(request)
- .unwrap()
- .is_none()
+ stores.lease_native_credential_exact(request).err()
+ == Some(NativeCredentialLeaseErrorV1::Denied(
+ NativeCredentialAccessDeniedV1::SourceDisabled
+ ))
);
}
save(
@@ -203,10 +203,10 @@ fn frozen_native_requests_obey_current_source_and_key_switches_after_restart() {
"lease-reenable-source",
);
assert!(
- stores
- .lease_native_credential_exact(&requests[0])
- .unwrap()
- .is_none()
+ stores.lease_native_credential_exact(&requests[0]).err()
+ == Some(NativeCredentialLeaseErrorV1::Denied(
+ NativeCredentialAccessDeniedV1::CredentialDisabled
+ ))
);
assert!(
stores
@@ -224,10 +224,10 @@ fn frozen_native_requests_obey_current_source_and_key_switches_after_restart() {
);
for request in &requests {
assert!(
- stores
- .lease_native_credential_exact(request)
- .unwrap()
- .is_none()
+ stores.lease_native_credential_exact(request).err()
+ == Some(NativeCredentialLeaseErrorV1::Denied(
+ NativeCredentialAccessDeniedV1::SourceDisabled
+ ))
);
}
save(
diff --git a/crates/local-storage/src/secrets/runtime.rs b/crates/local-storage/src/secrets/runtime.rs
index fdc6727d..b9455703 100644
--- a/crates/local-storage/src/secrets/runtime.rs
+++ b/crates/local-storage/src/secrets/runtime.rs
@@ -3,8 +3,9 @@ use std::sync::Mutex;
use hiroute_domain::{
ComputeManagementRepositoryPort, CredentialRefV1, GatewayAuthenticationSemanticsV1,
- HeaderSecretLeaseRequestV1, MaterializationState, NativeCredentialAuthorityV1,
- NativeCredentialAuthorizationCapabilityV1, NativeCredentialCapabilityErrorV1,
+ HeaderSecretLeaseRequestV1, MaterializationState, NativeCredentialAccessDeniedV1,
+ NativeCredentialAuthorityV1, NativeCredentialAuthorizationCapabilityV1,
+ NativeCredentialCapabilityErrorV1, NativeCredentialLeaseErrorV1,
NativeCredentialLeaseRequestV1, NativeCredentialLeaseV1, PortErrorCode, PortResult,
ProtectedSecret, SensitiveAuthorizationTargetV1,
};
@@ -19,7 +20,7 @@ impl crate::LocalStorageSet {
pub fn lease_native_credential_exact(
&self,
request: &NativeCredentialLeaseRequestV1,
- ) -> PortResult<Option<NativeCredentialLeaseV1>> {
+ ) -> Result<Option<NativeCredentialLeaseV1>, NativeCredentialLeaseErrorV1> {
self.secrets()
.lease_native_credential_with_guard(request, |reference| {
// Only the current management producer owns this identity namespace.
@@ -30,13 +31,30 @@ impl crate::LocalStorageSet {
let Some(source_id) = reference.owner_scope().strip_prefix("source/") else {
return Ok(false);
};
- let source = self.control().compute_management_source(source_id)?;
- Ok(source.is_some_and(|source| {
- source.state == MaterializationState::Ready
- && source.credentials.iter().any(|credential| {
- credential.enabled && credential.credential == *reference
- })
- }))
+ let Some(source) = self.control().compute_management_source(source_id)? else {
+ return Ok(false);
+ };
+ if source.state == MaterializationState::Disabled {
+ return Err(NativeCredentialLeaseErrorV1::Denied(
+ NativeCredentialAccessDeniedV1::SourceDisabled,
+ ));
+ }
+ if source.state != MaterializationState::Ready {
+ return Ok(false);
+ }
+ let Some(credential) = source
+ .credentials
+ .iter()
+ .find(|credential| credential.credential == *reference)
+ else {
+ return Ok(false);
+ };
+ if !credential.enabled {
+ return Err(NativeCredentialLeaseErrorV1::Denied(
+ NativeCredentialAccessDeniedV1::CredentialDisabled,
+ ));
+ }
+ Ok(true)
})
}
}
@@ -64,15 +82,15 @@ impl LocalSecretStore {
pub fn lease_native_credential_exact(
&self,
request: &NativeCredentialLeaseRequestV1,
- ) -> PortResult<Option<NativeCredentialLeaseV1>> {
+ ) -> Result<Option<NativeCredentialLeaseV1>, NativeCredentialLeaseErrorV1> {
self.lease_native_credential_with_guard(request, |_| Ok(true))
}
fn lease_native_credential_with_guard(
&self,
request: &NativeCredentialLeaseRequestV1,
- guard: impl FnOnce(&CredentialRefV1) -> PortResult<bool>,
- ) -> PortResult<Option<NativeCredentialLeaseV1>> {
+ guard: impl FnOnce(&CredentialRefV1) -> Result<bool, NativeCredentialLeaseErrorV1>,
+ ) -> Result<Option<NativeCredentialLeaseV1>, NativeCredentialLeaseErrorV1> {
request.validate().map_err(|_| {
port(
PortErrorCode::InvalidData,
@@ -106,7 +124,8 @@ impl LocalSecretStore {
return Err(port(
PortErrorCode::PermissionDenied,
"secret.runtime.destination_binding",
- ));
+ )
+ .into());
}
if request
.excluded_key_ids
@@ -129,7 +148,7 @@ impl LocalSecretStore {
});
NativeCredentialLeaseV1::issue(credential_id.clone(), credential_id, generation, capability)
.map(Some)
- .map_err(|_| port(PortErrorCode::Corrupt, "secret.runtime.lease"))
+ .map_err(|_| port(PortErrorCode::Corrupt, "secret.runtime.lease").into())
}
pub fn lease_header_secret_exact(
@@ -201,7 +220,7 @@ impl NativeCredentialAuthorityV1 for LocalNativeCredentialAuthority {
fn lease_native_credential(
&self,
request: &NativeCredentialLeaseRequestV1,
- ) -> PortResult<Option<NativeCredentialLeaseV1>> {
+ ) -> Result<Option<NativeCredentialLeaseV1>, NativeCredentialLeaseErrorV1> {
self.store
.lock()
.map_err(|_| port(PortErrorCode::Unavailable, "secret.runtime.authority_lock"))?
diff --git a/crates/local-storage/src/secrets/runtime_tests.rs b/crates/local-storage/src/secrets/runtime_tests.rs
index 9e3b2aed..ec0915a9 100644
--- a/crates/local-storage/src/secrets/runtime_tests.rs
+++ b/crates/local-storage/src/secrets/runtime_tests.rs
@@ -4,7 +4,7 @@ use hiroute_domain::{
GatewayOperationalTargetV1, HEADER_SECRET_LEASE_REQUEST_SCHEMA_V1, HeaderSecretLeaseRequestV1,
NATIVE_CREDENTIAL_LEASE_REQUEST_SCHEMA_V1, NativeCredentialAuthorityV1,
NativeCredentialCapabilityErrorV1, NativeCredentialLeaseRequestV1, NativeCredentialLeaseV1,
- OperationId, PortErrorCode, PortResult, ProtectedSecret, SecretMutationV1, SecretStorePort,
+ OperationId, PortErrorCode, ProtectedSecret, SecretMutationV1, SecretStorePort,
SensitiveAuthorizationTargetV1, UpstreamProtocol,
};
@@ -267,9 +267,16 @@ fn frozen_native_request_rejects_duplicate_exclusions_and_non_bearer_auth() {
assert!(auth.validate().is_err());
}
-fn lease_error_code(result: PortResult<Option<NativeCredentialLeaseV1>>) -> PortErrorCode {
+fn lease_error_code<E: Into<hiroute_domain::NativeCredentialLeaseErrorV1>>(
+ result: Result<Option<NativeCredentialLeaseV1>, E>,
+) -> PortErrorCode {
match result {
- Err(error) => error.code,
+ Err(error) => match error.into() {
+ hiroute_domain::NativeCredentialLeaseErrorV1::Authority(error) => error.code,
+ hiroute_domain::NativeCredentialLeaseErrorV1::Denied(_) => {
+ PortErrorCode::PermissionDenied
+ }
+ },
Ok(_) => panic!("expected credential authority rejection"),
}
}
diff --git a/docs/code-map/architecture.md b/docs/code-map/architecture.md
index 598511d1..8f04352e 100644
--- a/docs/code-map/architecture.md
+++ b/docs/code-map/architecture.md
@@ -92,6 +92,7 @@ Scan → Prepare → Save → restart → native use, including stale-source rej
| Capture completeness | [Content completeness](../../crates/observation/src/content/completeness.rs), [session query](../../crates/observation/src/query_v2/sessions.rs) | Aggregate every required request/direction; a finished stream or readable preview does not prove the session is complete |
| Streaming memory | [Capture](../../crates/gateway/src/observation/provider/capture.rs), [pending delivery](../../crates/gateway/src/observation/provider/capture/tracker.rs) | Decoder, projector and delivery tracker share the stream budget; queue capacity is not capture evidence |
| Upstream fault attribution | [Wire projection](../../crates/gateway/src/observation/provider/wire_diagnostic.rs), [attempt association](../../crates/gateway/src/observation/request/diagnostic.rs), [decision calls](../../crates/gateway/src/core_runtime/classification/diagnostic.rs) | Actual bounded model/control fields, negotiated HTTP protocol, closed error codes, hashed request IDs and separate client headers/body commit facts must preserve the real outcome without prompts or provider error text |
+| Source disabled before an upstream attempt | [Exact permission gate](../../crates/local-storage/src/secrets/runtime.rs), [product preparation reply](../../crates/gateway/src/runtime/driver/selection/preparation.rs), [Gateway error guide](../../crates/gateway/README.md#candidate-preparation-errors) | Preserve the closed lease denial in the request fact and local JSON response; a frozen route does not grant current source/Key permission, and rejected preparation remains zero attempts |
| Inspect CPA authentication recovery | [Private response metadata](../../crates/gateway/src/observation/provider/cpa_execution.rs), [pinned CPA adapter](../../vendor/cpa/README.md) | Logical attempts and actual HTTP sends differ; only verified CPA targets can supply recovery counters. Missing evidence remains unknown and does not create token usage. |
| Search and gaps | [Query](../../crates/observation/src/query_v2/search.rs), [index worker](../../crates/observation/src/text_index/worker.rs), [status](../../crates/observation/src/query/status.rs) | Bounded scan windows, wraparound and explicit gaps distinguish incomplete search from an empty result |
| Business relations and visibility | [Relation owner](../../crates/observation/src/query_v2/relation.rs), [managed content](../../crates/observation/src/managed_text/README.md) | `query_v2` also writes relations; every content read checks current authorization and visibility |
diff --git a/tools/e2e-harness/tests/p0_gateway_observation.rs b/tools/e2e-harness/tests/p0_gateway_observation.rs
index f12c1806..1f761c3c 100644
--- a/tools/e2e-harness/tests/p0_gateway_observation.rs
+++ b/tools/e2e-harness/tests/p0_gateway_observation.rs
@@ -17,6 +17,35 @@ const ACCEPTED_STREAM_CREATED: &[u8] = b"event: response.created\ndata: {\"type\
const ACCEPTED_STREAM_TEXT: &[u8] = b"event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"item_id\":\"observation-message\",\"output_index\":0,\"content_index\":0,\"delta\":\"accepted-stream-22008\"}\n\n";
const ACCEPTED_STREAM_COMPLETED: &[u8] = b"event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"observation-stream\",\"model\":\"native-observed\",\"status\":\"completed\",\"output\":[{\"type\":\"message\",\"id\":\"observation-message\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"accepted-stream-22008\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":13,\"output_tokens\":5,\"total_tokens\":18,\"input_tokens_details\":{\"cached_tokens\":4},\"output_tokens_details\":{\"reasoning_tokens\":3}}}}\n\n";
+#[test]
+fn observation_reader_waits_for_complete_rows_and_rejects_closed_corruption() {
+ use std::io::Write;
+
+ let root = tempfile::tempdir().unwrap();
+ let path = root.path().join("records.jsonl");
+ let first = serde_json::json!({"phase": "begin"});
+ let second = serde_json::json!({"phase": "finish", "text": "尾帧"});
+ let mut prefix = serde_json::to_vec(&first).unwrap();
+ prefix.push(b'\n');
+ let second_bytes = serde_json::to_vec(&second).unwrap();
+ let partial_utf8 = second_bytes.iter().position(|byte| *byte >= 0x80).unwrap() + 1;
+ for end in [9, partial_utf8, second_bytes.len()] {
+ let mut snapshot = prefix.clone();
+ snapshot.extend_from_slice(&second_bytes[..end]);
+ fs::write(&path, snapshot).unwrap();
+ assert_eq!(read_records(&path).as_slice(), std::slice::from_ref(&first));
+ let mut writer = fs::OpenOptions::new().append(true).open(&path).unwrap();
+ writer.write_all(&second_bytes[end..]).unwrap();
+ writer.write_all(b"\n").unwrap();
+ writer.flush().unwrap();
+ assert_eq!(read_records(&path), [first.clone(), second.clone()]);
+ }
+ for corruption in [b"invalid-json\n".as_slice(), &[0xff, b'\n']] {
+ fs::write(&path, corruption).unwrap();
+ assert!(std::panic::catch_unwind(|| read_records(&path)).is_err());
+ }
+}
+
#[test]
fn production_info_cross_group_relay_links_only_actual_attempts() {
for relay_allowed in [true, false] {
@@ -314,11 +343,9 @@ fn production_unknown_completion_retains_transport_usage_and_an_unknown_agent_tu
fn wait_diagnostic_terminal(path: &Path, outcome: &str) -> (String, Vec<Value>) {
let deadline = Instant::now() + Duration::from_secs(5);
loop {
- let log = fs::read_to_string(path).unwrap_or_default();
- let records = log
- .lines()
- .map(|line| serde_json::from_str::<Value>(line).unwrap())
- .collect::<Vec<_>>();
+ let bytes = fs::read(path).unwrap_or_default();
+ let log = String::from_utf8_lossy(&bytes).into_owned();
+ let records = complete_records(&bytes);
if records.iter().any(|record| {
record
.pointer("/event/request_end/outcome")
@@ -1017,10 +1044,18 @@ fn real_hirouted_emits_zero_attempt_when_runtime_state_excludes_the_sealed_candi
401,
"seed disabled credential state"
);
+ let response = fixture.request();
+ assert_eq!(response.status, 502, "all runtime candidates excluded");
assert_eq!(
- fixture.request().status,
- 502,
- "all runtime candidates excluded"
+ response.headers.get("content-type").map(String::as_str),
+ Some("application/json")
+ );
+ assert!(!response.headers.contains_key("retry-after"));
+ assert_eq!(
+ serde_json::from_slice::<Value>(&response.body).unwrap(),
+ serde_json::json!({
+ "schema_version": "hiroute.gateway.error/v1", "code": "CANDIDATE_PREPARATION_FAILED", "phase": "candidate_preparation"
+ })
);
assert_eq!(provider.calls(), 1);
@@ -1503,12 +1538,16 @@ fn content_complete_for_requests(records: &[Value], request_ids: &[String]) -> b
}
fn read_records(path: &Path) -> Vec<Value> {
- fs::read_to_string(path)
- .ok()
- .into_iter()
- .flat_map(|contents| contents.lines().map(str::to_owned).collect::<Vec<_>>())
- .filter(|line| !line.trim().is_empty())
- .map(|line| serde_json::from_str(&line).unwrap())
+ complete_records(&fs::read(path).unwrap_or_default())
+}
+
+fn complete_records(contents: &[u8]) -> Vec<Value> {
+ // A live JSONL writer may be between its payload and newline writes.
+ contents
+ .split_inclusive(|byte| *byte == b'\n')
+ .filter(|line| line.ends_with(b"\n"))
+ .filter(|line| !line.iter().all(u8::is_ascii_whitespace))
+ .map(|line| serde_json::from_slice(line).unwrap())
.collect()
}