Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.
Rust ai-agentsai-gatewayclaude-codecodexcoding-agentcost-optimizationdeepseekllm-gatewayllm-routerlocal-firstmodel-routingrusttask-routingtauri
fix(gateway): return structured candidate preparation errors (#110)
澄潭ac012a3cec94
diff --git a/crates/daemon/src/control/runtime.rs b/crates/daemon/src/control/runtime.rsindex 444e51e6..f198421f 100644--- a/crates/daemon/src/control/runtime.rs+++ b/crates/daemon/src/control/runtime.rs@@ -829,7 +829,7 @@ impl NativeCredentialAuthorityV1 for ControlNativeCredentialAuthority {fn lease_native_credential(&self,request: &NativeCredentialLeaseRequestV1,- ) -> PortResult<Option<NativeCredentialLeaseV1>> {+ ) -> Result<Option<NativeCredentialLeaseV1>, hiroute_domain::NativeCredentialLeaseErrorV1> {self.adapter.stores.lock()diff --git a/crates/daemon/src/gateway_ports/credential.rs b/crates/daemon/src/gateway_ports/credential.rsindex 4b1ee040..23fb1f80 100644--- a/crates/daemon/src/gateway_ports/credential.rs+++ b/crates/daemon/src/gateway_ports/credential.rs@@ -99,11 +99,18 @@ wherelet Some(lease) = self.native.lease_native_credential(&native_request)- .map_err(|error| match error.code {- hiroute_domain::PortErrorCode::Unavailable => {- PortError::Unavailable("NativeCredentialAuthorityV1")+ .map_err(|error| match error {+ hiroute_domain::NativeCredentialLeaseErrorV1::Denied(reason) => {+ PortError::CredentialDenied(reason)+ }+ hiroute_domain::NativeCredentialLeaseErrorV1::Authority(error) => {+ match error.code {+ hiroute_domain::PortErrorCode::Unavailable => {+ PortError::Unavailable("NativeCredentialAuthorityV1")+ }+ _ => PortError::Rejected,+ }}- _ => PortError::Rejected,})?else {return Ok(None);diff --git a/crates/daemon/src/gateway_ports/tests.rs b/crates/daemon/src/gateway_ports/tests.rsindex 8d38e96e..cff0f0f9 100644--- a/crates/daemon/src/gateway_ports/tests.rs+++ b/crates/daemon/src/gateway_ports/tests.rs@@ -1,4 +1,5 @@mod cpa_async;+mod credential_denial;use std::sync::{Arc, Mutex};use std::time::{Duration, Instant};@@ -410,17 +411,21 @@ async fn gateway_ports_runtime_state_preserves_durable_cas_and_typed_probe_outcostruct FakeNativeAuthority {requests: Mutex<Vec<NativeCredentialLeaseRequestV1>>,header_secret_requests: Mutex<Vec<HeaderSecretLeaseRequestV1>>,+ denial: Option<hiroute_domain::NativeCredentialAccessDeniedV1>,}impl NativeCredentialAuthorityV1 for FakeNativeAuthority {fn lease_native_credential(&self,request: &NativeCredentialLeaseRequestV1,- ) -> PortResult<Option<NativeCredentialLeaseV1>> {+ ) -> Result<Option<NativeCredentialLeaseV1>, hiroute_domain::NativeCredentialLeaseErrorV1> {request.validate().map_err(|_| {ProductPortError::new(PortErrorCode::InvalidData, "test.native.request")})?;self.requests.lock().unwrap().push(request.clone());+ if let Some(reason) = self.denial {+ return Err(hiroute_domain::NativeCredentialLeaseErrorV1::Denied(reason));+ }NativeCredentialLeaseV1::issue(request.credential_id.clone(),"native-key",@@ -430,7 +435,7 @@ impl NativeCredentialAuthorityV1 for FakeNativeAuthority {}),).map(Some)- .map_err(|_| ProductPortError::new(PortErrorCode::Corrupt, "test.native.lease"))+ .map_err(|_| ProductPortError::new(PortErrorCode::Corrupt, "test.native.lease").into())}fn lease_header_secret(diff --git a/crates/daemon/src/gateway_ports/tests/credential_denial.rs b/crates/daemon/src/gateway_ports/tests/credential_denial.rsnew file mode 100644index 00000000..02a24404--- /dev/null+++ b/crates/daemon/src/gateway_ports/tests/credential_denial.rs@@ -0,0 +1,51 @@+use super::*;+use hiroute_domain::NativeCredentialAccessDeniedV1;+use hiroute_gateway::server::composition::PortError;++#[tokio::test]+async fn exact_native_lease_keeps_source_and_key_denials_distinct_from_authority_errors() {+ for reason in [+ NativeCredentialAccessDeniedV1::SourceDisabled,+ NativeCredentialAccessDeniedV1::CredentialDisabled,+ ] {+ let native = Arc::new(FakeNativeAuthority {+ denial: Some(reason),+ ..FakeNativeAuthority::default()+ });+ let resolver = GatewayCredentialResolver::new(+ Arc::clone(&native),+ Arc::new(FakeCpaAuthority::default()),+ );+ let target = GatewayOperationalTargetV1::RegisteredHttps {+ uri: "https://api.anthropic.com/v1/messages".into(),+ };+ let digest = CanonicalDigest::of(&target).unwrap();+ let profile = CanonicalDigest::of_bytes(b"native-profile");+ let result = resolver+ .lease_exact(+ CredentialLeaseRequest {+ stable_binding_id: "binding/claude",+ credential_ref: "credential/claude",+ credential_destination_ref: "connection-option/claude.custom.v1",+ excluded_key_ids: &[],+ connector_runtime: ConnectorRuntimeKind::BuiltinNative,+ connector_id: "builtin-anthropic",+ upstream_protocol: IngressProtocol::Messages,+ upstream_model_id: "claude-sonnet",+ native_transport_model: "claude-sonnet",+ logical_endpoint: target.uri(),+ operational_target: target.uri(),+ operational_target_digest: digest.as_str(),+ runtime_epoch: None,+ target_epoch: None,+ protocol_profile_digest: profile.as_str(),+ request_path: "/v1/messages",+ authentication: &AuthenticationSemantics::Bearer,+ },+ &scope(),+ )+ .await;+ assert_eq!(result.err(), Some(PortError::CredentialDenied(reason)));+ assert_eq!(native.requests.lock().unwrap().len(), 1);+ }+}diff --git a/crates/daemon/src/gateway_ports/tests/observation_collection.rs b/crates/daemon/src/gateway_ports/tests/observation_collection.rsindex 4e34127a..f5517925 100644--- a/crates/daemon/src/gateway_ports/tests/observation_collection.rs+++ b/crates/daemon/src/gateway_ports/tests/observation_collection.rs@@ -1,5 +1,8 @@use std::sync::Arc;+use hiroute_domain::{+ FactsCompleteness, ObservationQueryPort, ReceiptId, RequestOutcome, WorkspaceId,+};use hiroute_gateway::server::core_runtime::observation::{EXECUTION_FACT_PORT_DIGEST, EXECUTION_FACT_SCHEMA,};@@ -11,6 +14,116 @@ use super::super::GatewayExecutionFactSink;const DIGEST: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";+#[test]+fn current_credential_denials_reach_real_product_ingestion_with_released_outcomes() {+ let directory = tempfile::tempdir().unwrap();+ let store = Arc::new(+ LocalObservationStore::open(directory.path(), DigestAuthority::new([32; 32])).unwrap(),+ );+ let sink = GatewayExecutionFactSink::new(store);+ for (index, outcome) in [+ "leased",+ "exhausted",+ "authority_error",+ "source_disabled",+ "credential_disabled",+ ]+ .into_iter()+ .enumerate()+ {+ let sequence = u64::try_from(index).unwrap() + 1;+ let mut payload = execution_payload(sequence, &format!("request-{sequence}"), None);+ payload.as_object_mut().unwrap().remove("attempt_id");+ payload["fact"] = json!({+ "kind": "credential_lease", "stable_binding_id": "binding-main", "credential_ref": "credential-main",+ "key_id": if outcome == "leased" { json!("key-main") } else { Value::Null },+ "credential_generation": if outcome == "leased" { json!(1) } else { Value::Null },+ "excluded_key_count": 0, "outcome": outcome+ });+ let result = sink+ .project_and_ingest_payload_for_test(&serde_json::to_vec(&payload).unwrap())+ .unwrap();+ assert!(matches!(result, IngestOutcome::Ack(_)), "outcome {outcome}");+ }+}++#[test]+fn current_credential_denials_close_complete_zero_attempt_receipts_and_reopen() {+ let directory = tempfile::tempdir().unwrap();+ let store = Arc::new(+ LocalObservationStore::open(directory.path(), DigestAuthority::new([33; 32])).unwrap(),+ );+ let sink = GatewayExecutionFactSink::new(store.clone());+ let mut sequence = 0;+ let mut receipts = Vec::new();+ for outcome in ["source_disabled", "credential_disabled"] {+ for stream in [false, true] {+ let request_id = format!("request-{outcome}-{stream}");+ let lease = json!({+ "kind": "credential_lease", "stable_binding_id": "binding-main",+ "credential_ref": "credential-main", "key_id": null,+ "credential_generation": null, "excluded_key_count": 0, "outcome": outcome+ });+ let finished = json!({+ "kind": "request_finished", "outcome": "failed", "attempts_started": 0,+ "attempts_finished": 0, "accepted_attempt_ordinal": null,+ "facts_completeness": "complete"+ });+ for fact in [route_decision(stream), lease.clone(), finished.clone()] {+ sequence += 1;+ let mut payload = execution_payload(sequence, &request_id, None);+ payload.as_object_mut().unwrap().remove("attempt_id");+ payload["fact"] = fact;+ let result = sink+ .project_and_ingest_payload_for_test(&serde_json::to_vec(&payload).unwrap())+ .unwrap();+ assert!(matches!(result, IngestOutcome::Ack(_)));+ }+ let receipt_id = ReceiptId::parse(request_id.clone()).unwrap();+ let receipt = store+ .get_receipt(&WorkspaceId::default(), &receipt_id)+ .unwrap();+ receipt.validate().unwrap();+ assert_eq!(receipt.request_id.as_str(), request_id);+ assert_eq!(receipt.outcome, RequestOutcome::Failed);+ assert_eq!(receipt.facts_completeness, FactsCompleteness::Complete);+ assert!(receipt.final_attempt_id.is_none());+ assert_eq!(receipt.ordered_facts.len(), 3);+ assert!(+ receipt+ .ordered_facts+ .iter()+ .all(|event| event.attempt_id.is_none())+ );+ assert_eq!(+ serde_json::to_value(&receipt.ordered_facts[0].fact).unwrap(),+ route_decision(stream)+ );+ assert_eq!(+ serde_json::to_value(&receipt.ordered_facts[1].fact).unwrap(),+ lease+ );+ assert_eq!(+ serde_json::to_value(&receipt.ordered_facts[2].fact).unwrap(),+ finished+ );+ let digest = receipt.digest().unwrap();+ receipts.push((receipt_id, receipt, digest));+ }+ }+ drop(sink);+ drop(store);+ let reopened =+ LocalObservationStore::open(directory.path(), DigestAuthority::new([33; 32])).unwrap();+ for (receipt_id, receipt, digest) in receipts {+ let read = reopened+ .get_receipt(&WorkspaceId::default(), &receipt_id)+ .unwrap();+ assert_eq!(read, receipt);+ assert_eq!(read.digest().unwrap(), digest);+ }+}+#[test]fn current_gateway_payloads_collect_priced_and_unpriced_attempts_in_one_product_table() {let directory = tempfile::tempdir().unwrap();@@ -111,6 +224,28 @@ fn execution_payload(sequence: u64, request_id: &str, pricing: Option<Value>) ->})}+fn route_decision(stream: bool) -> Value {+ json!({+ "kind": "route_decision", "planner_version": "hiroute-deterministic-planner/v1",+ "plan_id": "plan/coding",+ "route": {"kind": "plan", "revision": 7, "semantic_digest": DIGEST},+ "input_digest": DIGEST, "policy_digest": DIGEST, "output_digest": DIGEST,+ "branch": "custom_exact_order", "complexity": null, "groups": [], "reason_ledger": [],+ "requirements": {+ "text": true, "image_url": false, "image_base64": false, "image_media_types": [],+ "function_tools": false, "strict_tools": false, "parallel_tools": false,+ "tool_choice": {"kind": "auto"}, "tool_result_text": false, "tool_result_json": false,+ "tool_roundtrip": false, "logical_tool_id_mapping": false, "provider_state": false,+ "initial_instructions": false, "mid_conversation_instructions": false,+ "streaming": stream, "stream_text": stream, "stream_reasoning": false,+ "stream_tool_arguments": false, "stream_usage": false, "ingress_protocol": "responses"+ },+ "requested_reasoning_disposition": "absent", "requested_reasoning_value": null,+ "requested_max_output_tokens": null, "stream": stream, "outcome": "ready",+ "outcome_code": null, "max_attempts": 1+ })+}+fn pricing() -> Value {json!({"schema_version": "hiroute.observation.execution-pricing/v1",diff --git a/crates/daemon/tests/support/model_connections_product.py b/crates/daemon/tests/support/model_connections_product.pyindex 11e3135f..6fd62f50 100644--- a/crates/daemon/tests/support/model_connections_product.py+++ b/crates/daemon/tests/support/model_connections_product.py@@ -458,13 +458,13 @@ def publish_plan_and_agent(product, binding_id, native_model_mode='preserve_avaiassert [model['id'] for model in catalog['data']] == expected_models, catalog-def gateway_request(product, allowed=True):+def gateway_request(product, allowed=True, stream=False):client = http.client.HTTPConnection('127.0.0.1', product.port, timeout=30)try:client.request('POST', '/v1/responses', body=encoded({'model': product.model_alias,'input': 'native product request',- 'stream': False,+ 'stream': stream,}), headers={'X-HiRoute-Token': product.bearer(product.agent_connection),'Content-Type': 'application/json',@@ -475,7 +475,13 @@ def gateway_request(product, allowed=True):if allowed:assert response.status == 200 and b'native product answer' in body, (response.status, body)else:- assert response.status >= 400 and b'native product answer' not in body, (response.status, body)+ assert response.status == 502, (response.status, body)+ assert response.getheader('Content-Type') == 'application/json', response.getheaders()+ assert response.getheader('Retry-After') is None, response.getheaders()+ assert json.loads(body) == {+ 'schema_version': 'hiroute.gateway.error/v1',+ 'code': 'SOURCE_DISABLED', 'phase': 'credential_lease',+ }, bodyfinally:client.close()@@ -515,7 +521,8 @@ def assert_live_credential_switches(product, upstream, saved):def rejected_without_upstream():with upstream.lock:before = len(upstream.requests)- gateway_request(product, allowed=False)+ for stream in (False, True):+ gateway_request(product, allowed=False, stream=stream)with upstream.lock:assert len(upstream.requests) == before, 'disabled credential reached the upstream'@@ -643,7 +650,9 @@ def run(repository, expected_sha=None):'saved_binding': saved['binding_id'],'saved_source_rechecks': 2,'gateway_attempts': 3,- 'disabled_requests_without_upstream': 3,+ 'disabled_requests_without_upstream': 6,+ 'preparation_error_code': 'SOURCE_DISABLED',+ 'preparation_error_modes': ['nonstream', 'prebody_stream'],'credential_resolver_mode': 'saved_bearer','session_record': session_id,'session_model': observed_model,diff --git a/crates/domain/src/compute/credential_runtime.rs b/crates/domain/src/compute/credential_runtime.rsindex 69374dca..20ed174a 100644--- a/crates/domain/src/compute/credential_runtime.rs+++ b/crates/domain/src/compute/credential_runtime.rs@@ -244,10 +244,12 @@ impl NativeCredentialLeaseV1 {/// Replaceable Product authority. Local storage, a Vault client, or a remote authority can/// implement this same port; Gateway adapters never depend on SQLite types.pub trait NativeCredentialAuthorityV1: Send + Sync {+ /// An empty lease means exact exhaustion. Current managed source/Key permission+ /// denials retain their closed reason before any secret is decrypted.fn lease_native_credential(&self,request: &NativeCredentialLeaseRequestV1,- ) -> PortResult<Option<NativeCredentialLeaseV1>>;+ ) -> Result<Option<NativeCredentialLeaseV1>, NativeCredentialLeaseErrorV1>;fn lease_header_secret(&self,@@ -255,6 +257,21 @@ pub trait NativeCredentialAuthorityV1: Send + Sync {) -> PortResult<Option<NativeCredentialLeaseV1>>;}+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub enum NativeCredentialAccessDeniedV1 {+ SourceDisabled,+ CredentialDisabled,+}++/// In-process lease failure, separate from persisted credential and request schemas.+#[derive(Clone, Debug, Error, Eq, PartialEq)]+pub enum NativeCredentialLeaseErrorV1 {+ #[error("current managed credential permission was denied: {0:?}")]+ Denied(NativeCredentialAccessDeniedV1),+ #[error(transparent)]+ Authority(#[from] crate::PortError),+}+#[derive(Clone, Copy, Debug, Error, Eq, PartialEq)]pub enum NativeCredentialContractErrorV1 {#[error("native credential request is invalid")]diff --git a/crates/domain/src/observation/execution_attempt.rs b/crates/domain/src/observation/execution_attempt.rsindex 4bf0478c..791f5596 100644--- a/crates/domain/src/observation/execution_attempt.rs+++ b/crates/domain/src/observation/execution_attempt.rs@@ -28,6 +28,8 @@ pub enum CredentialLeaseOutcomeV1 {Leased,Exhausted,AuthorityError,+ SourceDisabled,+ CredentialDisabled,}#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]diff --git a/crates/domain/src/observation/execution_tests.rs b/crates/domain/src/observation/execution_tests.rsindex 5e570925..84eae7fe 100644--- a/crates/domain/src/observation/execution_tests.rs+++ b/crates/domain/src/observation/execution_tests.rs@@ -4,6 +4,31 @@ use crate::{AgentPlanId, CanonicalDigest, WorkspaceId};use super::*;+#[test]+fn credential_denial_extension_keeps_released_outcomes_and_rejects_unknown_values() {+ // The three original outcomes were verified at supported v0.2.0/v0.3.0/v0.3.1 sources.+ for (wire, value) in [+ ("leased", CredentialLeaseOutcomeV1::Leased),+ ("exhausted", CredentialLeaseOutcomeV1::Exhausted),+ ("authority_error", CredentialLeaseOutcomeV1::AuthorityError),+ ("source_disabled", CredentialLeaseOutcomeV1::SourceDisabled),+ (+ "credential_disabled",+ CredentialLeaseOutcomeV1::CredentialDisabled,+ ),+ ] {+ let bytes = format!("\"{wire}\"");+ assert_eq!(+ serde_json::from_str::<CredentialLeaseOutcomeV1>(&bytes).unwrap(),+ value+ );+ assert_eq!(serde_json::to_string(&value).unwrap(), bytes);+ }+ assert!(+ serde_json::from_str::<CredentialLeaseOutcomeV1>("\"provider_private_detail\"").is_err()+ );+}+#[test]fn adapter_fact_contract_roundtrips_every_gateway_variant_and_identity_field() {let envelopes = gateway_fact_envelopes();diff --git a/crates/gateway-core/src/runtime/driver/decision.rs b/crates/gateway-core/src/runtime/driver/decision.rsindex 03c84a70..ec60ccf4 100644--- a/crates/gateway-core/src/runtime/driver/decision.rs+++ b/crates/gateway-core/src/runtime/driver/decision.rs@@ -456,9 +456,10 @@ pub trait DecisionSessionPort: Send {request: SelectionRequest<'_>,) -> Result<Option<SelectedGatewayAttempt>, Arc<str>>;- /// Optional product-owned response when selection ended before any- /// upstream attempt. Core keeps its generic 502 when no exact response is- /// available.+ /// Optional product-owned response when selection is exhausted or a provisional+ /// candidate terminates before an upstream attempt. Core keeps its generic 502+ /// when no exact response is available. This does not change failure disposition+ /// or publish a formal attempt for a provisional candidate.fn exhaustion_local_reply(&mut self, _now: Instant) -> Result<Option<LocalReply>, Arc<str>> {Ok(None)}diff --git a/crates/gateway-core/src/runtime/driver/lifecycle/preexchange/failure.rs b/crates/gateway-core/src/runtime/driver/lifecycle/preexchange/failure.rsindex 5cfbec1c..c6bb3856 100644--- a/crates/gateway-core/src/runtime/driver/lifecycle/preexchange/failure.rs+++ b/crates/gateway-core/src/runtime/driver/lifecycle/preexchange/failure.rs@@ -170,6 +170,15 @@ where).map_err(GatewayExecutionError::ProviderTerminalRelease)?;let _release_ready = leases.mark_terminal_and_release_ready()?;+ let local_reply = decision_session+ .exhaustion_local_reply(Instant::now())+ .map_err(GatewayExecutionError::Selection)?+ .unwrap_or_else(|| LocalReply {+ status: StatusCode::BAD_GATEWAY,+ headers: HeaderMap::new(),+ body: Bytes::from_static(b"upstream candidate preparation failed"),+ provenance: SemanticProvenance::NonSemantic,+ });let result = emit_local_response(session,final_writer,@@ -178,12 +187,7 @@ wherebinding,request_configs,&mut request_filters.filters,- LocalReply {- status: StatusCode::BAD_GATEWAY,- headers: HeaderMap::new(),- body: Bytes::from_static(b"upstream candidate preparation failed"),- provenance: SemanticProvenance::NonSemantic,- },+ local_reply,budget,request_id,Some(route_binding),diff --git a/crates/gateway-core/tests/gateway_lifecycle/fixture/selection.rs b/crates/gateway-core/tests/gateway_lifecycle/fixture/selection.rsindex 90d19c65..cd9c7cea 100644--- a/crates/gateway-core/tests/gateway_lifecycle/fixture/selection.rs+++ b/crates/gateway-core/tests/gateway_lifecycle/fixture/selection.rs@@ -19,6 +19,8 @@ pub(crate) struct SelectionState {pub(crate) fail_observation_sink: bool,pub(crate) routing_snapshots: VecDeque<RealtimeRoutingFacts>,pub(crate) preexchange_completion_trace: Option<Arc<Mutex<Vec<PreexchangeCompletionStep>>>>,+ pub(crate) provisional_candidates: bool,+ pub(crate) exhaustion_reply: Option<LocalReply>,}#[derive(Clone, Copy, Debug, Eq, PartialEq)]@@ -61,6 +63,8 @@ impl TestSelection {fail_observation_sink: false,routing_snapshots: VecDeque::new(),preexchange_completion_trace: None,+ provisional_candidates: false,+ exhaustion_reply: None,})),}}@@ -70,6 +74,14 @@ impl TestSelection {self}+ pub(crate) fn with_provisional_reply(self, reply: Option<LocalReply>) -> Self {+ let mut state = self.state.lock().expect("selection state");+ state.provisional_candidates = true;+ state.exhaustion_reply = reply;+ drop(state);+ self+ }+pub(crate) fn with_attempt_budgets(self,attempt_budgets: impl IntoIterator<Item = Duration>,@@ -268,7 +280,18 @@ impl DecisionSessionPort for TestDecisionSession {.unwrap_or(request.generation);let selected = SelectedGatewayAttempt {request_id: request.request_id,- attempt_id: hiroute_gateway_core::runtime::attempt::AttemptId(self.next_attempt_id),+ attempt_id: hiroute_gateway_core::runtime::attempt::AttemptId(+ if self+ .state+ .lock()+ .map_err(|_| Arc::from("poisoned selector"))?+ .provisional_candidates+ {+ 0+ } else {+ self.next_attempt_id+ },+ ),generation,binding,credential_ref: CredentialRef::new(format!(@@ -302,6 +325,15 @@ impl DecisionSessionPort for TestDecisionSession {Ok(Some(selected))}+ fn exhaustion_local_reply(&mut self, _now: Instant) -> Result<Option<LocalReply>, Arc<str>> {+ Ok(self+ .state+ .lock()+ .map_err(|_| Arc::from("poisoned selector"))?+ .exhaustion_reply+ .clone())+ }+fn decide(&mut self,_selected: &SelectedGatewayAttempt,diff --git a/crates/gateway-core/tests/gateway_lifecycle/preexchange.rs b/crates/gateway-core/tests/gateway_lifecycle/preexchange.rsindex 49e8d904..90430963 100644--- a/crates/gateway-core/tests/gateway_lifecycle/preexchange.rs+++ b/crates/gateway-core/tests/gateway_lifecycle/preexchange.rs@@ -1,5 +1,91 @@use crate::fixture::*;+#[tokio::test]+async fn provisional_termination_uses_product_reply_without_publishing_an_attempt()+-> Result<(), TestError> {+ let _network_guard = NETWORK_TEST_LOCK.lock().await;+ for product_reply in [true, false] {+ let listener = TcpListener::bind("127.0.0.1:0").await?;+ let plan = PlanRevision(7591);+ let binding = ResolvedTargetBindingId::new(plan, 1);+ let publications =+ install_publication(BootstrapPublicationBuilder::new(plan.0, 59).route(+ "gateway.test",+ "/preparation-denied",+ 1,+ plain_target(listener.local_addr()?, 59),+ )?)?;+ let mut headers = HeaderMap::new();+ headers.insert(+ http::header::CONTENT_TYPE,+ http::HeaderValue::from_static("application/json"),+ );+ let reply = LocalReply {+ status: StatusCode::BAD_GATEWAY,+ headers,+ body: Bytes::from_static(br#"{"schema_version":"hiroute.gateway.error/v1","code":"SOURCE_DISABLED","phase":"credential_lease"}"#),+ provenance: SemanticProvenance::NonSemantic,+ };+ let selection = TestSelection::new([binding])+ .with_provisional_reply(product_reply.then_some(reply.clone()));+ let provider = PassthroughProvider {+ materialization_failure: Some(AttemptMaterializationFailure {+ class: AttemptMaterializationFailureClass::Credential,+ provider: ProviderClassificationFacts {+ error_class: Some(ObservationLabel::new("source_disabled").unwrap()),+ retryability: RetryabilityFact::Retryable,+ readiness: ObservationLabel::new("materialization_failed").unwrap(),+ ..ProviderClassificationFacts::default()+ },+ termination_reason: ObservationLabel::new("credential").unwrap(),+ }),+ materialization_failures_remaining: Arc::new(AtomicUsize::new(1)),+ materialization_raw_error: Some(Arc::from("private provider credential detail")),+ ..PassthroughProvider::default()+ };+ let finalized = Arc::clone(&provider.finalized_attempts);+ let gateway = lifecycle_with_provider(+ publications,+ selection.clone(),+ provider,+ TrackingFilters::default(),+ )?;+ let mut session = RecordingSession::new(+ "gateway.test",+ "/preparation-denied",+ Bytes::from_static(b"prompt"),+ HttpProtocol::Http1,+ );+ assert_eq!(gateway.process(&mut session).await?, SessionReuse::Reusable);+ assert_eq!(+ session.response_head.as_ref().unwrap().status,+ StatusCode::BAD_GATEWAY+ );+ if product_reply {+ assert_eq!(session.response_body, reply.body);+ assert_eq!(+ session.response_head.as_ref().unwrap().headers[http::header::CONTENT_TYPE],+ "application/json"+ );+ } else {+ assert_eq!(+ session.response_body,+ b"upstream candidate preparation failed"+ );+ }+ assert_eq!(selection.failures(), [AttemptFailureClass::Credential]);+ assert!(selection.completed().is_empty());+ assert!(selection.published().is_empty());+ assert_eq!(finalized.load(Ordering::Relaxed), 0);+ assert!(+ tokio::time::timeout(Duration::from_millis(50), listener.accept())+ .await+ .is_err()+ );+ }+ Ok(())+}+#[tokio::test]async fn materialized_attempt_filter_failure_finalizes_before_fallback_selection()-> Result<(), TestError> {diff --git a/crates/gateway/README.md b/crates/gateway/README.mdindex c20b96fc..ab0fe797 100644--- a/crates/gateway/README.md+++ b/crates/gateway/README.md@@ -161,6 +161,41 @@ error message or a substitute for evidence of credential failure.## Response failure evidence+### Candidate preparation errors++A published plan freezes candidate selection, while the exact credential authority+still checks current source and Key permission before decryption. A managed source+or Key denial retains its closed cause through the daemon adapter; an empty lease+means only that the exact credential is exhausted. Neither outcome starts an upstream+attempt or broadens the frozen fallback policy.++When preparation terminates with no upstream attempt, the core asks the existing+decision-session local-reply port for the product response. HiRoute returns HTTP 502+with `Content-Type: application/json`, `schema_version: hiroute.gateway.error/v1`,+and these closed fields:++| Preparation evidence | `code` | `phase` |+| --- | --- | --- |+| Every rejected candidate has a managed source disabled | `SOURCE_DISABLED` | `credential_lease` |+| Every rejected candidate has a managed Key disabled | `CREDENTIAL_DISABLED` | `credential_lease` |+| Every exact credential lease is exhausted | `CREDENTIAL_UNAVAILABLE` | `credential_lease` |+| Other or mixed preparation failures | `CANDIDATE_PREPARATION_FAILED` | `candidate_preparation` |++These replies have no `Retry-After`. Pure cooldown/probe exhaustion retains its+existing HTTP 503 runtime-state response and bounded retry hint. A failure after a+formal upstream attempt keeps the existing attempt response; this port cannot rewrite+committed bytes. No provider text, credential identity or authorization material is+included in these local replies.++Request-level `credential_lease` execution facts retain the denial as `source_disabled`+or `credential_disabled`; exhausted and authority errors remain distinct. Their+existing envelope is unchanged; the Domain ingestion enum accepts these two new values+alongside the original leased, exhausted and authority-error outcomes. With observation enabled and a+healthy sink, the request finishes as failed with zero started/finished attempts,+no accepted attempt and no fabricated usage, and can produce a zero-attempt receipt.+Follow the [live source/Key regression](../daemon/tests/support/model_connections_product.py)+for disable, restart, independent-source isolation and re-enable evidence.+`runtime/driver/response_diagnostics.rs` owns the closed, payload-free failureclassification; `adapters/response/decoder.rs` reports SSE event ordinal andreceived-byte upper bound without per-token logs. Unix Debug-onlydiff --git a/crates/gateway/src/composition/ports.rs b/crates/gateway/src/composition/ports.rsindex 69d7915b..3c7ffcc2 100644--- a/crates/gateway/src/composition/ports.rs+++ b/crates/gateway/src/composition/ports.rs@@ -707,6 +707,8 @@ pub enum PortError {Unavailable(&'static str),#[error("production port rejected the operation")]Rejected,+ #[error("current managed credential permission was denied: {0:?}")]+ CredentialDenied(hiroute_domain::NativeCredentialAccessDeniedV1),#[error("requested fixed-model reasoning cannot be represented")]InvalidReasoningControl,}diff --git a/crates/gateway/src/observation/ports.rs b/crates/gateway/src/observation/ports.rsindex 08293daf..1d296506 100644--- a/crates/gateway/src/observation/ports.rs+++ b/crates/gateway/src/observation/ports.rs@@ -57,6 +57,20 @@ impl CredentialResolver for ObservedCredentialResolver {None,"exhausted",),+ Err(PortError::CredentialDenied(reason)) => observation.credential_lease(+ &stable_binding_id,+ &credential_ref,+ excluded_key_count,+ None,+ match reason {+ hiroute_domain::NativeCredentialAccessDeniedV1::SourceDisabled => {+ "source_disabled"+ }+ hiroute_domain::NativeCredentialAccessDeniedV1::CredentialDisabled => {+ "credential_disabled"+ }+ },+ ),Err(_) => observation.credential_lease(&stable_binding_id,&credential_ref,diff --git a/crates/gateway/src/runtime/driver.rs b/crates/gateway/src/runtime/driver.rsindex fa710cc0..5a69e036 100644--- a/crates/gateway/src/runtime/driver.rs+++ b/crates/gateway/src/runtime/driver.rs@@ -48,6 +48,8 @@ pub use selection::{ProductionDecisionSession, ProductionSelection};const MATERIALIZATION_BINDING_UNAVAILABLE: &str = "binding_unavailable";const MATERIALIZATION_CREDENTIAL_EXHAUSTED: &str = "credential_exhausted";+const MATERIALIZATION_SOURCE_DISABLED: &str = "managed_source_disabled";+const MATERIALIZATION_KEY_DISABLED: &str = "managed_key_disabled";const MATERIALIZATION_BINDING_COOLING_PREFIX: &str = "binding_cooling_down:";const MATERIALIZATION_CREDENTIAL_COOLING_PREFIX: &str = "credential_cooling_down:";const MATERIALIZATION_BINDING_PROBE_BUSY: &str = "binding_probe_busy";diff --git a/crates/gateway/src/runtime/driver/materialization.rs b/crates/gateway/src/runtime/driver/materialization.rsindex b6b34236..4fa65951 100644--- a/crates/gateway/src/runtime/driver/materialization.rs+++ b/crates/gateway/src/runtime/driver/materialization.rs@@ -44,11 +44,11 @@ use super::{MATERIALIZATION_BINDING_UNAVAILABLE, MATERIALIZATION_CREDENTIAL_COOLING_PREFIX,MATERIALIZATION_CREDENTIAL_DISABLED, MATERIALIZATION_CREDENTIAL_EXHAUSTED,MATERIALIZATION_CREDENTIAL_PROBE_BUSY, MATERIALIZATION_DNS_FAILED,- MATERIALIZATION_PROTOCOL_FAILED, MechanicalFailureSignal, PermitAvailability,- ProductionAttemptState, ProductionLogicalRequest, ProductionProvider,- ProductionReplaySeedEnvelope, RUNTIME_STATE_AUTHORITY_FAILED, RuntimeCooldownPolicy,- StatePermit, acquire_target_permit_status, confirm_success, label, record_failure, safe_error,- validate_attempt_permits,+ MATERIALIZATION_KEY_DISABLED, MATERIALIZATION_PROTOCOL_FAILED, MATERIALIZATION_SOURCE_DISABLED,+ MechanicalFailureSignal, PermitAvailability, ProductionAttemptState, ProductionLogicalRequest,+ ProductionProvider, ProductionReplaySeedEnvelope, RUNTIME_STATE_AUTHORITY_FAILED,+ RuntimeCooldownPolicy, StatePermit, acquire_target_permit_status, confirm_success, label,+ record_failure, safe_error, validate_attempt_permits,};pub(super) struct ProductionStateAdapter {@@ -369,7 +369,13 @@ pub(super) async fn materialize_attempt()).await.map_err(|_| Arc::from(MATERIALIZATION_AUTHORITY_FAILED))?- .map_err(|_| Arc::from(MATERIALIZATION_AUTHORITY_FAILED))?+ .map_err(|error| Arc::from(match error {+ crate::server::composition::PortError::CredentialDenied(reason) => match reason {+ hiroute_domain::NativeCredentialAccessDeniedV1::SourceDisabled => MATERIALIZATION_SOURCE_DISABLED,+ hiroute_domain::NativeCredentialAccessDeniedV1::CredentialDisabled => MATERIALIZATION_KEY_DISABLED,+ },+ _ => MATERIALIZATION_AUTHORITY_FAILED,+ }))?.ok_or_else(|| Arc::from(MATERIALIZATION_CREDENTIAL_EXHAUSTED))?,)};@@ -629,11 +635,15 @@ pub(super) fn classify_materialization_failure(error: &Arc<str>) -> AttemptMatererror.as_ref(),MATERIALIZATION_BINDING_UNAVAILABLE| MATERIALIZATION_CREDENTIAL_EXHAUSTED+ | MATERIALIZATION_SOURCE_DISABLED+ | MATERIALIZATION_KEY_DISABLED| MATERIALIZATION_DNS_FAILED);let class = match error.as_ref() {MATERIALIZATION_DNS_FAILED => AttemptMaterializationFailureClass::Dns,MATERIALIZATION_CREDENTIAL_EXHAUSTED+ | MATERIALIZATION_SOURCE_DISABLED+ | MATERIALIZATION_KEY_DISABLED| MATERIALIZATION_CREDENTIAL_PROBE_BUSY| MATERIALIZATION_CREDENTIAL_DISABLED => AttemptMaterializationFailureClass::Credential,_ if credential_cooldown.is_some() => AttemptMaterializationFailureClass::Credential,@@ -648,6 +658,12 @@ pub(super) fn classify_materialization_failure(error: &Arc<str>) -> AttemptMater"runtime_state_probe_busy"} else if disabled {"runtime_state_disabled"+ } else if error.as_ref() == MATERIALIZATION_SOURCE_DISABLED {+ "source_disabled"+ } else if error.as_ref() == MATERIALIZATION_KEY_DISABLED {+ "credential_disabled"+ } else if error.as_ref() == MATERIALIZATION_CREDENTIAL_EXHAUSTED {+ "credential_exhausted"} else if retryable {"materialization_retryable"} else {diff --git a/crates/gateway/src/runtime/driver/selection.rs b/crates/gateway/src/runtime/driver/selection.rsindex d49772b6..f84f034a 100644--- a/crates/gateway/src/runtime/driver/selection.rs+++ b/crates/gateway/src/runtime/driver/selection.rs@@ -22,6 +22,9 @@ use crate::context_hold::{ContextHoldStore, HoldCompletion};use super::ProductionRouteContext;+mod preparation;+use preparation::PreparationFailure;+#[derive(Clone, Default)]pub struct ProductionSelection;@@ -40,6 +43,7 @@ pub struct ProductionDecisionSession {earliest_retry_at: Option<Instant>,non_temporary_block_seen: bool,upstream_attempted: bool,+ preparation_failure: Option<PreparationFailure>,}impl DecisionSessionPort for ProductionDecisionSession {@@ -92,9 +96,14 @@ impl DecisionSessionPort for ProductionDecisionSession {}fn exhaustion_local_reply(&mut self, now: Instant) -> Result<Option<LocalReply>, Arc<str>> {- if !self.temporary_block_seen || self.non_temporary_block_seen || self.upstream_attempted {+ if self.upstream_attempted {return Ok(None);}+ if !self.temporary_block_seen || self.non_temporary_block_seen {+ return Ok(self+ .preparation_failure+ .map(PreparationFailure::local_reply));+ }let (code, retry_after_seconds) = if self.probe_busy_seen {("RECOVERY_PROBE_IN_PROGRESS", 1)} else {@@ -195,6 +204,13 @@ impl DecisionSessionPort for ProductionDecisionSession {error_class,Some("runtime_state_cooling_down" | "runtime_state_probe_busy"));+ if selected.attempt_id.0 == 0 {+ let cause = PreparationFailure::from_error_class(error_class);+ self.preparation_failure = Some(+ self.preparation_failure+ .map_or(cause, |previous| previous.merge(cause)),+ );+ }if temporary && selected.attempt_id.0 == 0 {self.temporary_block_seen = true;if error_class == Some("runtime_state_probe_busy") {@@ -351,6 +367,7 @@ impl SelectionPublicationPort<ProductionRouteContext> for ProductionSelection {earliest_retry_at: None,non_temporary_block_seen: false,upstream_attempted: false,+ preparation_failure: None,})}}@@ -411,686 +428,4 @@ impl ProductionDecisionSession {}#[cfg(test)]-mod tests {- use std::time::Duration;-- use hiroute_gateway_core::core::execution_plan::{- CredentialRef, PlanRevision, ResolvedTargetBindingId,- };- use hiroute_gateway_core::runtime::attempt::{- AttemptGeneration, AttemptTransportFacts, CommitFence, RequestId,- };- use hiroute_gateway_core::runtime::driver::{- AttemptCommitFacts, AttemptFailureClass, AttemptFailureFacts,- };-- use super::*;-- fn candidate(binding_local_id: u32, credential_refs: &[&str]) -> DecisionCandidateAuthority {- DecisionCandidateAuthority {- binding: ResolvedTargetBindingId::new(PlanRevision(1), binding_local_id),- stable_target: ObservationLabel::new(format!("binding-{binding_local_id}")).unwrap(),- credential_refs: credential_refs- .iter()- .map(|value| CredentialRef::new(*value).unwrap())- .collect::<Vec<_>>()- .into(),- candidate_id: None,- profile_digest: None,- reason_ledger_identity: None,- provider_profile: None,- }- }-- fn selection_request<'a>(- _now: Instant,- generation: u64,- remaining_attempts: u32,- realtime_facts: &'a RealtimeRoutingFacts,- ) -> SelectionRequest<'a> {- SelectionRequest {- request_id: RequestId(1),- generation: AttemptGeneration(generation),- route_binding: ResolvedTargetBindingId::new(PlanRevision(1), 1),- remaining_total: Duration::from_secs(1),- remaining_attempts,- realtime_facts,- previous_attempts: &[],- }- }-- fn session(- now: Instant,- candidates: Vec<DecisionCandidateAuthority>,- ) -> ProductionDecisionSession {- ProductionDecisionSession {- request_id: RequestId(1),- route_decision_id: RouteDecisionId(1),- candidates: candidates.into(),- candidate_index: 0,- credential_index: 0,- overall_deadline: now + Duration::from_secs(10),- hold: None,- switch_fallback: None,- reasoning_cleanup: Arc::new(super::super::ReasoningCleanup::new(None, None)),- temporary_block_seen: false,- probe_busy_seen: false,- earliest_retry_at: None,- non_temporary_block_seen: false,- upstream_attempted: false,- }- }-- fn materialization_failure(- now: Instant,- error_class: &str,- retry_after: Option<Duration>,- ) -> AttemptFailureFacts {- AttemptFailureFacts {- class: AttemptFailureClass::Materialization,- provider: Some(ProviderClassificationFacts {- error_class: Some(ObservationLabel::new(error_class).unwrap()),- retryability: RetryabilityFact::Retryable,- retry_after,- readiness: ObservationLabel::new("materialization_failed").unwrap(),- ..ProviderClassificationFacts::default()- }),- transport: AttemptTransportFacts {- started_at: now,- upstream_protocol: None,- connect_elapsed: None,- request_write_elapsed: None,- upstream_ttfb: None,- last_upstream_progress_at: None,- local_read_suppressed: Duration::ZERO,- upstream_body_bytes: 0,- timeout: None,- },- request_committed: false,- termination_reason: ObservationLabel::new("materialization").unwrap(),- }- }-- #[test]- fn retryable_quota_advances_exact_credential_before_binding() {- let now = Instant::now();- let realtime_facts = RealtimeRoutingFacts::default();- let transport = AttemptTransportFacts {- started_at: now,- upstream_protocol: None,- connect_elapsed: None,- request_write_elapsed: None,- upstream_ttfb: None,- last_upstream_progress_at: None,- local_read_suppressed: Duration::ZERO,- upstream_body_bytes: 0,- timeout: None,- };- let mut session = ProductionDecisionSession {- request_id: RequestId(1),- route_decision_id: RouteDecisionId(1),- candidates: vec![- candidate(1, &["credential-a", "credential-b"]),- candidate(2, &["credential-c"]),- ]- .into(),- candidate_index: 0,- credential_index: 0,- overall_deadline: now + Duration::from_secs(1),- hold: None,- switch_fallback: None,- reasoning_cleanup: Arc::new(super::super::ReasoningCleanup::new(None, None)),- temporary_block_seen: false,- probe_busy_seen: false,- earliest_retry_at: None,- non_temporary_block_seen: false,- upstream_attempted: false,- };- let first = session- .select_next(selection_request(now, 1, 3, &realtime_facts))- .unwrap()- .unwrap();- assert_eq!(first.binding.local_id(), 1);- assert_eq!(first.credential_ref.as_str(), "credential-a");-- let quota = ProviderClassificationFacts {- error_class: Some(ObservationLabel::new("quota").unwrap()),- retryability: RetryabilityFact::Retryable,- readiness: ObservationLabel::new("rejected").unwrap(),- ..ProviderClassificationFacts::default()- };- assert_eq!(- session.decide(&first, "a, &transport).unwrap(),- Disposition::Continue- );- let second = session- .select_next(selection_request(now, 2, 2, &realtime_facts))- .unwrap()- .unwrap();- assert_eq!(second.binding.local_id(), 1);- assert_eq!(second.credential_ref.as_str(), "credential-b");-- assert_eq!(- session.decide(&second, "a, &transport).unwrap(),- Disposition::Continue- );- let third = session- .select_next(selection_request(now, 3, 1, &realtime_facts))- .unwrap()- .unwrap();- assert_eq!(third.binding.local_id(), 2);- assert_eq!(third.credential_ref.as_str(), "credential-c");- }-- #[test]- fn switched_candidate_precommit_400_falls_back_to_previous_once() {- let now = Instant::now();- let realtime_facts = RealtimeRoutingFacts::default();- let mut session = session(- now,- vec![candidate(1, &["new-key"]), candidate(2, &["original-key"])],- );- session.switch_fallback = Some(ResolvedTargetBindingId::new(PlanRevision(1), 2));- let mut selected = session- .select_next(selection_request(now, 1, 2, &realtime_facts))- .unwrap()- .unwrap();- // The core promotes the candidate after pre-exchange materialization.- selected.attempt_id = AttemptId(1);- let rejected = ProviderClassificationFacts {- error_class: Some(ObservationLabel::new("permanent_client").unwrap()),- retryability: RetryabilityFact::NonRetryable,- http_status: Some(StatusCode::BAD_REQUEST),- ..ProviderClassificationFacts::default()- };- assert_eq!(- session- .decide(- &selected,- &rejected,- &AttemptTransportFacts {- started_at: now,- upstream_protocol: None,- connect_elapsed: None,- request_write_elapsed: None,- upstream_ttfb: None,- last_upstream_progress_at: None,- local_read_suppressed: Duration::ZERO,- upstream_body_bytes: 0,- timeout: None,- }- )- .unwrap(),- Disposition::Continue- );- let original = session- .select_next(selection_request(now, 2, 1, &realtime_facts))- .unwrap()- .unwrap();- assert_eq!(original.binding.local_id(), 2);- assert_eq!(- session- .decide(- &original,- &rejected,- &AttemptTransportFacts {- started_at: now,- upstream_protocol: None,- connect_elapsed: None,- request_write_elapsed: None,- upstream_ttfb: None,- last_upstream_progress_at: None,- local_read_suppressed: Duration::ZERO,- upstream_body_bytes: 0,- timeout: None,- }- )- .unwrap(),- Disposition::Terminate- );- }-- #[test]- fn explicit_history_rejection_retries_only_eligible_first_candidate_once() {- let now = Instant::now();- let facts = RealtimeRoutingFacts::default();- let mut session = session(now, vec![candidate(1, &["b"]), candidate(2, &["a"])]);- let binding = session.candidates[0].binding;- session.reasoning_cleanup = Arc::new(super::super::ReasoningCleanup::new(- Some((binding, 3)),- None,- ));- let rejected = ProviderClassificationFacts {- error_class: Some(ObservationLabel::new("reasoning_history").unwrap()),- retryability: RetryabilityFact::NonRetryable,- http_status: Some(StatusCode::BAD_REQUEST),- ..ProviderClassificationFacts::default()- };- let transport = AttemptTransportFacts {- started_at: now,- upstream_protocol: None,- connect_elapsed: None,- request_write_elapsed: None,- upstream_ttfb: None,- last_upstream_progress_at: None,- local_read_suppressed: Duration::ZERO,- upstream_body_bytes: 0,- timeout: None,- };- let first = session- .select_next(selection_request(now, 1, 3, &facts))- .unwrap()- .unwrap();- assert_eq!(session.reasoning_cleanup.prefix_for(binding), None);- assert_eq!(- session.decide(&first, &rejected, &transport).unwrap(),- Disposition::Continue- );- let retry = session- .select_next(selection_request(now, 2, 2, &facts))- .unwrap()- .unwrap();- assert_eq!(retry.binding, binding);- assert_eq!(session.reasoning_cleanup.prefix_for(binding), Some(3));- assert_eq!(- session.decide(&retry, &rejected, &transport).unwrap(),- Disposition::Continue- );- let fallback = session- .select_next(selection_request(now, 3, 1, &facts))- .unwrap()- .unwrap();- assert_ne!(fallback.binding, binding);- assert_eq!(session.reasoning_cleanup.prefix_for(fallback.binding), None);- assert_eq!(- session.decide(&fallback, &rejected, &transport).unwrap(),- Disposition::Terminate- );- assert!(- session- .select_next(selection_request(now, 4, 0, &facts))- .unwrap()- .is_none()- );- }-- #[test]- fn history_error_without_context_break_authorization_does_not_enable_cleanup() {- let now = Instant::now();- let facts = RealtimeRoutingFacts::default();- let mut session = session(now, vec![candidate(1, &["a"]), candidate(2, &["b"])]);- let selected = session- .select_next(selection_request(now, 1, 3, &facts))- .unwrap()- .unwrap();- let mut failure = materialization_failure(now, "reasoning_history", None);- failure.provider.as_mut().unwrap().retryability = RetryabilityFact::NonRetryable;- assert!(!session.reasoning_cleanup.consume(selected.binding));- assert_eq!(session.reasoning_cleanup.prefix_for(selected.binding), None);- assert_eq!(- session.decide_failure(&selected, &failure).unwrap(),- Disposition::Terminate- );- }-- #[test]- fn unknown_or_nonretryable_materialization_never_advances_selection() {- let now = Instant::now();- let selected = SelectedGatewayAttempt {- request_id: RequestId(1),- attempt_id: AttemptId(0),- generation: AttemptGeneration(1),- binding: ResolvedTargetBindingId::new(PlanRevision(1), 1),- credential_ref: CredentialRef::new("credential").unwrap(),- route_decision_id: RouteDecisionId(1),- budget: AttemptBudgetGrant {- issued_at: now,- allocated: Duration::from_secs(1),- deadline: now + Duration::from_secs(1),- },- };- let mut session = ProductionDecisionSession {- request_id: RequestId(1),- route_decision_id: RouteDecisionId(1),- candidates: Arc::new([]),- candidate_index: 0,- credential_index: 0,- overall_deadline: now + Duration::from_secs(1),- hold: None,- switch_fallback: None,- reasoning_cleanup: Arc::new(super::super::ReasoningCleanup::new(None, None)),- temporary_block_seen: false,- probe_busy_seen: false,- earliest_retry_at: None,- non_temporary_block_seen: false,- upstream_attempted: false,- };- for retryability in [RetryabilityFact::Unknown, RetryabilityFact::NonRetryable] {- let failure = AttemptFailureFacts {- class: AttemptFailureClass::Materialization,- provider: Some(ProviderClassificationFacts {- retryability,- readiness: ObservationLabel::new("materialization_failed").unwrap(),- ..ProviderClassificationFacts::default()- }),- transport: AttemptTransportFacts {- started_at: now,- upstream_protocol: None,- connect_elapsed: None,- request_write_elapsed: None,- upstream_ttfb: None,- last_upstream_progress_at: None,- local_read_suppressed: Duration::ZERO,- upstream_body_bytes: 0,- timeout: None,- },- request_committed: false,- termination_reason: ObservationLabel::new("materialization").unwrap(),- };- assert_eq!(- session.decide_failure(&selected, &failure).unwrap(),- Disposition::Terminate- );- assert_eq!(session.candidate_index, 0);- assert_eq!(session.credential_index, 0);- }- }-- #[test]- fn switched_provisional_nonretryable_failure_does_not_call_previous_model() {- let now = Instant::now();- let realtime_facts = RealtimeRoutingFacts::default();- let mut session = session(- now,- vec![candidate(1, &["new-key"]), candidate(2, &["original-key"])],- );- session.switch_fallback = Some(ResolvedTargetBindingId::new(PlanRevision(1), 2));- let selected = session- .select_next(selection_request(now, 1, 2, &realtime_facts))- .unwrap()- .unwrap();- assert_eq!(selected.attempt_id, AttemptId(0));-- let mut failure = materialization_failure(now, "materialization_fail_closed", None);- failure.provider.as_mut().unwrap().retryability = RetryabilityFact::NonRetryable;- assert_eq!(- session.decide_failure(&selected, &failure).unwrap(),- Disposition::Terminate- );- assert_eq!(session.candidate_index, 0);- }-- #[test]- fn switched_promoted_preexchange_failure_does_not_call_previous_model() {- let now = Instant::now();- let realtime_facts = RealtimeRoutingFacts::default();- let mut session = session(- now,- vec![candidate(1, &["new-key"]), candidate(2, &["original-key"])],- );- session.switch_fallback = Some(ResolvedTargetBindingId::new(PlanRevision(1), 2));- let mut selected = session- .select_next(selection_request(now, 1, 2, &realtime_facts))- .unwrap()- .unwrap();- selected.attempt_id = AttemptId(1);-- let mut failure = materialization_failure(now, "preexchange_filter_failure", None);- failure.provider.as_mut().unwrap().retryability = RetryabilityFact::NonRetryable;- assert!(!failure.request_committed);- assert_eq!(- session.decide_failure(&selected, &failure).unwrap(),- Disposition::Terminate- );- assert_eq!(session.candidate_index, 0);- }-- #[test]- fn pure_runtime_state_exhaustion_returns_retryable_503_and_probe_busy_wins() {- let now = Instant::now();- let realtime = RealtimeRoutingFacts::default();- let mut session = session(- now,- vec![- candidate(1, &["credential-a"]),- candidate(2, &["credential-b"]),- ],- );- let first = session- .select_next(selection_request(now, 1, 3, &realtime))- .unwrap()- .unwrap();- assert_eq!(- session- .decide_failure(- &first,- &materialization_failure(- now,- "runtime_state_cooling_down",- Some(Duration::from_millis(1_500)),- ),- )- .unwrap(),- Disposition::Continue- );- let second = session- .select_next(selection_request(now, 1, 3, &realtime))- .unwrap()- .unwrap();- assert_eq!(- session- .decide_failure(- &second,- &materialization_failure(- now,- "runtime_state_probe_busy",- Some(Duration::from_secs(1)),- ),- )- .unwrap(),- Disposition::Continue- );- assert!(- session- .select_next(selection_request(now, 1, 3, &realtime))- .unwrap()- .is_none()- );- let reply = session- .exhaustion_local_reply(now)- .unwrap()- .expect("pure temporary exhaustion has a retry response");- assert_eq!(reply.status, StatusCode::SERVICE_UNAVAILABLE);- assert_eq!(reply.headers.get(RETRY_AFTER).unwrap(), "1");- assert_eq!(reply.headers.get(CONTENT_TYPE).unwrap(), "application/json");- assert_eq!(- serde_json::from_slice::<serde_json::Value>(&reply.body).unwrap(),- serde_json::json!({- "schema_version": "hiroute.gateway.error/v1",- "code": "RECOVERY_PROBE_IN_PROGRESS",- "phase": "runtime_state",- "retry_after_seconds": 1,- })- );- }-- #[test]- fn cooling_exhaustion_uses_earliest_deadline_and_rounds_up() {- assert_eq!(retry_after_seconds(Duration::from_nanos(1)), 1);- assert_eq!(retry_after_seconds(Duration::from_nanos(1_000_000_001)), 2);-- let now = Instant::now();- let realtime = RealtimeRoutingFacts::default();- let mut session = session(- now,- vec![- candidate(1, &["credential-a"]),- candidate(2, &["credential-b"]),- ],- );- let first = session- .select_next(selection_request(now, 1, 3, &realtime))- .unwrap()- .unwrap();- assert_eq!(- session- .decide_failure(- &first,- &materialization_failure(- now,- "runtime_state_cooling_down",- Some(Duration::from_secs(12)),- ),- )- .unwrap(),- Disposition::Continue- );- let second = session- .select_next(selection_request(now, 1, 3, &realtime))- .unwrap()- .unwrap();- assert_eq!(- session- .decide_failure(- &second,- &materialization_failure(- now,- "runtime_state_cooling_down",- Some(Duration::from_millis(3_200)),- ),- )- .unwrap(),- Disposition::Continue- );- assert!(- session- .select_next(selection_request(now, 1, 3, &realtime))- .unwrap()- .is_none()- );- let reply = session- .exhaustion_local_reply(now)- .unwrap()- .expect("pure cooldown exhaustion has a retry response");- assert_eq!(reply.headers.get(RETRY_AFTER).unwrap(), "4");- assert_eq!(- serde_json::from_slice::<serde_json::Value>(&reply.body).unwrap(),- serde_json::json!({- "schema_version": "hiroute.gateway.error/v1",- "code": "CANDIDATES_COOLING_DOWN",- "phase": "runtime_state",- "retry_after_seconds": 4,- })- );- }-- #[test]- fn mixed_runtime_state_exhaustion_keeps_generic_failure() {- let now = Instant::now();- let realtime = RealtimeRoutingFacts::default();- let mut session = session(- now,- vec![- candidate(1, &["credential-a"]),- candidate(2, &["credential-b"]),- ],- );- let first = session- .select_next(selection_request(now, 1, 3, &realtime))- .unwrap()- .unwrap();- session- .decide_failure(- &first,- &materialization_failure(- now,- "runtime_state_cooling_down",- Some(Duration::from_secs(2)),- ),- )- .unwrap();- let second = session- .select_next(selection_request(now, 1, 3, &realtime))- .unwrap()- .unwrap();- assert_eq!(- session- .decide_failure(- &second,- &materialization_failure(now, "runtime_state_disabled", None),- )- .unwrap(),- Disposition::Terminate- );- assert!(session.exhaustion_local_reply(now).unwrap().is_none());- }-- #[test]- fn hold_success_requires_full_accepted_eos_downstream_and_cleanup() {- let now = Instant::now();- let accepted = CompletedAttemptObservation {- route_decision_id: RouteDecisionId(1),- attempt_id: AttemptId(1),- generation: AttemptGeneration(1),- binding: ResolvedTargetBindingId::new(PlanRevision(1), 1),- credential_ref: CredentialRef::new("credential").unwrap(),- budget: AttemptBudgetGrant {- issued_at: now,- allocated: Duration::from_secs(1),- deadline: now + Duration::from_secs(1),- },- routing_facts: RealtimeRoutingFacts::default(),- provider: Some(ProviderClassificationFacts {- model_event: Some(ObservationLabel::new("response_complete").unwrap()),- ..ProviderClassificationFacts::default()- }),- failure: None,- transport: AttemptTransportFacts {- started_at: now,- upstream_protocol: None,- connect_elapsed: None,- request_write_elapsed: None,- upstream_ttfb: None,- last_upstream_progress_at: None,- local_read_suppressed: Duration::ZERO,- upstream_body_bytes: 1,- timeout: None,- },- disposition: Disposition::Accept,- commits: AttemptCommitFacts {- upstream_request: CommitFence::WriteConfirmed,- downstream_headers: CommitFence::WriteConfirmed,- downstream_semantic: CommitFence::WriteConfirmed,- },- stream: AttemptStreamOutcome::CompletedEos,- downstream: AttemptDownstreamOutcome::Completed,- cleanup: AttemptCleanupOutcome::Completed,- ended_at: now,- termination_reason: AttemptTerminationReason::AcceptedEos,- };- assert!(is_complete_accept(&accepted));- let mut interrupted = accepted.clone();- interrupted.stream = AttemptStreamOutcome::StreamStartedNoRetry;- assert!(!is_complete_accept(&interrupted));- let mut downstream_failed = accepted.clone();- downstream_failed.downstream = AttemptDownstreamOutcome::Failed;- assert!(!is_complete_accept(&downstream_failed));- let mut cleanup_failed = accepted.clone();- cleanup_failed.cleanup = AttemptCleanupOutcome::Failed;- assert!(!is_complete_accept(&cleanup_failed));- let mut incomplete = accepted.clone();- incomplete.provider.as_mut().unwrap().model_event =- Some(ObservationLabel::new("response_incomplete").unwrap());- assert!(!is_complete_accept(&incomplete));- let mut failed = accepted.clone();- failed.provider.as_mut().unwrap().model_event =- Some(ObservationLabel::new("response_failed").unwrap());- assert!(!is_complete_accept(&failed));- let mut missing_terminal = accepted.clone();- missing_terminal.provider.as_mut().unwrap().model_event = None;- assert!(!is_complete_accept(&missing_terminal));- let mut terminated = accepted;- terminated.disposition = Disposition::Terminate;- assert!(!is_complete_accept(&terminated));- }-}+mod tests;diff --git a/crates/gateway/src/runtime/driver/selection/preparation.rs b/crates/gateway/src/runtime/driver/selection/preparation.rsnew file mode 100644index 00000000..d97cd4de--- /dev/null+++ b/crates/gateway/src/runtime/driver/selection/preparation.rs@@ -0,0 +1,49 @@+use bytes::Bytes;+use hiroute_gateway_core::core::filter::LocalReply;+use hiroute_gateway_core::runtime::sse::SemanticProvenance;+use http::header::CONTENT_TYPE;+use http::{HeaderMap, HeaderValue, StatusCode};++/// Request-local summary of the closed preparation facts. Different causes cannot+/// be projected as if every candidate was rejected for one source or Key permission.+#[derive(Clone, Copy, Eq, PartialEq)]+pub(super) enum PreparationFailure {+ SourceDisabled,+ CredentialDisabled,+ CredentialUnavailable,+ Other,+}++impl PreparationFailure {+ pub(super) fn from_error_class(error_class: Option<&str>) -> Self {+ match error_class {+ Some("source_disabled") => Self::SourceDisabled,+ Some("credential_disabled") => Self::CredentialDisabled,+ Some("credential_exhausted") => Self::CredentialUnavailable,+ _ => Self::Other,+ }+ }++ pub(super) fn merge(self, next: Self) -> Self {+ if self == next { self } else { Self::Other }+ }++ pub(super) fn local_reply(self) -> LocalReply {+ let (code, phase) = match self {+ Self::SourceDisabled => ("SOURCE_DISABLED", "credential_lease"),+ Self::CredentialDisabled => ("CREDENTIAL_DISABLED", "credential_lease"),+ Self::CredentialUnavailable => ("CREDENTIAL_UNAVAILABLE", "credential_lease"),+ Self::Other => ("CANDIDATE_PREPARATION_FAILED", "candidate_preparation"),+ };+ let mut headers = HeaderMap::new();+ headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));+ LocalReply {+ status: StatusCode::BAD_GATEWAY,+ headers,+ body: Bytes::from(format!(+ "{{\"schema_version\":\"hiroute.gateway.error/v1\",\"code\":\"{code}\",\"phase\":\"{phase}\"}}"+ )),+ provenance: SemanticProvenance::NonSemantic,+ }+ }+}diff --git a/crates/gateway/src/runtime/driver/selection/tests.rs b/crates/gateway/src/runtime/driver/selection/tests.rsnew file mode 100644index 00000000..ffae44f3--- /dev/null+++ b/crates/gateway/src/runtime/driver/selection/tests.rs@@ -0,0 +1,774 @@+use std::time::Duration;++use hiroute_gateway_core::core::execution_plan::{+ CredentialRef, PlanRevision, ResolvedTargetBindingId,+};+use hiroute_gateway_core::runtime::attempt::{+ AttemptGeneration, AttemptTransportFacts, CommitFence, RequestId,+};+use hiroute_gateway_core::runtime::driver::{+ AttemptCommitFacts, AttemptFailureClass, AttemptFailureFacts,+};++use super::*;++fn candidate(binding_local_id: u32, credential_refs: &[&str]) -> DecisionCandidateAuthority {+ DecisionCandidateAuthority {+ binding: ResolvedTargetBindingId::new(PlanRevision(1), binding_local_id),+ stable_target: ObservationLabel::new(format!("binding-{binding_local_id}")).unwrap(),+ credential_refs: credential_refs+ .iter()+ .map(|value| CredentialRef::new(*value).unwrap())+ .collect::<Vec<_>>()+ .into(),+ candidate_id: None,+ profile_digest: None,+ reason_ledger_identity: None,+ provider_profile: None,+ }+}++fn selection_request<'a>(+ _now: Instant,+ generation: u64,+ remaining_attempts: u32,+ realtime_facts: &'a RealtimeRoutingFacts,+) -> SelectionRequest<'a> {+ SelectionRequest {+ request_id: RequestId(1),+ generation: AttemptGeneration(generation),+ route_binding: ResolvedTargetBindingId::new(PlanRevision(1), 1),+ remaining_total: Duration::from_secs(1),+ remaining_attempts,+ realtime_facts,+ previous_attempts: &[],+ }+}++fn session(now: Instant, candidates: Vec<DecisionCandidateAuthority>) -> ProductionDecisionSession {+ ProductionDecisionSession {+ request_id: RequestId(1),+ route_decision_id: RouteDecisionId(1),+ candidates: candidates.into(),+ candidate_index: 0,+ credential_index: 0,+ overall_deadline: now + Duration::from_secs(10),+ hold: None,+ switch_fallback: None,+ reasoning_cleanup: Arc::new(super::super::super::ReasoningCleanup::new(None, None)),+ temporary_block_seen: false,+ probe_busy_seen: false,+ earliest_retry_at: None,+ non_temporary_block_seen: false,+ upstream_attempted: false,+ preparation_failure: None,+ }+}++fn materialization_failure(+ now: Instant,+ error_class: &str,+ retry_after: Option<Duration>,+) -> AttemptFailureFacts {+ AttemptFailureFacts {+ class: AttemptFailureClass::Materialization,+ provider: Some(ProviderClassificationFacts {+ error_class: Some(ObservationLabel::new(error_class).unwrap()),+ retryability: RetryabilityFact::Retryable,+ retry_after,+ readiness: ObservationLabel::new("materialization_failed").unwrap(),+ ..ProviderClassificationFacts::default()+ }),+ transport: AttemptTransportFacts {+ started_at: now,+ upstream_protocol: None,+ connect_elapsed: None,+ request_write_elapsed: None,+ upstream_ttfb: None,+ last_upstream_progress_at: None,+ local_read_suppressed: Duration::ZERO,+ upstream_body_bytes: 0,+ timeout: None,+ },+ request_committed: false,+ termination_reason: ObservationLabel::new("materialization").unwrap(),+ }+}++#[test]+fn uniform_provisional_denials_retain_the_closed_code_without_retry_or_attempts() {+ for (class, code) in [+ ("source_disabled", "SOURCE_DISABLED"),+ ("credential_disabled", "CREDENTIAL_DISABLED"),+ ("credential_exhausted", "CREDENTIAL_UNAVAILABLE"),+ (+ "private_unrecognized_detail",+ "CANDIDATE_PREPARATION_FAILED",+ ),+ ] {+ let now = Instant::now();+ let realtime = RealtimeRoutingFacts::default();+ let mut session = session(+ now,+ vec![candidate(1, &["key-a"]), candidate(2, &["key-b"])],+ );+ for index in 0..2 {+ let selected = session+ .select_next(selection_request(now, 1, 3, &realtime))+ .unwrap()+ .unwrap();+ assert_eq!(selected.attempt_id, AttemptId(0));+ assert_eq!(+ session+ .decide_failure(&selected, &materialization_failure(now, class, None))+ .unwrap(),+ if index == 0 {+ Disposition::Continue+ } else {+ Disposition::Terminate+ }+ );+ }+ let reply = session.exhaustion_local_reply(now).unwrap().unwrap();+ assert_eq!(reply.status, StatusCode::BAD_GATEWAY);+ assert_eq!(reply.headers[CONTENT_TYPE], "application/json");+ assert!(!reply.headers.contains_key(RETRY_AFTER));+ assert_eq!(+ serde_json::from_slice::<serde_json::Value>(&reply.body).unwrap(),+ serde_json::json!({+ "schema_version": "hiroute.gateway.error/v1", "code": code,+ "phase": if code == "CANDIDATE_PREPARATION_FAILED" { "candidate_preparation" } else { "credential_lease" }+ })+ );+ }+}++#[test]+fn provisional_reply_does_not_override_a_previous_formal_attempt() {+ let now = Instant::now();+ let realtime = RealtimeRoutingFacts::default();+ let mut session = session(+ now,+ vec![candidate(1, &["key-a"]), candidate(2, &["key-b"])],+ );+ let mut first = session+ .select_next(selection_request(now, 1, 3, &realtime))+ .unwrap()+ .unwrap();+ first.attempt_id = AttemptId(1);+ assert_eq!(+ session+ .decide_failure(+ &first,+ &materialization_failure(now, "materialization_retryable", None)+ )+ .unwrap(),+ Disposition::Continue+ );+ let second = session+ .select_next(selection_request(now, 2, 2, &realtime))+ .unwrap()+ .unwrap();+ assert_eq!(+ session+ .decide_failure(+ &second,+ &materialization_failure(now, "source_disabled", None)+ )+ .unwrap(),+ Disposition::Terminate+ );+ assert!(session.exhaustion_local_reply(now).unwrap().is_none());+}++#[test]+fn retryable_quota_advances_exact_credential_before_binding() {+ let now = Instant::now();+ let realtime_facts = RealtimeRoutingFacts::default();+ let transport = AttemptTransportFacts {+ started_at: now,+ upstream_protocol: None,+ connect_elapsed: None,+ request_write_elapsed: None,+ upstream_ttfb: None,+ last_upstream_progress_at: None,+ local_read_suppressed: Duration::ZERO,+ upstream_body_bytes: 0,+ timeout: None,+ };+ let mut session = ProductionDecisionSession {+ request_id: RequestId(1),+ route_decision_id: RouteDecisionId(1),+ candidates: vec![+ candidate(1, &["credential-a", "credential-b"]),+ candidate(2, &["credential-c"]),+ ]+ .into(),+ candidate_index: 0,+ credential_index: 0,+ overall_deadline: now + Duration::from_secs(1),+ hold: None,+ switch_fallback: None,+ reasoning_cleanup: Arc::new(super::super::super::ReasoningCleanup::new(None, None)),+ temporary_block_seen: false,+ probe_busy_seen: false,+ earliest_retry_at: None,+ non_temporary_block_seen: false,+ upstream_attempted: false,+ preparation_failure: None,+ };+ let first = session+ .select_next(selection_request(now, 1, 3, &realtime_facts))+ .unwrap()+ .unwrap();+ assert_eq!(first.binding.local_id(), 1);+ assert_eq!(first.credential_ref.as_str(), "credential-a");++ let quota = ProviderClassificationFacts {+ error_class: Some(ObservationLabel::new("quota").unwrap()),+ retryability: RetryabilityFact::Retryable,+ readiness: ObservationLabel::new("rejected").unwrap(),+ ..ProviderClassificationFacts::default()+ };+ assert_eq!(+ session.decide(&first, "a, &transport).unwrap(),+ Disposition::Continue+ );+ let second = session+ .select_next(selection_request(now, 2, 2, &realtime_facts))+ .unwrap()+ .unwrap();+ assert_eq!(second.binding.local_id(), 1);+ assert_eq!(second.credential_ref.as_str(), "credential-b");++ assert_eq!(+ session.decide(&second, "a, &transport).unwrap(),+ Disposition::Continue+ );+ let third = session+ .select_next(selection_request(now, 3, 1, &realtime_facts))+ .unwrap()+ .unwrap();+ assert_eq!(third.binding.local_id(), 2);+ assert_eq!(third.credential_ref.as_str(), "credential-c");+}++#[test]+fn switched_candidate_precommit_400_falls_back_to_previous_once() {+ let now = Instant::now();+ let realtime_facts = RealtimeRoutingFacts::default();+ let mut session = session(+ now,+ vec![candidate(1, &["new-key"]), candidate(2, &["original-key"])],+ );+ session.switch_fallback = Some(ResolvedTargetBindingId::new(PlanRevision(1), 2));+ let mut selected = session+ .select_next(selection_request(now, 1, 2, &realtime_facts))+ .unwrap()+ .unwrap();+ // The core promotes the candidate after pre-exchange materialization.+ selected.attempt_id = AttemptId(1);+ let rejected = ProviderClassificationFacts {+ error_class: Some(ObservationLabel::new("permanent_client").unwrap()),+ retryability: RetryabilityFact::NonRetryable,+ http_status: Some(StatusCode::BAD_REQUEST),+ ..ProviderClassificationFacts::default()+ };+ assert_eq!(+ session+ .decide(+ &selected,+ &rejected,+ &AttemptTransportFacts {+ started_at: now,+ upstream_protocol: None,+ connect_elapsed: None,+ request_write_elapsed: None,+ upstream_ttfb: None,+ last_upstream_progress_at: None,+ local_read_suppressed: Duration::ZERO,+ upstream_body_bytes: 0,+ timeout: None,+ }+ )+ .unwrap(),+ Disposition::Continue+ );+ let original = session+ .select_next(selection_request(now, 2, 1, &realtime_facts))+ .unwrap()+ .unwrap();+ assert_eq!(original.binding.local_id(), 2);+ assert_eq!(+ session+ .decide(+ &original,+ &rejected,+ &AttemptTransportFacts {+ started_at: now,+ upstream_protocol: None,+ connect_elapsed: None,+ request_write_elapsed: None,+ upstream_ttfb: None,+ last_upstream_progress_at: None,+ local_read_suppressed: Duration::ZERO,+ upstream_body_bytes: 0,+ timeout: None,+ }+ )+ .unwrap(),+ Disposition::Terminate+ );+}++#[test]+fn explicit_history_rejection_retries_only_eligible_first_candidate_once() {+ let now = Instant::now();+ let facts = RealtimeRoutingFacts::default();+ let mut session = session(now, vec![candidate(1, &["b"]), candidate(2, &["a"])]);+ let binding = session.candidates[0].binding;+ session.reasoning_cleanup = Arc::new(super::super::ReasoningCleanup::new(+ Some((binding, 3)),+ None,+ ));+ let rejected = ProviderClassificationFacts {+ error_class: Some(ObservationLabel::new("reasoning_history").unwrap()),+ retryability: RetryabilityFact::NonRetryable,+ http_status: Some(StatusCode::BAD_REQUEST),+ ..ProviderClassificationFacts::default()+ };+ let transport = AttemptTransportFacts {+ started_at: now,+ upstream_protocol: None,+ connect_elapsed: None,+ request_write_elapsed: None,+ upstream_ttfb: None,+ last_upstream_progress_at: None,+ local_read_suppressed: Duration::ZERO,+ upstream_body_bytes: 0,+ timeout: None,+ };+ let first = session+ .select_next(selection_request(now, 1, 3, &facts))+ .unwrap()+ .unwrap();+ assert_eq!(session.reasoning_cleanup.prefix_for(binding), None);+ assert_eq!(+ session.decide(&first, &rejected, &transport).unwrap(),+ Disposition::Continue+ );+ let retry = session+ .select_next(selection_request(now, 2, 2, &facts))+ .unwrap()+ .unwrap();+ assert_eq!(retry.binding, binding);+ assert_eq!(session.reasoning_cleanup.prefix_for(binding), Some(3));+ assert_eq!(+ session.decide(&retry, &rejected, &transport).unwrap(),+ Disposition::Continue+ );+ let fallback = session+ .select_next(selection_request(now, 3, 1, &facts))+ .unwrap()+ .unwrap();+ assert_ne!(fallback.binding, binding);+ assert_eq!(session.reasoning_cleanup.prefix_for(fallback.binding), None);+ assert_eq!(+ session.decide(&fallback, &rejected, &transport).unwrap(),+ Disposition::Terminate+ );+ assert!(+ session+ .select_next(selection_request(now, 4, 0, &facts))+ .unwrap()+ .is_none()+ );+}++#[test]+fn history_error_without_context_break_authorization_does_not_enable_cleanup() {+ let now = Instant::now();+ let facts = RealtimeRoutingFacts::default();+ let mut session = session(now, vec![candidate(1, &["a"]), candidate(2, &["b"])]);+ let selected = session+ .select_next(selection_request(now, 1, 3, &facts))+ .unwrap()+ .unwrap();+ let mut failure = materialization_failure(now, "reasoning_history", None);+ failure.provider.as_mut().unwrap().retryability = RetryabilityFact::NonRetryable;+ assert!(!session.reasoning_cleanup.consume(selected.binding));+ assert_eq!(session.reasoning_cleanup.prefix_for(selected.binding), None);+ assert_eq!(+ session.decide_failure(&selected, &failure).unwrap(),+ Disposition::Terminate+ );+}++#[test]+fn unknown_or_nonretryable_materialization_never_advances_selection() {+ let now = Instant::now();+ let selected = SelectedGatewayAttempt {+ request_id: RequestId(1),+ attempt_id: AttemptId(0),+ generation: AttemptGeneration(1),+ binding: ResolvedTargetBindingId::new(PlanRevision(1), 1),+ credential_ref: CredentialRef::new("credential").unwrap(),+ route_decision_id: RouteDecisionId(1),+ budget: AttemptBudgetGrant {+ issued_at: now,+ allocated: Duration::from_secs(1),+ deadline: now + Duration::from_secs(1),+ },+ };+ let mut session = ProductionDecisionSession {+ request_id: RequestId(1),+ route_decision_id: RouteDecisionId(1),+ candidates: Arc::new([]),+ candidate_index: 0,+ credential_index: 0,+ overall_deadline: now + Duration::from_secs(1),+ hold: None,+ switch_fallback: None,+ reasoning_cleanup: Arc::new(super::super::super::ReasoningCleanup::new(None, None)),+ temporary_block_seen: false,+ probe_busy_seen: false,+ earliest_retry_at: None,+ non_temporary_block_seen: false,+ upstream_attempted: false,+ preparation_failure: None,+ };+ for retryability in [RetryabilityFact::Unknown, RetryabilityFact::NonRetryable] {+ let failure = AttemptFailureFacts {+ class: AttemptFailureClass::Materialization,+ provider: Some(ProviderClassificationFacts {+ retryability,+ readiness: ObservationLabel::new("materialization_failed").unwrap(),+ ..ProviderClassificationFacts::default()+ }),+ transport: AttemptTransportFacts {+ started_at: now,+ upstream_protocol: None,+ connect_elapsed: None,+ request_write_elapsed: None,+ upstream_ttfb: None,+ last_upstream_progress_at: None,+ local_read_suppressed: Duration::ZERO,+ upstream_body_bytes: 0,+ timeout: None,+ },+ request_committed: false,+ termination_reason: ObservationLabel::new("materialization").unwrap(),+ };+ assert_eq!(+ session.decide_failure(&selected, &failure).unwrap(),+ Disposition::Terminate+ );+ assert_eq!(session.candidate_index, 0);+ assert_eq!(session.credential_index, 0);+ }+}++#[test]+fn switched_provisional_nonretryable_failure_does_not_call_previous_model() {+ let now = Instant::now();+ let realtime_facts = RealtimeRoutingFacts::default();+ let mut session = session(+ now,+ vec![candidate(1, &["new-key"]), candidate(2, &["original-key"])],+ );+ session.switch_fallback = Some(ResolvedTargetBindingId::new(PlanRevision(1), 2));+ let selected = session+ .select_next(selection_request(now, 1, 2, &realtime_facts))+ .unwrap()+ .unwrap();+ assert_eq!(selected.attempt_id, AttemptId(0));++ let mut failure = materialization_failure(now, "materialization_fail_closed", None);+ failure.provider.as_mut().unwrap().retryability = RetryabilityFact::NonRetryable;+ assert_eq!(+ session.decide_failure(&selected, &failure).unwrap(),+ Disposition::Terminate+ );+ assert_eq!(session.candidate_index, 0);+}++#[test]+fn switched_promoted_preexchange_failure_does_not_call_previous_model() {+ let now = Instant::now();+ let realtime_facts = RealtimeRoutingFacts::default();+ let mut session = session(+ now,+ vec![candidate(1, &["new-key"]), candidate(2, &["original-key"])],+ );+ session.switch_fallback = Some(ResolvedTargetBindingId::new(PlanRevision(1), 2));+ let mut selected = session+ .select_next(selection_request(now, 1, 2, &realtime_facts))+ .unwrap()+ .unwrap();+ selected.attempt_id = AttemptId(1);++ let mut failure = materialization_failure(now, "preexchange_filter_failure", None);+ failure.provider.as_mut().unwrap().retryability = RetryabilityFact::NonRetryable;+ assert!(!failure.request_committed);+ assert_eq!(+ session.decide_failure(&selected, &failure).unwrap(),+ Disposition::Terminate+ );+ assert_eq!(session.candidate_index, 0);+}++#[test]+fn pure_runtime_state_exhaustion_returns_retryable_503_and_probe_busy_wins() {+ let now = Instant::now();+ let realtime = RealtimeRoutingFacts::default();+ let mut session = session(+ now,+ vec![+ candidate(1, &["credential-a"]),+ candidate(2, &["credential-b"]),+ ],+ );+ let first = session+ .select_next(selection_request(now, 1, 3, &realtime))+ .unwrap()+ .unwrap();+ assert_eq!(+ session+ .decide_failure(+ &first,+ &materialization_failure(+ now,+ "runtime_state_cooling_down",+ Some(Duration::from_millis(1_500)),+ ),+ )+ .unwrap(),+ Disposition::Continue+ );+ let second = session+ .select_next(selection_request(now, 1, 3, &realtime))+ .unwrap()+ .unwrap();+ assert_eq!(+ session+ .decide_failure(+ &second,+ &materialization_failure(+ now,+ "runtime_state_probe_busy",+ Some(Duration::from_secs(1)),+ ),+ )+ .unwrap(),+ Disposition::Continue+ );+ assert!(+ session+ .select_next(selection_request(now, 1, 3, &realtime))+ .unwrap()+ .is_none()+ );+ let reply = session+ .exhaustion_local_reply(now)+ .unwrap()+ .expect("pure temporary exhaustion has a retry response");+ assert_eq!(reply.status, StatusCode::SERVICE_UNAVAILABLE);+ assert_eq!(reply.headers.get(RETRY_AFTER).unwrap(), "1");+ assert_eq!(reply.headers.get(CONTENT_TYPE).unwrap(), "application/json");+ assert_eq!(+ serde_json::from_slice::<serde_json::Value>(&reply.body).unwrap(),+ serde_json::json!({+ "schema_version": "hiroute.gateway.error/v1",+ "code": "RECOVERY_PROBE_IN_PROGRESS",+ "phase": "runtime_state",+ "retry_after_seconds": 1,+ })+ );+}++#[test]+fn cooling_exhaustion_uses_earliest_deadline_and_rounds_up() {+ assert_eq!(retry_after_seconds(Duration::from_nanos(1)), 1);+ assert_eq!(retry_after_seconds(Duration::from_nanos(1_000_000_001)), 2);++ let now = Instant::now();+ let realtime = RealtimeRoutingFacts::default();+ let mut session = session(+ now,+ vec![+ candidate(1, &["credential-a"]),+ candidate(2, &["credential-b"]),+ ],+ );+ let first = session+ .select_next(selection_request(now, 1, 3, &realtime))+ .unwrap()+ .unwrap();+ assert_eq!(+ session+ .decide_failure(+ &first,+ &materialization_failure(+ now,+ "runtime_state_cooling_down",+ Some(Duration::from_secs(12)),+ ),+ )+ .unwrap(),+ Disposition::Continue+ );+ let second = session+ .select_next(selection_request(now, 1, 3, &realtime))+ .unwrap()+ .unwrap();+ assert_eq!(+ session+ .decide_failure(+ &second,+ &materialization_failure(+ now,+ "runtime_state_cooling_down",+ Some(Duration::from_millis(3_200)),+ ),+ )+ .unwrap(),+ Disposition::Continue+ );+ assert!(+ session+ .select_next(selection_request(now, 1, 3, &realtime))+ .unwrap()+ .is_none()+ );+ let reply = session+ .exhaustion_local_reply(now)+ .unwrap()+ .expect("pure cooldown exhaustion has a retry response");+ assert_eq!(reply.headers.get(RETRY_AFTER).unwrap(), "4");+ assert_eq!(+ serde_json::from_slice::<serde_json::Value>(&reply.body).unwrap(),+ serde_json::json!({+ "schema_version": "hiroute.gateway.error/v1",+ "code": "CANDIDATES_COOLING_DOWN",+ "phase": "runtime_state",+ "retry_after_seconds": 4,+ })+ );+}++#[test]+fn mixed_runtime_state_exhaustion_returns_a_generic_typed_preparation_failure() {+ let now = Instant::now();+ let realtime = RealtimeRoutingFacts::default();+ let mut session = session(+ now,+ vec![+ candidate(1, &["credential-a"]),+ candidate(2, &["credential-b"]),+ ],+ );+ let first = session+ .select_next(selection_request(now, 1, 3, &realtime))+ .unwrap()+ .unwrap();+ session+ .decide_failure(+ &first,+ &materialization_failure(+ now,+ "runtime_state_cooling_down",+ Some(Duration::from_secs(2)),+ ),+ )+ .unwrap();+ let second = session+ .select_next(selection_request(now, 1, 3, &realtime))+ .unwrap()+ .unwrap();+ assert_eq!(+ session+ .decide_failure(+ &second,+ &materialization_failure(now, "runtime_state_disabled", None),+ )+ .unwrap(),+ Disposition::Terminate+ );+ let reply = session.exhaustion_local_reply(now).unwrap().unwrap();+ assert_eq!(reply.status, StatusCode::BAD_GATEWAY);+ assert!(!reply.headers.contains_key(RETRY_AFTER));+ assert_eq!(+ serde_json::from_slice::<serde_json::Value>(&reply.body).unwrap(),+ serde_json::json!({"schema_version": "hiroute.gateway.error/v1",+ "code": "CANDIDATE_PREPARATION_FAILED", "phase": "candidate_preparation"})+ );+}++#[test]+fn hold_success_requires_full_accepted_eos_downstream_and_cleanup() {+ let now = Instant::now();+ let accepted = CompletedAttemptObservation {+ route_decision_id: RouteDecisionId(1),+ attempt_id: AttemptId(1),+ generation: AttemptGeneration(1),+ binding: ResolvedTargetBindingId::new(PlanRevision(1), 1),+ credential_ref: CredentialRef::new("credential").unwrap(),+ budget: AttemptBudgetGrant {+ issued_at: now,+ allocated: Duration::from_secs(1),+ deadline: now + Duration::from_secs(1),+ },+ routing_facts: RealtimeRoutingFacts::default(),+ provider: Some(ProviderClassificationFacts {+ model_event: Some(ObservationLabel::new("response_complete").unwrap()),+ ..ProviderClassificationFacts::default()+ }),+ failure: None,+ transport: AttemptTransportFacts {+ started_at: now,+ upstream_protocol: None,+ connect_elapsed: None,+ request_write_elapsed: None,+ upstream_ttfb: None,+ last_upstream_progress_at: None,+ local_read_suppressed: Duration::ZERO,+ upstream_body_bytes: 1,+ timeout: None,+ },+ disposition: Disposition::Accept,+ commits: AttemptCommitFacts {+ upstream_request: CommitFence::WriteConfirmed,+ downstream_headers: CommitFence::WriteConfirmed,+ downstream_semantic: CommitFence::WriteConfirmed,+ },+ stream: AttemptStreamOutcome::CompletedEos,+ downstream: AttemptDownstreamOutcome::Completed,+ cleanup: AttemptCleanupOutcome::Completed,+ ended_at: now,+ termination_reason: AttemptTerminationReason::AcceptedEos,+ };+ assert!(is_complete_accept(&accepted));+ let mut interrupted = accepted.clone();+ interrupted.stream = AttemptStreamOutcome::StreamStartedNoRetry;+ assert!(!is_complete_accept(&interrupted));+ let mut downstream_failed = accepted.clone();+ downstream_failed.downstream = AttemptDownstreamOutcome::Failed;+ assert!(!is_complete_accept(&downstream_failed));+ let mut cleanup_failed = accepted.clone();+ cleanup_failed.cleanup = AttemptCleanupOutcome::Failed;+ assert!(!is_complete_accept(&cleanup_failed));+ let mut incomplete = accepted.clone();+ incomplete.provider.as_mut().unwrap().model_event =+ Some(ObservationLabel::new("response_incomplete").unwrap());+ assert!(!is_complete_accept(&incomplete));+ let mut failed = accepted.clone();+ failed.provider.as_mut().unwrap().model_event =+ Some(ObservationLabel::new("response_failed").unwrap());+ assert!(!is_complete_accept(&failed));+ let mut missing_terminal = accepted.clone();+ missing_terminal.provider.as_mut().unwrap().model_event = None;+ assert!(!is_complete_accept(&missing_terminal));+ let mut terminated = accepted;+ terminated.disposition = Disposition::Terminate;+ assert!(!is_complete_accept(&terminated));+}diff --git a/crates/local-storage/src/control/compute/management/tests/runtime_leases.rs b/crates/local-storage/src/control/compute/management/tests/runtime_leases.rsindex 66ba7a0a..6d3099be 100644--- a/crates/local-storage/src/control/compute/management/tests/runtime_leases.rs+++ b/crates/local-storage/src/control/compute/management/tests/runtime_leases.rs@@ -2,7 +2,7 @@ use super::*;use hiroute_domain::{ComputeManagementRepositoryPort, ComputeManagementSourceV2, ConnectorRuntimeKind,GatewayOperationalTargetV1, NATIVE_CREDENTIAL_LEASE_REQUEST_SCHEMA_V1,- NativeCredentialLeaseRequestV1,+ NativeCredentialAccessDeniedV1, NativeCredentialLeaseErrorV1, NativeCredentialLeaseRequestV1,};fn save(@@ -147,10 +147,10 @@ fn frozen_native_requests_obey_current_source_and_key_switches_after_restart() {);for request in &requests[..2] {assert!(- stores- .lease_native_credential_exact(request)- .unwrap()- .is_none(),+ stores.lease_native_credential_exact(request).err()+ == Some(NativeCredentialLeaseErrorV1::Denied(+ NativeCredentialAccessDeniedV1::CredentialDisabled+ )),"the same disabled key must be unavailable to both frozen model bindings");assert!(@@ -178,20 +178,20 @@ fn frozen_native_requests_obey_current_source_and_key_switches_after_restart() {);for request in &requests {assert!(- stores- .lease_native_credential_exact(request)- .unwrap()- .is_none()+ stores.lease_native_credential_exact(request).err()+ == Some(NativeCredentialLeaseErrorV1::Denied(+ NativeCredentialAccessDeniedV1::SourceDisabled+ )));}drop(stores);let stores = LocalStorageSet::open_for_daemon_startup(root.path()).unwrap();for request in &requests {assert!(- stores- .lease_native_credential_exact(request)- .unwrap()- .is_none()+ stores.lease_native_credential_exact(request).err()+ == Some(NativeCredentialLeaseErrorV1::Denied(+ NativeCredentialAccessDeniedV1::SourceDisabled+ )));}save(@@ -203,10 +203,10 @@ fn frozen_native_requests_obey_current_source_and_key_switches_after_restart() {"lease-reenable-source",);assert!(- stores- .lease_native_credential_exact(&requests[0])- .unwrap()- .is_none()+ stores.lease_native_credential_exact(&requests[0]).err()+ == Some(NativeCredentialLeaseErrorV1::Denied(+ NativeCredentialAccessDeniedV1::CredentialDisabled+ )));assert!(stores@@ -224,10 +224,10 @@ fn frozen_native_requests_obey_current_source_and_key_switches_after_restart() {);for request in &requests {assert!(- stores- .lease_native_credential_exact(request)- .unwrap()- .is_none()+ stores.lease_native_credential_exact(request).err()+ == Some(NativeCredentialLeaseErrorV1::Denied(+ NativeCredentialAccessDeniedV1::SourceDisabled+ )));}save(diff --git a/crates/local-storage/src/secrets/runtime.rs b/crates/local-storage/src/secrets/runtime.rsindex fdc6727d..b9455703 100644--- a/crates/local-storage/src/secrets/runtime.rs+++ b/crates/local-storage/src/secrets/runtime.rs@@ -3,8 +3,9 @@ use std::sync::Mutex;use hiroute_domain::{ComputeManagementRepositoryPort, CredentialRefV1, GatewayAuthenticationSemanticsV1,- HeaderSecretLeaseRequestV1, MaterializationState, NativeCredentialAuthorityV1,- NativeCredentialAuthorizationCapabilityV1, NativeCredentialCapabilityErrorV1,+ HeaderSecretLeaseRequestV1, MaterializationState, NativeCredentialAccessDeniedV1,+ NativeCredentialAuthorityV1, NativeCredentialAuthorizationCapabilityV1,+ NativeCredentialCapabilityErrorV1, NativeCredentialLeaseErrorV1,NativeCredentialLeaseRequestV1, NativeCredentialLeaseV1, PortErrorCode, PortResult,ProtectedSecret, SensitiveAuthorizationTargetV1,};@@ -19,7 +20,7 @@ impl crate::LocalStorageSet {pub fn lease_native_credential_exact(&self,request: &NativeCredentialLeaseRequestV1,- ) -> PortResult<Option<NativeCredentialLeaseV1>> {+ ) -> Result<Option<NativeCredentialLeaseV1>, NativeCredentialLeaseErrorV1> {self.secrets().lease_native_credential_with_guard(request, |reference| {// Only the current management producer owns this identity namespace.@@ -30,13 +31,30 @@ impl crate::LocalStorageSet {let Some(source_id) = reference.owner_scope().strip_prefix("source/") else {return Ok(false);};- let source = self.control().compute_management_source(source_id)?;- Ok(source.is_some_and(|source| {- source.state == MaterializationState::Ready- && source.credentials.iter().any(|credential| {- credential.enabled && credential.credential == *reference- })- }))+ let Some(source) = self.control().compute_management_source(source_id)? else {+ return Ok(false);+ };+ if source.state == MaterializationState::Disabled {+ return Err(NativeCredentialLeaseErrorV1::Denied(+ NativeCredentialAccessDeniedV1::SourceDisabled,+ ));+ }+ if source.state != MaterializationState::Ready {+ return Ok(false);+ }+ let Some(credential) = source+ .credentials+ .iter()+ .find(|credential| credential.credential == *reference)+ else {+ return Ok(false);+ };+ if !credential.enabled {+ return Err(NativeCredentialLeaseErrorV1::Denied(+ NativeCredentialAccessDeniedV1::CredentialDisabled,+ ));+ }+ Ok(true)})}}@@ -64,15 +82,15 @@ impl LocalSecretStore {pub fn lease_native_credential_exact(&self,request: &NativeCredentialLeaseRequestV1,- ) -> PortResult<Option<NativeCredentialLeaseV1>> {+ ) -> Result<Option<NativeCredentialLeaseV1>, NativeCredentialLeaseErrorV1> {self.lease_native_credential_with_guard(request, |_| Ok(true))}fn lease_native_credential_with_guard(&self,request: &NativeCredentialLeaseRequestV1,- guard: impl FnOnce(&CredentialRefV1) -> PortResult<bool>,- ) -> PortResult<Option<NativeCredentialLeaseV1>> {+ guard: impl FnOnce(&CredentialRefV1) -> Result<bool, NativeCredentialLeaseErrorV1>,+ ) -> Result<Option<NativeCredentialLeaseV1>, NativeCredentialLeaseErrorV1> {request.validate().map_err(|_| {port(PortErrorCode::InvalidData,@@ -106,7 +124,8 @@ impl LocalSecretStore {return Err(port(PortErrorCode::PermissionDenied,"secret.runtime.destination_binding",- ));+ )+ .into());}if request.excluded_key_ids@@ -129,7 +148,7 @@ impl LocalSecretStore {});NativeCredentialLeaseV1::issue(credential_id.clone(), credential_id, generation, capability).map(Some)- .map_err(|_| port(PortErrorCode::Corrupt, "secret.runtime.lease"))+ .map_err(|_| port(PortErrorCode::Corrupt, "secret.runtime.lease").into())}pub fn lease_header_secret_exact(@@ -201,7 +220,7 @@ impl NativeCredentialAuthorityV1 for LocalNativeCredentialAuthority {fn lease_native_credential(&self,request: &NativeCredentialLeaseRequestV1,- ) -> PortResult<Option<NativeCredentialLeaseV1>> {+ ) -> Result<Option<NativeCredentialLeaseV1>, NativeCredentialLeaseErrorV1> {self.store.lock().map_err(|_| port(PortErrorCode::Unavailable, "secret.runtime.authority_lock"))?diff --git a/crates/local-storage/src/secrets/runtime_tests.rs b/crates/local-storage/src/secrets/runtime_tests.rsindex 9e3b2aed..ec0915a9 100644--- a/crates/local-storage/src/secrets/runtime_tests.rs+++ b/crates/local-storage/src/secrets/runtime_tests.rs@@ -4,7 +4,7 @@ use hiroute_domain::{GatewayOperationalTargetV1, HEADER_SECRET_LEASE_REQUEST_SCHEMA_V1, HeaderSecretLeaseRequestV1,NATIVE_CREDENTIAL_LEASE_REQUEST_SCHEMA_V1, NativeCredentialAuthorityV1,NativeCredentialCapabilityErrorV1, NativeCredentialLeaseRequestV1, NativeCredentialLeaseV1,- OperationId, PortErrorCode, PortResult, ProtectedSecret, SecretMutationV1, SecretStorePort,+ OperationId, PortErrorCode, ProtectedSecret, SecretMutationV1, SecretStorePort,SensitiveAuthorizationTargetV1, UpstreamProtocol,};@@ -267,9 +267,16 @@ fn frozen_native_request_rejects_duplicate_exclusions_and_non_bearer_auth() {assert!(auth.validate().is_err());}-fn lease_error_code(result: PortResult<Option<NativeCredentialLeaseV1>>) -> PortErrorCode {+fn lease_error_code<E: Into<hiroute_domain::NativeCredentialLeaseErrorV1>>(+ result: Result<Option<NativeCredentialLeaseV1>, E>,+) -> PortErrorCode {match result {- Err(error) => error.code,+ Err(error) => match error.into() {+ hiroute_domain::NativeCredentialLeaseErrorV1::Authority(error) => error.code,+ hiroute_domain::NativeCredentialLeaseErrorV1::Denied(_) => {+ PortErrorCode::PermissionDenied+ }+ },Ok(_) => panic!("expected credential authority rejection"),}}diff --git a/docs/code-map/architecture.md b/docs/code-map/architecture.mdindex 598511d1..8f04352e 100644--- a/docs/code-map/architecture.md+++ b/docs/code-map/architecture.md@@ -92,6 +92,7 @@ Scan → Prepare → Save → restart → native use, including stale-source rej| Capture completeness | [Content completeness](../../crates/observation/src/content/completeness.rs), [session query](../../crates/observation/src/query_v2/sessions.rs) | Aggregate every required request/direction; a finished stream or readable preview does not prove the session is complete || Streaming memory | [Capture](../../crates/gateway/src/observation/provider/capture.rs), [pending delivery](../../crates/gateway/src/observation/provider/capture/tracker.rs) | Decoder, projector and delivery tracker share the stream budget; queue capacity is not capture evidence || Upstream fault attribution | [Wire projection](../../crates/gateway/src/observation/provider/wire_diagnostic.rs), [attempt association](../../crates/gateway/src/observation/request/diagnostic.rs), [decision calls](../../crates/gateway/src/core_runtime/classification/diagnostic.rs) | Actual bounded model/control fields, negotiated HTTP protocol, closed error codes, hashed request IDs and separate client headers/body commit facts must preserve the real outcome without prompts or provider error text |+| Source disabled before an upstream attempt | [Exact permission gate](../../crates/local-storage/src/secrets/runtime.rs), [product preparation reply](../../crates/gateway/src/runtime/driver/selection/preparation.rs), [Gateway error guide](../../crates/gateway/README.md#candidate-preparation-errors) | Preserve the closed lease denial in the request fact and local JSON response; a frozen route does not grant current source/Key permission, and rejected preparation remains zero attempts || Inspect CPA authentication recovery | [Private response metadata](../../crates/gateway/src/observation/provider/cpa_execution.rs), [pinned CPA adapter](../../vendor/cpa/README.md) | Logical attempts and actual HTTP sends differ; only verified CPA targets can supply recovery counters. Missing evidence remains unknown and does not create token usage. || Search and gaps | [Query](../../crates/observation/src/query_v2/search.rs), [index worker](../../crates/observation/src/text_index/worker.rs), [status](../../crates/observation/src/query/status.rs) | Bounded scan windows, wraparound and explicit gaps distinguish incomplete search from an empty result || Business relations and visibility | [Relation owner](../../crates/observation/src/query_v2/relation.rs), [managed content](../../crates/observation/src/managed_text/README.md) | `query_v2` also writes relations; every content read checks current authorization and visibility |diff --git a/tools/e2e-harness/tests/p0_gateway_observation.rs b/tools/e2e-harness/tests/p0_gateway_observation.rsindex f12c1806..1f761c3c 100644--- a/tools/e2e-harness/tests/p0_gateway_observation.rs+++ b/tools/e2e-harness/tests/p0_gateway_observation.rs@@ -17,6 +17,35 @@ const ACCEPTED_STREAM_CREATED: &[u8] = b"event: response.created\ndata: {\"type\const ACCEPTED_STREAM_TEXT: &[u8] = b"event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"item_id\":\"observation-message\",\"output_index\":0,\"content_index\":0,\"delta\":\"accepted-stream-22008\"}\n\n";const ACCEPTED_STREAM_COMPLETED: &[u8] = b"event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"observation-stream\",\"model\":\"native-observed\",\"status\":\"completed\",\"output\":[{\"type\":\"message\",\"id\":\"observation-message\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"accepted-stream-22008\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":13,\"output_tokens\":5,\"total_tokens\":18,\"input_tokens_details\":{\"cached_tokens\":4},\"output_tokens_details\":{\"reasoning_tokens\":3}}}}\n\n";+#[test]+fn observation_reader_waits_for_complete_rows_and_rejects_closed_corruption() {+ use std::io::Write;++ let root = tempfile::tempdir().unwrap();+ let path = root.path().join("records.jsonl");+ let first = serde_json::json!({"phase": "begin"});+ let second = serde_json::json!({"phase": "finish", "text": "尾帧"});+ let mut prefix = serde_json::to_vec(&first).unwrap();+ prefix.push(b'\n');+ let second_bytes = serde_json::to_vec(&second).unwrap();+ let partial_utf8 = second_bytes.iter().position(|byte| *byte >= 0x80).unwrap() + 1;+ for end in [9, partial_utf8, second_bytes.len()] {+ let mut snapshot = prefix.clone();+ snapshot.extend_from_slice(&second_bytes[..end]);+ fs::write(&path, snapshot).unwrap();+ assert_eq!(read_records(&path).as_slice(), std::slice::from_ref(&first));+ let mut writer = fs::OpenOptions::new().append(true).open(&path).unwrap();+ writer.write_all(&second_bytes[end..]).unwrap();+ writer.write_all(b"\n").unwrap();+ writer.flush().unwrap();+ assert_eq!(read_records(&path), [first.clone(), second.clone()]);+ }+ for corruption in [b"invalid-json\n".as_slice(), &[0xff, b'\n']] {+ fs::write(&path, corruption).unwrap();+ assert!(std::panic::catch_unwind(|| read_records(&path)).is_err());+ }+}+#[test]fn production_info_cross_group_relay_links_only_actual_attempts() {for relay_allowed in [true, false] {@@ -314,11 +343,9 @@ fn production_unknown_completion_retains_transport_usage_and_an_unknown_agent_tufn wait_diagnostic_terminal(path: &Path, outcome: &str) -> (String, Vec<Value>) {let deadline = Instant::now() + Duration::from_secs(5);loop {- let log = fs::read_to_string(path).unwrap_or_default();- let records = log- .lines()- .map(|line| serde_json::from_str::<Value>(line).unwrap())- .collect::<Vec<_>>();+ let bytes = fs::read(path).unwrap_or_default();+ let log = String::from_utf8_lossy(&bytes).into_owned();+ let records = complete_records(&bytes);if records.iter().any(|record| {record.pointer("/event/request_end/outcome")@@ -1017,10 +1044,18 @@ fn real_hirouted_emits_zero_attempt_when_runtime_state_excludes_the_sealed_candi401,"seed disabled credential state");+ let response = fixture.request();+ assert_eq!(response.status, 502, "all runtime candidates excluded");assert_eq!(- fixture.request().status,- 502,- "all runtime candidates excluded"+ response.headers.get("content-type").map(String::as_str),+ Some("application/json")+ );+ assert!(!response.headers.contains_key("retry-after"));+ assert_eq!(+ serde_json::from_slice::<Value>(&response.body).unwrap(),+ serde_json::json!({+ "schema_version": "hiroute.gateway.error/v1", "code": "CANDIDATE_PREPARATION_FAILED", "phase": "candidate_preparation"+ }));assert_eq!(provider.calls(), 1);@@ -1503,12 +1538,16 @@ fn content_complete_for_requests(records: &[Value], request_ids: &[String]) -> b}fn read_records(path: &Path) -> Vec<Value> {- fs::read_to_string(path)- .ok()- .into_iter()- .flat_map(|contents| contents.lines().map(str::to_owned).collect::<Vec<_>>())- .filter(|line| !line.trim().is_empty())- .map(|line| serde_json::from_str(&line).unwrap())+ complete_records(&fs::read(path).unwrap_or_default())+}++fn complete_records(contents: &[u8]) -> Vec<Value> {+ // A live JSONL writer may be between its payload and newline writes.+ contents+ .split_inclusive(|byte| *byte == b'\n')+ .filter(|line| line.ends_with(b"\n"))+ .filter(|line| !line.iter().all(u8::is_ascii_whitespace))+ .map(|line| serde_json::from_slice(line).unwrap()).collect()}