Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.
Rust ai-agentsai-gatewayclaude-codecodexcoding-agentcost-optimizationdeepseekllm-gatewayllm-routerlocal-firstmodel-routingrusttask-routingtauri
fix: forward accepted streams promptly and bound observation ingestion (#38)
澄潭7b3b08e06ab0
diff --git a/crates/daemon/tests/support/standalone_headless_product.py b/crates/daemon/tests/support/standalone_headless_product.pyindex a82a1374..b748a1db 100644--- a/crates/daemon/tests/support/standalone_headless_product.py+++ b/crates/daemon/tests/support/standalone_headless_product.py@@ -587,8 +587,19 @@ def run(repository):snapshot = product.cli('compute', 'list')['data']native_source = next(source for source in snapshot['sources']if source['display_name'] == 'Headless Native API')+ # Make the price presentation's seconds-resolution evaluation clock advance.+ time.sleep(1.05)shown = product.cli('compute', 'show', native_source['source_id'])['data']- assert shown['sources'] == [native_source]+ assert len(shown['sources']) == 1+ comparable = json.loads(json.dumps(shown['sources'][0]))+ assert len(comparable['models']) == len(native_source['models'])+ for original, current in zip(native_source['models'], comparable['models']):+ earlier = original['presentation']['evaluated_at_ms']+ later = current['presentation']['evaluated_at_ms']+ assert isinstance(earlier, int) and isinstance(later, int) and 0 < earlier < later+ # Only query evaluation time changes; every saved/runtime field still matches.+ current['presentation']['evaluated_at_ms'] = earlier+ assert comparable == native_sourcebinding_id = native_source['models'][0]['binding_id']product.cli('protected-input', 'release', '--candidate', candidate_ref)diff --git a/crates/diagnostics/src/event/mod.rs b/crates/diagnostics/src/event/mod.rsindex 7cd2c0eb..7bad2924 100644--- a/crates/diagnostics/src/event/mod.rs+++ b/crates/diagnostics/src/event/mod.rs@@ -60,6 +60,7 @@ pub enum DiagnosticEvent {AttemptEnd(AttemptEnd),UpstreamWire(UpstreamWire),ReasoningCleanup(ReasoningCleanup),+ ResponseFailure(ResponseFailure),Fallback(Fallback),SemanticCommit(SemanticCommit),RequestCancel(RequestCancel),@@ -152,6 +153,7 @@ impl DiagnosticEvent {DiagnosticEvent::SemanticCommit(_) => Info,DiagnosticEvent::ModelStage(_) | DiagnosticEvent::UpstreamWire(_) => Debug,DiagnosticEvent::ReasoningCleanup(_) => Warn,+ DiagnosticEvent::ResponseFailure(_) => Warn,DiagnosticEvent::ContentCaptureEnd(end) => match end.outcome {CaptureOutcome::Success => Info,CaptureOutcome::Abort => Warn,@@ -217,6 +219,7 @@ impl DiagnosticEvent {DiagnosticEvent::AttemptEnd(_) => "attempt_end",DiagnosticEvent::UpstreamWire(_) => "upstream_wire",DiagnosticEvent::ReasoningCleanup(_) => "reasoning_cleanup",+ DiagnosticEvent::ResponseFailure(_) => "response_failure",DiagnosticEvent::Fallback(_) => "fallback",DiagnosticEvent::SemanticCommit(_) => "semantic_commit",DiagnosticEvent::RequestCancel(_) => "request_cancel",diff --git a/crates/diagnostics/src/event/model.rs b/crates/diagnostics/src/event/model.rsindex 0d997620..76111b45 100644--- a/crates/diagnostics/src/event/model.rs+++ b/crates/diagnostics/src/event/model.rs@@ -85,6 +85,43 @@ pub enum ReasoningCleanupReason {ResponseProtocolProjection,}+/// Closed categories only: never serializes provider messages, payloads or field values.+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct ResponseFailure {+ pub request_token: Option<CorrelationToken>,+ pub attempt_index: u64,+ pub stage: ResponseFailureStage,+ pub reason: ResponseFailureReason,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum ResponseFailureStage {+ NativeProjection,+ Decode,+ Render,+ PrefixBuffer,+ ProviderStream,+ EndOfStream,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum ResponseFailureReason {+ ResourceLimit,+ InvalidJson,+ InvalidSse,+ InvalidField,+ InvalidLifecycle,+ MissingTerminal,+ DuplicateTerminal,+ Unrepresentable,+ NoSemanticOutput,+ ProviderRejected,+ OtherProtocol,+}+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]#[serde(rename_all = "snake_case")]pub enum AttemptOutcome {diff --git a/crates/gateway/src/adapters/response/client_stream.rs b/crates/gateway/src/adapters/response/client_stream.rsindex 474042cf..b2909f1a 100644--- a/crates/gateway/src/adapters/response/client_stream.rs+++ b/crates/gateway/src/adapters/response/client_stream.rs@@ -61,6 +61,10 @@ pub struct IncrementalClientSseRenderer {}impl IncrementalClientSseRenderer {+ pub(crate) fn usage(&self) -> &ModelUsage {+ &self.usage+ }+pub(crate) fn with_budget(mut self,budget: hiroute_gateway_core::runtime::body::StreamBudget,diff --git a/crates/gateway/src/adapters/response/mod.rs b/crates/gateway/src/adapters/response/mod.rsindex b10010ba..061a4b8a 100644--- a/crates/gateway/src/adapters/response/mod.rs+++ b/crates/gateway/src/adapters/response/mod.rs@@ -16,6 +16,8 @@ mod wire;#[cfg(test)]#[path = "native_passthrough_tests.rs"]mod native_passthrough_tests;+#[cfg(test)]+mod native_reasoning_tests;#[cfg(test)]mod size_tests;diff --git a/crates/gateway/src/adapters/response/native_passthrough_evidence.rs b/crates/gateway/src/adapters/response/native_passthrough_evidence.rsindex b849e9a8..701f4e4f 100644--- a/crates/gateway/src/adapters/response/native_passthrough_evidence.rs+++ b/crates/gateway/src/adapters/response/native_passthrough_evidence.rs@@ -259,7 +259,7 @@ impl ProjectionState {.and_then(Value::as_str);let expected_type = match kind {"output_text" | "refusal" => "message",- "reasoning_summary_text" => "reasoning",+ "reasoning_summary_text" | "reasoning_text" => "reasoning","function_call_arguments" => "function_call","custom_tool_call_input" => "custom_tool_call",_ => unreachable!("tracked delta kind"),@@ -309,7 +309,12 @@ fn added_text_prefixes(item: &Map<String, Value>) -> Vec<AddedTextPrefix> {if let Some(content) = item.get("content").and_then(Value::as_array) {for (position, block) in content.iter().enumerate() {if let Some(text) = block.get("text").and_then(Value::as_str) {- capture(position, "output_text", text);+ let kind = if item.get("type").and_then(Value::as_str) == Some("reasoning") {+ "reasoning_text"+ } else {+ "output_text"+ };+ capture(position, kind, text);}if let Some(refusal) = block.get("refusal").and_then(Value::as_str) {capture(position, "refusal", refusal);@@ -341,6 +346,8 @@ fn delta_kind(event: &str) -> Option<(&'static str, &'static str, Option<&'statiSome(("refusal", "refusal", Some("content_index")))} else if event.starts_with("response.reasoning_summary_text.") {Some(("reasoning_summary_text", "text", Some("summary_index")))+ } else if event.starts_with("response.reasoning_text.") {+ Some(("reasoning_text", "text", Some("content_index")))} else if event.starts_with("response.function_call_arguments.") {Some(("function_call_arguments", "arguments", None))} else if event.starts_with("response.custom_tool_call_input.") {@@ -352,15 +359,11 @@ fn delta_kind(event: &str) -> Option<(&'static str, &'static str, Option<&'statifn final_text<'a>(item: &'a Map<String, Value>, position: usize, kind: &str) -> Option<&'a str> {match kind {- "output_text" | "refusal" => item+ "output_text" | "refusal" | "reasoning_text" => item.get("content")?.as_array()?.get(position)?- .get(if kind == "output_text" {- "text"- } else {- "refusal"- })?+ .get(if kind != "refusal" { "text" } else { "refusal" })?.as_str(),"reasoning_summary_text" => item.get("summary")?diff --git a/crates/gateway/src/adapters/response/native_passthrough_helpers.rs b/crates/gateway/src/adapters/response/native_passthrough_helpers.rsindex 80fa82e0..57f1b566 100644--- a/crates/gateway/src/adapters/response/native_passthrough_helpers.rs+++ b/crates/gateway/src/adapters/response/native_passthrough_helpers.rs@@ -57,6 +57,8 @@ pub(super) fn responses_semantic_event(event: &str) -> bool {| "response.refusal.done"| "response.reasoning_summary_text.delta"| "response.reasoning_summary_text.done"+ | "response.reasoning_text.delta"+ | "response.reasoning_text.done"| "response.function_call_arguments.delta"| "response.function_call_arguments.done"| "response.custom_tool_call_input.delta"@@ -142,6 +144,16 @@ pub(super) fn response_item_is_semantic(item: &Map<String, Value>) -> bool {item.get("encrypted_content").and_then(Value::as_str).is_some_and(|value| !value.is_empty())+ || item+ .get("content")+ .and_then(Value::as_array)+ .is_some_and(|content| {+ content.iter().any(|part| {+ part.get("text")+ .and_then(Value::as_str)+ .is_some_and(|text| !text.is_empty())+ })+ })|| item.get("summary").and_then(Value::as_array)diff --git a/crates/gateway/src/adapters/response/native_passthrough_sse.rs b/crates/gateway/src/adapters/response/native_passthrough_sse.rsindex 4bb0d2d1..b4354fa6 100644--- a/crates/gateway/src/adapters/response/native_passthrough_sse.rs+++ b/crates/gateway/src/adapters/response/native_passthrough_sse.rs@@ -22,7 +22,7 @@ pub(super) fn feed_projected_sse(end_stream: bool,) -> Result<Vec<NativeProjectedUnit>, ProtocolAdapterError> {let charged = ChargedBytes::copy_from_opaque(budget, MemoryRole::TransportInflight, bytes)- .map_err(|error| ModelIrError::InvalidSse(error.to_string()))?;+ .map_err(|_| ModelIrError::BufferLimit(bytes.len()))?;let mut metadata = VecDeque::new();let mut sink = ProjectedSink::default();let mut visitor = ProjectionVisitor {@@ -41,7 +41,10 @@ pub(super) fn feed_projected_sse().into());}- outcome.map_err(|error| ModelIrError::InvalidSse(error.to_string()))?;+ outcome.map_err(|error| match error {+ SseError::BudgetExceeded => ModelIrError::BufferLimit(bytes.len()),+ other => ModelIrError::InvalidSse(other.to_string()),+ })?;if sink.output.len() != metadata.len() {return Err(ModelIrError::InvalidSse("native projector lost SSE output metadata".into()).into(),diff --git a/crates/gateway/src/adapters/response/native_reasoning_tests.rs b/crates/gateway/src/adapters/response/native_reasoning_tests.rsnew file mode 100644index 00000000..1c691ee8--- /dev/null+++ b/crates/gateway/src/adapters/response/native_reasoning_tests.rs@@ -0,0 +1,103 @@+use super::native_passthrough::{NativeResponseProjector, NativeTerminalOutcome};+use crate::server::{+ core_runtime::profiles::{CandidateProtocolProfile, fixed_reasoning},+ request_plan::IngressProtocol,+};+use hiroute_gateway_core::runtime::body::BudgetTree;+use serde_json::{Value, json};++fn wire(value: Value) -> Vec<u8> {+ format!("data: {value}\n\n").into_bytes()+}++#[test]+fn long_plain_reasoning_commits_on_first_delta_and_keeps_terminal_usage() {+ let budget = BudgetTree::new(8 * 1024 * 1024, 8 * 1024 * 1024)+ .unwrap()+ .stream(8 * 1024 * 1024)+ .unwrap();+ let profile = CandidateProtocolProfile::exact_portable_path(+ IngressProtocol::Responses,+ IngressProtocol::Responses,+ "physical",+ fixed_reasoning("fixed"),+ );+ let mut projector =+ NativeResponseProjector::new_for_attempt(&profile, true, "alias".into(), None, budget)+ .unwrap();+ let reasoning = |text: &str| {+ json!({"type":"reasoning","id":"r","summary":[],+ "content":[{"type":"reasoning_text","text":text}]})+ };+ projector.feed(&wire(json!({"type":"response.created","response":{"id":"resp","model":"physical","status":"in_progress"}})), false).unwrap();+ projector+ .feed(+ &wire(+ json!({"type":"response.output_item.added","output_index":0,"item":reasoning("")}),+ ),+ false,+ )+ .unwrap();+ let chunk = "think ".repeat(512);+ let mut text = String::new();+ for _ in 0..800 {+ text.push_str(&chunk);+ let event = wire(+ json!({"type":"response.reasoning_text.delta","item_id":"r","output_index":0,"content_index":0,"delta":chunk}),+ );+ let units = projector.feed(&event, false).unwrap();+ assert!(+ units.iter().any(|unit| unit.semantic),+ "plain reasoning is output, not an indefinitely buffered opaque prefix"+ );+ assert_eq!(+ units+ .iter()+ .flat_map(|u| u.bytes.iter().copied())+ .collect::<Vec<_>>(),+ event+ );+ }+ assert!(text.len() > 2 * 1024 * 1024);+ let terminal = wire(json!({"type":"response.completed","response":{+ "id":"resp","model":"physical","status":"completed","output":[reasoning(&text)],+ "usage":{"input_tokens":100,"output_tokens":800,"total_tokens":900}}}));+ for chunk in terminal.chunks(16 * 1024) {+ projector.feed(chunk, false).unwrap();+ }+ let last = projector.feed(&[], true).unwrap();+ assert_eq!(+ last.last().unwrap().terminal,+ Some(NativeTerminalOutcome::Complete)+ );+ assert_eq!(projector.usage().output_tokens, Some(800));+}++#[test]+fn failed_response_preserves_reported_usage() {+ let budget = BudgetTree::new(1024 * 1024, 1024 * 1024)+ .unwrap()+ .stream(1024 * 1024)+ .unwrap();+ let profile = CandidateProtocolProfile::exact_portable_path(+ IngressProtocol::Responses,+ IngressProtocol::Responses,+ "physical",+ fixed_reasoning("fixed"),+ );+ let mut projector =+ NativeResponseProjector::new_for_attempt(&profile, true, "alias".into(), None, budget)+ .unwrap();+ let units = projector+ .feed(+ &wire(json!({"type":"response.failed","response":{+ "id":"resp","status":"failed","model":"physical","output":[],+ "error":{"code":"server_error","message":"private body"},+ "usage":{"input_tokens":100,"output_tokens":2}}})),+ true,+ )+ .unwrap();+ assert!(units.iter().any(|unit| unit.failure.is_some()));+ assert_eq!(projector.usage().input_tokens, Some(100));+ assert_eq!(projector.usage().output_tokens, Some(2));+}diff --git a/crates/gateway/src/observation/request.rs b/crates/gateway/src/observation/request.rsindex e054cd61..50d2525c 100644--- a/crates/gateway/src/observation/request.rs+++ b/crates/gateway/src/observation/request.rs@@ -243,6 +243,32 @@ impl RequestObservation {.token(CorrelationDomain::Attempt, attempt_id)}+ pub(crate) fn response_failure(+ &self,+ stage: hiroute_diagnostics::event::ResponseFailureStage,+ reason: hiroute_diagnostics::event::ResponseFailureReason,+ ) {+ let state = self.inner.state.lock().unwrap_or_else(|e| e.into_inner());+ let attempt = state+ .accepted_attempt+ .as_ref()+ .or(state.current_attempt.as_ref())+ .or(state.pending_attempt.as_ref());+ let ordinal = attempt+ .map(|a| a.ordinal)+ .filter(|n| *n != 0)+ .unwrap_or(state.next_attempt_ordinal);+ drop(state);+ self.emit_diagnostic(DiagnosticEvent::ResponseFailure(+ hiroute_diagnostics::event::ResponseFailure {+ request_token: self.inner.request_token,+ attempt_index: u64::from(ordinal),+ stage,+ reason,+ },+ ));+ }+pub(crate) fn reasoning_cleanup(&self,reason: hiroute_diagnostics::event::ReasoningCleanupReason,diff --git a/crates/gateway/src/observation/tests.rs b/crates/gateway/src/observation/tests.rsindex a2086f07..d852fbd3 100644--- a/crates/gateway/src/observation/tests.rs+++ b/crates/gateway/src/observation/tests.rs@@ -189,6 +189,10 @@ fn request_lifecycle_records_typed_diagnostics_without_raw_identities() {assert!(request.accept_current("frame:test", 128).is_some());request.reasoning_cleanup(ReasoningCleanupReason::ResponseProtocolProjection, 1, None);request.reasoning_cleanup(ReasoningCleanupReason::ResponseProtocolProjection, 3, None);+ request.response_failure(+ hiroute_diagnostics::event::ResponseFailureStage::NativeProjection,+ hiroute_diagnostics::event::ResponseFailureReason::ResourceLimit,+ );assert_eq!(request.lock_state()@@ -208,6 +212,7 @@ fn request_lifecycle_records_typed_diagnostics_without_raw_identities() {"semantic_commit","request_end","model_stage",+ "response_failure",] {assert!(log.contains(&format!("\"{kind}\":")),diff --git a/crates/gateway/src/publication/tests.rs b/crates/gateway/src/publication/tests.rsindex 45272d16..f27e9dd7 100644--- a/crates/gateway/src/publication/tests.rs+++ b/crates/gateway/src/publication/tests.rs@@ -37,9 +37,10 @@ fn domain_gateway_profile_roundtrip_preserves_model_evidence() {fn published_attempt_phases_default_to_ten_minutes() {let aggregate = super::compiler::compile(&snapshot(1, "renderer")).unwrap();#[cfg(feature = "e2e-test-control")]- let override_timeout = crate::server::test_control::attempt_timeout_override();+ let expected_timeout = crate::server::test_control::attempt_timeout_override()+ .unwrap_or(Duration::from_secs(10 * 60));#[cfg(not(feature = "e2e-test-control"))]- let override_timeout: Option<Duration> = None;+ let expected_timeout = Duration::from_secs(10 * 60);let plans = aggregate.envelope.attempt_plan_index_handle@@ -47,18 +48,9 @@ fn published_attempt_phases_default_to_ten_minutes() {.collect::<Vec<_>>();assert_eq!(plans.len(), 2);for (_, plan) in plans {- assert_eq!(- plan.timeouts.request_write,- override_timeout.unwrap_or(Duration::from_secs(10 * 60))- );- assert_eq!(- plan.timeouts.first_byte,- override_timeout.unwrap_or(Duration::from_secs(10 * 60))- );- assert_eq!(- plan.timeouts.stream_idle,- override_timeout.unwrap_or(Duration::from_secs(10 * 60))- );+ assert_eq!(plan.timeouts.request_write, expected_timeout);+ assert_eq!(plan.timeouts.first_byte, expected_timeout);+ assert_eq!(plan.timeouts.stream_idle, expected_timeout);}}diff --git a/crates/gateway/src/runtime/driver.rs b/crates/gateway/src/runtime/driver.rsindex 9128174f..04bdbdde 100644--- a/crates/gateway/src/runtime/driver.rs+++ b/crates/gateway/src/runtime/driver.rs@@ -40,6 +40,7 @@ use super::state::{mod materialization;mod response;+mod response_diagnostics;mod selection;pub(crate) use materialization::resolve_target;@@ -181,10 +182,8 @@ pub struct ProductionAttemptState {projector: Option<Box<adapters::NativeResponseProjector>>,prefix: Option<ChargedBodyQueue>,prefix_terminal_chunks: Option<usize>,- decoder_budget: Option<response::PrecommitDecoderBudget>,budget: StreamBudget,semantic_seen: bool,- terminal_seen: bool,semantic_terminal: Option<SemanticTerminalOutcome>,retry_after: Option<std::time::Duration>,runtime_state_authority: RuntimeStateAuthoritySignal,@@ -206,7 +205,6 @@ pub struct ProductionReadiness {decoder: Option<adapters::NativeResponseDecoder>,renderer: Option<adapters::IncrementalClientSseRenderer>,projector: Option<Box<adapters::NativeResponseProjector>>,- _decoder_budget: Option<response::PrecommitDecoderBudget>,_chat_tool_projection_budget: Option<Reservation>,budget: StreamBudget,streaming: bool,diff --git a/crates/gateway/src/runtime/driver/materialization.rs b/crates/gateway/src/runtime/driver/materialization.rsindex 244ff9db..3c620f20 100644--- a/crates/gateway/src/runtime/driver/materialization.rs+++ b/crates/gateway/src/runtime/driver/materialization.rs@@ -34,7 +34,7 @@ use crate::server::core_runtime::profiles::{PlannerCandidateFactsV1,};-use super::response::{PrecommitDecoderBudget, connector_error_profile, failure_facts};+use super::response::{connector_error_profile, failure_facts};#[path = "materialization/lease_target.rs"]mod lease_target;@@ -296,7 +296,6 @@ pub(super) async fn materialize_attempt(plan.precommit_event_capacity,).map_err(safe_error)?;- let decoder_budget = PrecommitDecoderBudget::new(context.budget)?;let lease_target = lease_target::for_request(&execution, profile)?;let lease_target_digest = hiroute_domain::CanonicalDigest::of(&lease_target).map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;@@ -564,10 +563,8 @@ pub(super) async fn materialize_attempt(projector: None,prefix: Some(prefix),prefix_terminal_chunks: None,- decoder_budget: Some(decoder_budget),budget: context.budget.clone(),semantic_seen: false,- terminal_seen: false,semantic_terminal: None,retry_after: None,runtime_state_authority,diff --git a/crates/gateway/src/runtime/driver/response.rs b/crates/gateway/src/runtime/driver/response.rsindex 33149cd7..7596feac 100644--- a/crates/gateway/src/runtime/driver/response.rs+++ b/crates/gateway/src/runtime/driver/response.rs@@ -1,3 +1,7 @@+use super::response_diagnostics;+use hiroute_diagnostics::event::{+ ResponseFailureReason as FailureReason, ResponseFailureStage as FailureStage,+};use std::sync::Arc;use hiroute_gateway_core::core::filter::LocalReply;@@ -5,7 +9,7 @@ use hiroute_gateway_core::runtime::attempt::{ChargedResponseHead, Disposition, PrecommitEvent, PublishedDisposition,};use hiroute_gateway_core::runtime::body::{- BodyMetadataOwner, ChargedBodyQueue, ChargedBytes, MemoryRole, Reservation, StreamBudget,+ BodyMetadataOwner, ChargedBodyQueue, ChargedBytes, MemoryRole, StreamBudget,};use hiroute_gateway_core::runtime::driver::{AcceptedBodyFrame, ClassifiedAttemptResult, NormalizedAttemptLocalReply,@@ -34,14 +38,6 @@ use super::{ProductionReadiness, SemanticTerminalOutcome, label, safe_error,};-pub(super) struct PrecommitDecoderBudget {- budget: StreamBudget,- raw_bytes: usize,- charged_bytes: usize,- charges: Vec<Reservation>,- _metadata: Reservation,-}-fn note_response_reasoning_loss(removed: usize) {if removed > 0&& let Some(observation) = crate::server::core_runtime::observation::active_request()@@ -54,62 +50,6 @@ fn note_response_reasoning_loss(removed: usize) {}}-impl PrecommitDecoderBudget {- const MAX_CHARGE_STEPS: usize = usize::BITS as usize + 1;-- pub(super) fn new(budget: &StreamBudget) -> Result<Self, Arc<str>> {- let metadata_bytes = Self::MAX_CHARGE_STEPS- .checked_mul(std::mem::size_of::<Reservation>())- .ok_or_else(|| Arc::from("native decoder metadata budget overflow"))?;- let metadata = budget- .reserve(MemoryRole::ResponsePrefix, metadata_bytes)- .map_err(safe_error)?;- let charges = Vec::with_capacity(Self::MAX_CHARGE_STEPS);- Ok(Self {- budget: budget.clone(),- raw_bytes: 0,- charged_bytes: 0,- charges,- _metadata: metadata,- })- }-- fn charge_frame(&mut self, bytes: usize) -> Result<(), Arc<str>> {- let raw_bytes = self- .raw_bytes- .checked_add(bytes)- .ok_or_else(|| Arc::from("native precommit decoder byte limit overflow"))?;- // Vec's amortized retained capacity is strictly below twice the live- // byte length after its minimum allocation. Shadow-charge that upper- // bound before the native decoder is allowed to copy this frame.- let target_charge = if raw_bytes == 0 {- 0- } else {- raw_bytes- .checked_mul(2)- .and_then(|bytes| bytes.checked_add(8))- .ok_or_else(|| Arc::from("native precommit decoder charge overflow"))?- };- if target_charge > self.charged_bytes {- // Transport chunk boundaries are arbitrary and must not become a- // correctness limit. Grow the accounting reservation geometrically- // so a long sequence of tiny chunks uses bounded metadata while the- // request memory budget remains enforced.- let next_charge = target_charge- .checked_next_power_of_two()- .unwrap_or(target_charge);- self.charges.push(- self.budget- .reserve(MemoryRole::ResponsePrefix, next_charge - self.charged_bytes)- .map_err(safe_error)?,- );- self.charged_bytes = next_charge;- }- self.raw_bytes = raw_bytes;- Ok(())- }-}-pub(super) fn classify_precommit(state: &mut ProductionAttemptState,event: PrecommitEvent,@@ -166,78 +106,29 @@ pub(super) fn classify_precommit(.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?,);}+ if state.streaming && status.is_success() {+ // HTTP acceptance, not the arrival of a recognized semantic event,+ // closes the transparent retry window. Body parsing stays bounded+ // and executes only on the accepted response path.+ return Ok(PrecommitClassification::classified(+ ClassifiedAttemptResult {+ facts: ProviderClassificationFacts {+ http_status: Some(status),+ readiness: label("response_headers"),+ retryability: RetryabilityFact::NonRetryable,+ ..ProviderClassificationFacts::default()+ },+ readiness: take_readiness(state, status, "text/event-stream", false)?,+ },+ ));+ }Ok(PrecommitClassification::pending())}PrecommitEvent::Body(bytes) => {- state- .decoder_budget- .as_mut()- .ok_or_else(|| Arc::from("native decoder budget owner is unavailable"))?- .charge_frame(bytes.retained_capacity())?;let status = state.response_status.ok_or_else(|| Arc::from("provider body arrived before final response head"))?;- if state.streaming && status.is_success() {- let outcome = match if state.native_output {- project_native_chunk_state(state, bytes.bytes(), false)- } else {- decode_stream_chunk_state(state, bytes.bytes(), false)- } {- Ok(outcome) => outcome,- Err(_) => {- drop(bytes);- return classify_state_failure(- state,- RawAttemptFailure::Protocol,- StatusCode::BAD_GATEWAY,- );- }- };- drop(bytes);- if let Some(error) = outcome.failure {- return classify_model_error(state, error, status, true);- }- if !state.native_output && outcome.terminal && outcome.semantic {- // Chat/Messages have already supplied their native terminal.- // Validate this fragment's tail before publishing the- // translated terminal, then close the downstream stream- // with the last queued output rather than waiting for the- // upstream transport to close as a separate event.- if finish_native_stream_on_terminal(&mut state.decoder).is_err() {- return classify_state_failure(- state,- RawAttemptFailure::Protocol,- StatusCode::BAD_GATEWAY,- );- }- let terminal_chunks = state.prefix.as_ref().map_or(0, ChargedBodyQueue::len);- if terminal_chunks == 0 {- return classify_state_failure(- state,- RawAttemptFailure::Protocol,- StatusCode::BAD_GATEWAY,- );- }- state.prefix_terminal_chunks = Some(terminal_chunks);- }- if outcome.semantic {- return Ok(PrecommitClassification::classified(- ClassifiedAttemptResult {- facts: semantic_response_facts(status),- readiness: take_readiness(- state,- StatusCode::OK,- "text/event-stream",- // Native protocol terminal metadata, rather- // than upstream transport EOF, marks the- // eventual downstream EOS frame.- false,- )?,- },- ));- }- Ok(PrecommitClassification::pending())- } else if state.native_output && status.is_success() {+ if state.native_output && status.is_success() {let projected = state.projector.as_mut()@@ -278,51 +169,7 @@ pub(super) fn classify_precommit(let status = state.response_status.ok_or_else(|| Arc::from("provider ended before final response head"))?;- if state.streaming && status.is_success() {- let outcome = match if state.native_output {- project_native_chunk_state(state, &[], true)- } else {- decode_stream_chunk_state(state, &[], true)- } {- Ok(outcome) => outcome,- Err(_) => {- return classify_state_failure(- state,- RawAttemptFailure::Protocol,- StatusCode::BAD_GATEWAY,- );- }- };- if let Some(error) = outcome.failure {- return classify_model_error(state, error, status, true);- }- if !state.semantic_seen {- return classify_state_failure(- state,- RawAttemptFailure::Protocol,- StatusCode::BAD_GATEWAY,- );- }- if !state.native_output {- state- .decoder- .take()- .ok_or_else(|| Arc::from("native response decoder is unavailable"))?- .finish()- .map_err(|_| Arc::from("native response terminal is malformed"))?;- }- Ok(PrecommitClassification::classified(- ClassifiedAttemptResult {- facts: semantic_response_facts(status),- readiness: take_readiness(- state,- StatusCode::OK,- "text/event-stream",- true,- )?,- },- ))- } else if state.native_output && status.is_success() {+ if state.native_output && status.is_success() {let units = match state.projector.as_mut()@@ -348,7 +195,7 @@ pub(super) fn classify_precommit(push_native_prefix_unit(state, unit.bytes, unit.terminal.is_some())?;}if let Some(error) = failure {- return classify_model_error(state, error, status, false);+ return classify_model_error(state, error, status);}if !state.semantic_seen || state.semantic_terminal.is_none() {return classify_state_failure(@@ -393,7 +240,7 @@ pub(super) fn classify_precommit(};state.semantic_terminal = semantic_terminal_for_response(&decoded.response);if let Some(error) = decoded.response.error.clone() {- return classify_model_error(state, error, status, false);+ return classify_model_error(state, error, status);}if !decoded.events@@ -488,7 +335,13 @@ pub(super) fn finalize_attempt_facts(let usage = readiness.and_then(|readiness| readiness.projector.as_ref()).map(|projector| projector.usage())- .or_else(|| state.projector.as_ref().map(|projector| projector.usage()));+ .or_else(|| state.projector.as_ref().map(|projector| projector.usage()))+ .or_else(|| {+ readiness+ .and_then(|r| r.renderer.as_ref())+ .map(|r| r.usage())+ })+ .or_else(|| state.renderer.as_ref().map(|r| r.usage()));if let Some(usage) = usage.filter(|usage| !usage.is_empty()) {facts.usage = Some(model_usage_fact(usage));}@@ -806,21 +659,13 @@ fn classify_model_error(state: &mut ProductionAttemptState,error: ModelError,response_status: StatusCode,- stream: bool,) -> Result<PrecommitClassification<ProductionReadiness, ProductionDecodedSse>, Arc<str>> {let kind = sanitized_provider_kind(&state.profile, &error);let status = error.status.unwrap_or(response_status.as_u16());- let raw = if stream && response_status.is_success() {- RawAttemptFailure::Stream {- kind,- retry_after: state.retry_after,- }- } else {- RawAttemptFailure::Http {- status,- kind,- retry_after: state.retry_after,- }+ let raw = RawAttemptFailure::Http {+ status,+ kind,+ retry_after: state.retry_after,};let local_status = StatusCode::from_u16(status).ok()@@ -885,7 +730,6 @@ fn take_readiness(decoder: state.decoder.take(),renderer: state.renderer.take(),projector: state.projector.take(),- _decoder_budget: state.decoder_budget.take(),_chat_tool_projection_budget: state.chat_tool_projection_budget.take(),budget: state.budget.clone(),streaming: state.streaming,@@ -895,43 +739,6 @@ fn take_readiness(})}-#[derive(Default)]-struct StreamDecodeOutcome {- semantic: bool,- terminal: bool,- failure: Option<ModelError>,-}--fn project_native_chunk_state(- state: &mut ProductionAttemptState,- bytes: &[u8],- end_stream: bool,-) -> Result<StreamDecodeOutcome, Arc<str>> {- let units = state- .projector- .as_mut()- .ok_or_else(|| Arc::from("native response projector is unavailable"))?- .feed(bytes, end_stream)- .map_err(|_| Arc::from("native stream response is malformed"))?;- let mut outcome = StreamDecodeOutcome::default();- for unit in units {- let semantic_before = state.semantic_seen || outcome.semantic;- if let Some(terminal) = unit.terminal {- let terminal = native_terminal(terminal);- state.semantic_terminal = Some(terminal);- state.terminal_seen = true;- outcome.terminal = true;- }- if unit.failure.is_some() && !semantic_before {- outcome.failure = unit.failure;- }- outcome.semantic |= unit.semantic;- push_native_prefix_unit(state, unit.bytes, unit.terminal.is_some())?;- }- state.semantic_seen |= outcome.semantic;- Ok(outcome)-}-fn project_native_chunk_readiness(readiness: &mut ProductionReadiness,bytes: &[u8],@@ -942,13 +749,19 @@ fn project_native_chunk_readiness(.as_mut().ok_or_else(|| Arc::from("accepted native projector is unavailable"))?.feed(bytes, end_stream)- .map_err(|_| Arc::from("accepted native stream is malformed"))?;+ .map_err(|error| response_diagnostics::adapter(FailureStage::NativeProjection, error))?;let mut output = Vec::new();let mut terminal = false;for unit in units {if terminal {return Err(Arc::from("native stream emitted bytes after terminal"));}+ if unit.failure.is_some() {+ response_diagnostics::note(+ FailureStage::ProviderStream,+ FailureReason::ProviderRejected,+ );+ }if let Some(outcome) = unit.terminal {readiness.semantic_terminal = Some(native_terminal(outcome));terminal = true;@@ -964,6 +777,7 @@ fn push_native_prefix_unit(terminal: bool,) -> Result<(), Arc<str>> {if state.prefix_terminal_chunks.is_some() {+ response_diagnostics::note(FailureStage::PrefixBuffer, FailureReason::InvalidLifecycle);return Err(Arc::from("native response emitted output after terminal"));}push_prefix_bytes(state, bytes)?;@@ -974,6 +788,7 @@ fn push_native_prefix_unit(.ok_or_else(|| Arc::from("response prefix owner is unavailable"))?.len();if chunks == 0 {+ response_diagnostics::note(FailureStage::PrefixBuffer, FailureReason::InvalidLifecycle);return Err(Arc::from("native terminal produced no wire bytes"));}state.prefix_terminal_chunks = Some(chunks);@@ -1004,70 +819,6 @@ fn model_usage_fact(usage: &ModelUsage) -> UsageFact {}}-fn decode_stream_chunk_state(- state: &mut ProductionAttemptState,- bytes: &[u8],- end_stream: bool,-) -> Result<StreamDecodeOutcome, Arc<str>> {- let decoder = state- .decoder- .as_mut()- .ok_or_else(|| Arc::from("native stream decoder is unavailable"))?;- let renderer = state- .renderer- .as_mut()- .ok_or_else(|| Arc::from("client stream renderer is unavailable"))?;- let prefix = state- .prefix- .as_mut()- .ok_or_else(|| Arc::from("response prefix owner is unavailable"))?;- let mut status = decoder- .feed(bytes, end_stream)- .map_err(|_| Arc::from("native stream response is malformed"))?;- let mut outcome = StreamDecodeOutcome::default();- loop {- for event in decoder.take_events() {- observe_semantic_terminal(&mut state.semantic_terminal, &event.event);- let semantic = event.is_semantic_output();- if let ModelEvent::ResponseFailed { error } = &event.event- && !state.semantic_seen- && !outcome.semantic- {- outcome.failure = Some(error.clone());- }- let rendered_events = renderer- .push(&event)- .map_err(|_| Arc::from("client stream event is not representable"))?;- let emitted = !rendered_events.is_empty();- for rendered in rendered_events {- push_queue_bytes(- prefix,- &state.budget,- rendered- .wire_bytes()- .map_err(|_| Arc::from("client stream event serialization failed"))?,- )?;- }- note_response_reasoning_loss(renderer.take_reasoning_loss());- outcome.semantic |= semantic && emitted;- let terminal = matches!(- event.event,- ModelEvent::ResponseCompleted { .. } | ModelEvent::ResponseFailed { .. }- );- outcome.terminal |= terminal;- state.terminal_seen |= terminal;- }- if status != adapters::ResponseDecodeStatus::NeedDrain {- break;- }- status = decoder- .resume()- .map_err(|_| Arc::from("native stream response drain failed"))?;- }- state.semantic_seen |= outcome.semantic;- Ok(outcome)-}-fn decode_stream_chunk_readiness(readiness: &mut ProductionReadiness,bytes: &[u8],@@ -1083,14 +834,20 @@ fn decode_stream_chunk_readiness(.ok_or_else(|| Arc::from("accepted client renderer is unavailable"))?;let mut status = decoder.feed(bytes, end_stream)- .map_err(|_| Arc::from("accepted native stream is malformed"))?;+ .map_err(|error| response_diagnostics::adapter(FailureStage::Decode, error))?;let mut output = Vec::new();loop {for event in decoder.take_events() {observe_semantic_terminal(&mut readiness.semantic_terminal, &event.event);+ if matches!(event.event, ModelEvent::ResponseFailed { .. }) {+ response_diagnostics::note(+ FailureStage::ProviderStream,+ FailureReason::ProviderRejected,+ );+ }for rendered in renderer.push(&event)- .map_err(|_| Arc::from("accepted client stream event is not representable"))?+ .map_err(|error| response_diagnostics::adapter(FailureStage::Render, error))?{output.extend_from_slice(&rendered@@ -1174,7 +931,9 @@ fn push_prefix_bytes(state: &mut ProductionAttemptState, bytes: Vec<u8>) -> Resu.prefix.as_mut().ok_or_else(|| Arc::from("response prefix owner is unavailable"))?;- push_queue_bytes(prefix, &state.budget, bytes)+ push_queue_bytes(prefix, &state.budget, bytes).inspect_err(|_| {+ response_diagnostics::note(FailureStage::PrefixBuffer, FailureReason::ResourceLimit);+ })}fn push_queue_bytes(diff --git a/crates/gateway/src/runtime/driver/response_budget_tests.rs b/crates/gateway/src/runtime/driver/response_budget_tests.rsindex 7c4963cb..5989bacf 100644--- a/crates/gateway/src/runtime/driver/response_budget_tests.rs+++ b/crates/gateway/src/runtime/driver/response_budget_tests.rs@@ -79,20 +79,22 @@ fn protocol_prefix_grows_beyond_initial_queue_capacity_and_releases_memory() {}#[test]-fn precommit_decoder_is_bounded_by_bytes_not_transport_chunk_count() {+fn drained_output_does_not_charge_cumulative_stream_bytes() {let budget = BudgetTree::new(4 * 1024 * 1024, 4 * 1024 * 1024).unwrap().stream(4 * 1024 * 1024).unwrap();- let mut decoder = PrecommitDecoderBudget::new(&budget).unwrap();+ let plan = hiroute_gateway_core::runtime::body::BodyPlan::PassThrough {+ max_chunk_bytes: 64 * 1024,+ };+ let mut queue =+ ChargedBodyQueue::new(&budget, MemoryRole::ResponsePrefix, &plan, usize::MAX, 1).unwrap();for _ in 0..1_000 {- decoder.charge_frame(128).unwrap();+ push_queue_bytes(&mut queue, &budget, vec![b'x'; 64 * 1024]).unwrap();+ drop(queue.pop_front().unwrap());}- decoder.charge_frame(256 * 1024).unwrap();- assert!(- decoder.charge_frame(4 * 1024 * 1024).is_err(),- "the actual memory budget remains authoritative"- );+ queue.clear_and_release();+ assert_eq!(budget.snapshot().unwrap().live, 0);}#[test]diff --git a/crates/gateway/src/runtime/driver/response_diagnostics.rs b/crates/gateway/src/runtime/driver/response_diagnostics.rsnew file mode 100644index 00000000..eaecf4d2--- /dev/null+++ b/crates/gateway/src/runtime/driver/response_diagnostics.rs@@ -0,0 +1,60 @@+use crate::server::core_runtime::{adapters::ProtocolAdapterError, model_ir::ModelIrError};+use hiroute_diagnostics::event::{ResponseFailureReason as Reason, ResponseFailureStage as Stage};+use std::sync::Arc;++pub(super) fn note(stage: Stage, reason: Reason) {+ if let Some(observation) = crate::server::core_runtime::observation::active_request() {+ observation.response_failure(stage, reason);+ }+}++pub(super) fn adapter(stage: Stage, error: ProtocolAdapterError) -> Arc<str> {+ note(stage, category(&error));+ Arc::from("provider response processing failed")+}++fn category(error: &ProtocolAdapterError) -> Reason {+ match error {+ ProtocolAdapterError::ModelIr(error) => match error {+ ModelIrError::BufferLimit(_) => Reason::ResourceLimit,+ ModelIrError::InvalidJson(_) => Reason::InvalidJson,+ ModelIrError::InvalidSse(_) => Reason::InvalidSse,+ ModelIrError::ExpectedObject | ModelIrError::InvalidField(_) => Reason::InvalidField,+ ModelIrError::InvalidResponseLifecycle(_) => Reason::InvalidLifecycle,+ ModelIrError::MissingTerminalEvent => Reason::MissingTerminal,+ ModelIrError::DuplicateTerminalEvent => Reason::DuplicateTerminal,+ _ => Reason::OtherProtocol,+ },+ ProtocolAdapterError::ClientUnrepresentable(_) => Reason::Unrepresentable,+ _ => Reason::OtherProtocol,+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn diagnostic_categories_distinguish_local_limits_and_protocol_failures_without_payloads() {+ for (error, expected) in [+ (ModelIrError::BufferLimit(100), Reason::ResourceLimit),+ (+ ModelIrError::InvalidJson("secret provider body".into()),+ Reason::InvalidJson,+ ),+ (+ ModelIrError::InvalidSse("secret event".into()),+ Reason::InvalidSse,+ ),+ (ModelIrError::MissingTerminalEvent, Reason::MissingTerminal),+ (+ ModelIrError::DuplicateTerminalEvent,+ Reason::DuplicateTerminal,+ ),+ ] {+ let category = category(&error.into());+ assert_eq!(category, expected);+ assert!(!serde_json::to_string(&category).unwrap().contains("secret"));+ }+ }+}diff --git a/crates/gateway/src/runtime/driver/response_terminal_tests.rs b/crates/gateway/src/runtime/driver/response_terminal_tests.rsindex 1ae344af..9ab98e58 100644--- a/crates/gateway/src/runtime/driver/response_terminal_tests.rs+++ b/crates/gateway/src/runtime/driver/response_terminal_tests.rs@@ -36,7 +36,6 @@ fn accepted_codex_terminal_event_crosses_transport_frames_and_keeps_eos() {decoder: None,renderer: None,projector: Some(Box::new(projector)),- _decoder_budget: None,_chat_tool_projection_budget: None,budget: budget.clone(),streaming: true,@@ -150,7 +149,6 @@ fn accepted_chat_terminal_closes_responses_stream_without_waiting_for_transport_),renderer: Some(adapters::IncrementalClientSseRenderer::new(client, "alias").unwrap()),projector: None,- _decoder_budget: None,_chat_tool_projection_budget: None,budget: budget.clone(),streaming: true,diff --git a/crates/observation/src/receipt/completeness.rs b/crates/observation/src/receipt/completeness.rsnew file mode 100644index 00000000..6e81fc76--- /dev/null+++ b/crates/observation/src/receipt/completeness.rs@@ -0,0 +1,45 @@+//! Incremental projection over the append-only fact log. Gap repair and retention+//! can remove evidence, so those exceptional paths still rebuild from surviving facts.+use super::*;++pub(super) fn append(+ transaction: &Transaction<'_>,+ envelope: &ExecutionFactEnvelopeV1,+) -> Result<(), FactProjectionError> {+ let workspace = envelope.correlation.workspace_id.as_str();+ let session = envelope.correlation.conversation_id.as_str();+ let (previous, has_prior, gap): (String, bool, bool) = transaction.query_row(+ "SELECT facts_completeness,+ EXISTS(SELECT 1 FROM execution_fact_events WHERE workspace_id=?1 AND session_id=?2+ AND NOT (producer_id=?3 AND producer_epoch=?4 AND stream_id=?5 AND sequence=?6)),+ EXISTS(SELECT 1 FROM observation_gaps WHERE channel='fact' AND workspace_id=?1 AND session_id=?2)+ FROM sessions WHERE workspace_id=?1 AND session_id=?2",+ params![workspace, session, envelope.producer.stream.producer_id.as_str(),+ envelope.producer.stream.producer_epoch.as_str(), envelope.producer.stream.stream_id.as_str(),+ i64::try_from(envelope.sequence).map_err(|_| FactProjectionError::Invalid)?],+ |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),+ ).map_err(|_| FactProjectionError::Storage)?;+ let current = completeness_from_facts(std::slice::from_ref(envelope));+ // A content/gap-created placeholder or a fully retired session has no prior+ // fact contribution. Its initial "unknown" must not taint the first fact.+ let result = if gap {+ FactsCompleteness::Partial+ } else if !has_prior {+ current+ } else {+ let previous = match previous.as_str() {+ "complete" => FactsCompleteness::Complete,+ "partial" => FactsCompleteness::Partial,+ "unknown" => FactsCompleteness::Unknown,+ _ => return Err(FactProjectionError::Corrupt),+ };+ least_complete(previous, current)+ };+ transaction+ .execute(+ "UPDATE sessions SET facts_completeness=?3 WHERE workspace_id=?1 AND session_id=?2",+ params![workspace, session, facts_completeness_str(result)],+ )+ .map_err(|_| FactProjectionError::Storage)?;+ Ok(())+}diff --git a/crates/observation/src/receipt/mod.rs b/crates/observation/src/receipt/mod.rsindex c8a3dfb6..2ec75d74 100644--- a/crates/observation/src/receipt/mod.rs+++ b/crates/observation/src/receipt/mod.rs@@ -1,5 +1,6 @@//! Immutable Receipt and Value projections derived from the durable execution-fact log.+mod completeness;mod plan_quality;mod validation;@@ -140,7 +141,7 @@ fn ensure_scope(if existing_correlation == "unproven" && correlation_name != "unproven" {transaction.execute(- "UPDATE sessions SET correlation=?3, facts_completeness='complete'+ "UPDATE sessions SET correlation=?3WHERE workspace_id=?1 AND session_id=?2",params![correlation.workspace_id.as_str(),@@ -269,7 +270,7 @@ fn ensure_scope().map_err(|_| FactProjectionError::Storage)?;}- update_completeness(transaction, facts)+ completeness::append(transaction, envelope)}fn classified_traffic(@@ -294,32 +295,6 @@ fn classified_traffic(Ok("unknown")}-fn update_completeness(- transaction: &Transaction<'_>,- facts: &[ExecutionFactEnvelopeV1],-) -> Result<(), FactProjectionError> {- let first = facts- .first()- .ok_or(FactProjectionError::MissingPrerequisite)?;- let completeness = session_facts_completeness(- transaction,- &first.correlation.workspace_id,- &first.correlation.conversation_id,- )?;- transaction- .execute(- "UPDATE sessions SET facts_completeness=?3- WHERE workspace_id=?1 AND session_id=?2",- params![- first.correlation.workspace_id.as_str(),- first.correlation.conversation_id.as_str(),- facts_completeness_str(completeness),- ],- )- .map_err(|_| FactProjectionError::Storage)?;- Ok(())-}-fn build_receipt(connection: &Connection,facts: &[ExecutionFactEnvelopeV1],diff --git a/crates/observation/src/store/fact_log.rs b/crates/observation/src/store/fact_log.rsindex 5e3c6543..210be784 100644--- a/crates/observation/src/store/fact_log.rs+++ b/crates/observation/src/store/fact_log.rs@@ -114,6 +114,8 @@ pub(crate) fn load_session(workspace_id: &WorkspaceId,session_id: &SessionId,) -> Result<Vec<ExecutionFactEnvelopeV1>, ObservationStoreError> {+ #[cfg(test)]+ SESSION_LOADS.with(|count| count.set(count.get() + 1));let request_ids = {let mut statement = connection.prepare(@@ -145,6 +147,11 @@ pub(crate) fn load_session(Ok(facts)}+#[cfg(test)]+thread_local! {+ pub(crate) static SESSION_LOADS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };+}+fn sql_u64(value: u64) -> Result<i64, ObservationStoreError> {value.try_into().map_err(|_| ObservationStoreError::Corrupt)}diff --git a/crates/observation/src/store/request_retention.rs b/crates/observation/src/store/request_retention.rsindex 55126d84..067b7dba 100644--- a/crates/observation/src/store/request_retention.rs+++ b/crates/observation/src/store/request_retention.rs@@ -77,6 +77,27 @@ impl LocalObservationStore {transaction.execute("DELETE FROM turns WHERE workspace_id=?1 AND session_id=?2 AND NOT EXISTS(SELECT 1 FROM logical_requests r WHERE r.workspace_id=?1 AND r.turn_id=turns.turn_id)",params![workspace,session]).map_err(|_|Error::Unavailable)?;transaction.execute("UPDATE sessions SET content_completeness='expired',facts_completeness='unknown',tombstone_reason='retention_expired',agent_id='',correlation='unproven' WHERE workspace_id=?1 AND session_id=?2 AND NOT EXISTS(SELECT 1 FROM logical_requests WHERE workspace_id=?1 AND session_id=?2)",params![workspace,session]).map_err(|_|Error::Unavailable)?;}+ // Removing the only Partial/Unknown request may improve completeness.+ // Rebuild once per affected session, never once per newly ingested fact.+ let sessions: std::collections::BTreeSet<_> = requests+ .iter()+ .map(|(workspace, session, _)| (workspace, session))+ .collect();+ for (workspace, session) in sessions {+ let completeness = crate::receipt::session_facts_completeness(+ &transaction,+ &WorkspaceId::parse(workspace).map_err(|_| Error::Corrupt)?,+ &SessionId::parse(session).map_err(|_| Error::Corrupt)?,+ )+ .map_err(|_| Error::Unavailable)?;+ let value = match completeness {+ hiroute_domain::FactsCompleteness::Complete => "complete",+ hiroute_domain::FactsCompleteness::Partial => "partial",+ hiroute_domain::FactsCompleteness::Unknown => "unknown",+ };+ transaction.execute("UPDATE sessions SET facts_completeness=?3 WHERE workspace_id=?1 AND session_id=?2",+ params![workspace, session, value]).map_err(|_| Error::Unavailable)?;+ }if !requests.is_empty() || quality_changed {// Index cleanup is part of the same visibility barrier. Physical// blob GC is resumable and can follow this committed logical erase.diff --git a/crates/observation/src/store/tests/incremental_completeness.rs b/crates/observation/src/store/tests/incremental_completeness.rsnew file mode 100644index 00000000..9055bac7--- /dev/null+++ b/crates/observation/src/store/tests/incremental_completeness.rs@@ -0,0 +1,102 @@+use super::support::*;+use crate::WriterCycleOutcome;+use hiroute_domain::{CompletenessDeltaV1, ExecutionFactV1, FactsCompleteness};++#[test]+fn long_session_appends_match_full_projection_without_reloading_history() {+ for delta in [None, Some(CompletenessDeltaV1::Unknown)] {+ let temp = tempfile::tempdir().unwrap();+ let store = open_store(temp.path());+ let writer = writer(&store);+ let session = Fixture::new("long-session").session;+ for index in 0..40 {+ let mut fixture = Fixture::new(&format!("long-{index}"));+ fixture.session = session.clone();+ let channel = fact_channel(&fixture, 512 * 1024);+ let attempt = format!("attempt-long-{index}");+ let mut facts = request_facts(&fixture, &attempt, finished(&attempt), 100 + index);+ // Exercise both envelope Unknown and terminal Partial contributions.+ if index == 3 {+ facts[0].completeness_delta = delta;+ }+ if index == 25+ && let ExecutionFactV1::RequestFinished {+ facts_completeness, ..+ } = &mut facts.last_mut().unwrap().fact+ {+ *facts_completeness = FactsCompleteness::Partial;+ }+ let before = crate::store::fact_log::SESSION_LOADS.with(|n| n.get());+ for fact in facts {+ let outcome = offer_fact(&writer, &channel, fact);+ assert!(matches!(outcome, WriterCycleOutcome::Ack(_)), "{outcome:?}");+ }+ assert_eq!(+ crate::store::fact_log::SESSION_LOADS.with(|n| n.get()),+ before,+ "ordinary ingestion must not hydrate older requests"+ );+ let connection = store.connection.lock();+ let rebuilt = crate::receipt::session_facts_completeness(+ &connection,+ &fixture.workspace,+ &session,+ )+ .unwrap();+ let projected: String = connection.query_row(+ "SELECT facts_completeness FROM sessions WHERE workspace_id=?1 AND session_id=?2",+ rusqlite::params![fixture.workspace.as_str(), session.as_str()], |row| row.get(0),+ ).unwrap();+ assert_eq!(+ serde_json::to_value(rebuilt).unwrap().as_str().unwrap(),+ projected+ );+ }+ }+}++#[test]+fn retention_removes_old_partial_contribution_before_next_append() {+ let temp = tempfile::tempdir().unwrap();+ let store = open_store(temp.path());+ let writer = writer(&store);+ let session = Fixture::new("retained-session").session;+ for (name, timestamp) in [+ ("old-partial", 100),+ ("fresh-complete", hiroute_domain::SEVEN_DAYS_MILLIS),+ ] {+ let mut fixture = Fixture::new(name);+ fixture.session = session.clone();+ let channel = fact_channel(&fixture, 512 * 1024);+ let mut facts = request_facts(&fixture, name, finished(name), timestamp);+ if name == "old-partial"+ && let ExecutionFactV1::RequestFinished {+ facts_completeness, ..+ } = &mut facts.last_mut().unwrap().fact+ {+ *facts_completeness = FactsCompleteness::Partial;+ }+ for fact in facts {+ assert!(matches!(+ offer_fact(&writer, &channel, fact),+ WriterCycleOutcome::Ack(_)+ ));+ }+ }+ assert_eq!(+ store+ .expire_request_details(hiroute_domain::SEVEN_DAYS_MILLIS + 1000, 100)+ .unwrap(),+ 1+ );+ let actual: String = store+ .connection+ .lock()+ .query_row(+ "SELECT facts_completeness FROM sessions WHERE session_id=?1",+ [session.as_str()],+ |row| row.get(0),+ )+ .unwrap();+ assert_eq!(actual, "complete");+}diff --git a/crates/observation/src/store/tests/mod.rs b/crates/observation/src/store/tests/mod.rsindex 98b07f3a..7c3174a6 100644--- a/crates/observation/src/store/tests/mod.rs+++ b/crates/observation/src/store/tests/mod.rs@@ -2,6 +2,7 @@ mod blob_gc;mod content_v2;mod fact_regression;mod failure_isolation;+mod incremental_completeness;mod plan_quality;mod retention;mod retention_matrix;diff --git a/tools/e2e-harness/tests/p0_gateway_runtime.rs b/tools/e2e-harness/tests/p0_gateway_runtime.rsindex 289851b3..7ffd968a 100644--- a/tools/e2e-harness/tests/p0_gateway_runtime.rs+++ b/tools/e2e-harness/tests/p0_gateway_runtime.rs@@ -294,9 +294,9 @@ fn forged_generic_error_header_cannot_change_connector_classification() {}#[test]-fn preoutput_native_sse_error_falls_back_but_semantic_output_is_rendered() {+fn successful_sse_headers_commit_even_without_semantic_output_and_never_fallback() {for first_body in [- b"event: response.failed\ndata: {\"type\":\"response.failed\",\"error\":{\"type\":\"server_error\",\"message\":\"retry\"}}\n\n".as_slice(),+ b"event: response.failed\ndata: {\"type\":\"response.failed\",\"response\":{\"id\":\"failed-stream\",\"model\":\"runtime-native\",\"status\":\"failed\",\"output\":[],\"error\":{\"type\":\"server_error\",\"message\":\"retry\"},\"usage\":{\"input_tokens\":7,\"output_tokens\":2}}}\n\n".as_slice(),b"event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"empty-stream\",\"model\":\"runtime-native\"}}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"empty-stream\",\"model\":\"runtime-native\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":1,\"output_tokens\":0,\"total_tokens\":1}}}\n\n".as_slice(),b": keepalive\n\nevent: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"control-only\",\"model\":\"runtime-native\"}}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"control-only\",\"model\":\"runtime-native\",\"status\":\"completed\",\"output\":[]}}\n\n".as_slice(),] {@@ -308,15 +308,64 @@ fn preoutput_native_sse_error_falls_back_but_semantic_output_is_rendered() {status: 200,body: RESPONSES_STREAM_OK,}]);- let fixture = RuntimeFixture::launch(&[&first, &second], 2);+ let fixture = RuntimeFixture::launch_with_observation(+ &[&first, &second], 2, ObservationFaults::healthy());let response =fixture.request_body(br#"{"model":"runtime-model","input":"hello","stream":true}"#);assert_eq!(response.status, 200);assert_eq!(first.calls(), 1);- assert_eq!(second.calls(), 1);- assert!(String::from_utf8_lossy(&response.body).contains("response.output_text.delta"));+ assert_eq!(second.calls(), 0);+ assert!(!String::from_utf8_lossy(&response.body).contains("response.output_text.delta"));+ assert!(String::from_utf8_lossy(&response.body).contains(+ if first_body.starts_with(b"event: response.failed") {+ "response.failed"+ } else {+ "response.completed"+ }+ ));+ if first_body.starts_with(b"event: response.failed") {+ let facts = wait_execution_facts(&fixture, 1);+ assert!(facts.iter().any(|row|+ row.pointer("/fact/kind").and_then(|v| v.as_str()) == Some("usage_and_cache")+ && row.pointer("/fact/input_tokens").and_then(|v| v.as_u64()) == Some(7)+ && row.pointer("/fact/output_tokens").and_then(|v| v.as_u64()) == Some(2)+ ), "failed stream must retain reported usage: {facts:?}");+ }+ }+}++#[test]+fn successful_stream_forwards_unknown_events_before_any_semantic_output() {+ use std::io::Read;+ let unknown = b"event: response.vendor_extension\ndata: {\"opaque\":\"early-event\"}\n\n";+ let provider = NativeProvider::start(vec![ProviderReply::StreamDrip {+ status: 200,+ chunks: vec![unknown.to_vec(), RESPONSES_STREAM_OK.to_vec()],+ interval: Duration::from_secs(5),+ }]);+ let fixture = RuntimeFixture::launch(&[&provider], 1);+ let mut stream = open_request(+ fixture.address,+ "POST",+ "/v1/responses",+ &[("X-HiRoute-Token", "runtime-token")],+ br#"{"model":"runtime-model","input":"hello","stream":true}"#,+ );+ stream+ .set_read_timeout(Some(Duration::from_secs(2)))+ .unwrap();+ let mut received = Vec::new();+ let mut chunk = [0u8; 4096];+ while !received.windows(unknown.len()).any(|part| part == unknown) {+ let count = stream+ .read(&mut chunk)+ .expect("unknown event must arrive before the provider resumes");+ assert!(count > 0, "stream ended before its unknown event");+ received.extend_from_slice(&chunk[..count]);}+ assert!(received.starts_with(b"HTTP/1.1 200"));+ assert_eq!(provider.calls(), 1);}#[test]