Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

fix: forward accepted streams promptly and bound observation ingestion (#38)

澄潭7b3b08e06ab0
diff --git a/crates/daemon/tests/support/standalone_headless_product.py b/crates/daemon/tests/support/standalone_headless_product.py
index a82a1374..b748a1db 100644
--- a/crates/daemon/tests/support/standalone_headless_product.py
+++ b/crates/daemon/tests/support/standalone_headless_product.py
@@ -587,8 +587,19 @@ def run(repository):
snapshot = product.cli('compute', 'list')['data']
native_source = next(source for source in snapshot['sources']
if source['display_name'] == 'Headless Native API')
+ # Make the price presentation's seconds-resolution evaluation clock advance.
+ time.sleep(1.05)
shown = product.cli('compute', 'show', native_source['source_id'])['data']
- assert shown['sources'] == [native_source]
+ assert len(shown['sources']) == 1
+ comparable = json.loads(json.dumps(shown['sources'][0]))
+ assert len(comparable['models']) == len(native_source['models'])
+ for original, current in zip(native_source['models'], comparable['models']):
+ earlier = original['presentation']['evaluated_at_ms']
+ later = current['presentation']['evaluated_at_ms']
+ assert isinstance(earlier, int) and isinstance(later, int) and 0 < earlier < later
+ # Only query evaluation time changes; every saved/runtime field still matches.
+ current['presentation']['evaluated_at_ms'] = earlier
+ assert comparable == native_source
binding_id = native_source['models'][0]['binding_id']
product.cli('protected-input', 'release', '--candidate', candidate_ref)
diff --git a/crates/diagnostics/src/event/mod.rs b/crates/diagnostics/src/event/mod.rs
index 7cd2c0eb..7bad2924 100644
--- a/crates/diagnostics/src/event/mod.rs
+++ b/crates/diagnostics/src/event/mod.rs
@@ -60,6 +60,7 @@ pub enum DiagnosticEvent {
AttemptEnd(AttemptEnd),
UpstreamWire(UpstreamWire),
ReasoningCleanup(ReasoningCleanup),
+ ResponseFailure(ResponseFailure),
Fallback(Fallback),
SemanticCommit(SemanticCommit),
RequestCancel(RequestCancel),
@@ -152,6 +153,7 @@ impl DiagnosticEvent {
DiagnosticEvent::SemanticCommit(_) => Info,
DiagnosticEvent::ModelStage(_) | DiagnosticEvent::UpstreamWire(_) => Debug,
DiagnosticEvent::ReasoningCleanup(_) => Warn,
+ DiagnosticEvent::ResponseFailure(_) => Warn,
DiagnosticEvent::ContentCaptureEnd(end) => match end.outcome {
CaptureOutcome::Success => Info,
CaptureOutcome::Abort => Warn,
@@ -217,6 +219,7 @@ impl DiagnosticEvent {
DiagnosticEvent::AttemptEnd(_) => "attempt_end",
DiagnosticEvent::UpstreamWire(_) => "upstream_wire",
DiagnosticEvent::ReasoningCleanup(_) => "reasoning_cleanup",
+ DiagnosticEvent::ResponseFailure(_) => "response_failure",
DiagnosticEvent::Fallback(_) => "fallback",
DiagnosticEvent::SemanticCommit(_) => "semantic_commit",
DiagnosticEvent::RequestCancel(_) => "request_cancel",
diff --git a/crates/diagnostics/src/event/model.rs b/crates/diagnostics/src/event/model.rs
index 0d997620..76111b45 100644
--- a/crates/diagnostics/src/event/model.rs
+++ b/crates/diagnostics/src/event/model.rs
@@ -85,6 +85,43 @@ pub enum ReasoningCleanupReason {
ResponseProtocolProjection,
}
+/// Closed categories only: never serializes provider messages, payloads or field values.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct ResponseFailure {
+ pub request_token: Option<CorrelationToken>,
+ pub attempt_index: u64,
+ pub stage: ResponseFailureStage,
+ pub reason: ResponseFailureReason,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ResponseFailureStage {
+ NativeProjection,
+ Decode,
+ Render,
+ PrefixBuffer,
+ ProviderStream,
+ EndOfStream,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ResponseFailureReason {
+ ResourceLimit,
+ InvalidJson,
+ InvalidSse,
+ InvalidField,
+ InvalidLifecycle,
+ MissingTerminal,
+ DuplicateTerminal,
+ Unrepresentable,
+ NoSemanticOutput,
+ ProviderRejected,
+ OtherProtocol,
+}
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AttemptOutcome {
diff --git a/crates/gateway/src/adapters/response/client_stream.rs b/crates/gateway/src/adapters/response/client_stream.rs
index 474042cf..b2909f1a 100644
--- a/crates/gateway/src/adapters/response/client_stream.rs
+++ b/crates/gateway/src/adapters/response/client_stream.rs
@@ -61,6 +61,10 @@ pub struct IncrementalClientSseRenderer {
}
impl IncrementalClientSseRenderer {
+ pub(crate) fn usage(&self) -> &ModelUsage {
+ &self.usage
+ }
+
pub(crate) fn with_budget(
mut self,
budget: hiroute_gateway_core::runtime::body::StreamBudget,
diff --git a/crates/gateway/src/adapters/response/mod.rs b/crates/gateway/src/adapters/response/mod.rs
index b10010ba..061a4b8a 100644
--- a/crates/gateway/src/adapters/response/mod.rs
+++ b/crates/gateway/src/adapters/response/mod.rs
@@ -16,6 +16,8 @@ mod wire;
#[cfg(test)]
#[path = "native_passthrough_tests.rs"]
mod native_passthrough_tests;
+#[cfg(test)]
+mod native_reasoning_tests;
#[cfg(test)]
mod size_tests;
diff --git a/crates/gateway/src/adapters/response/native_passthrough_evidence.rs b/crates/gateway/src/adapters/response/native_passthrough_evidence.rs
index b849e9a8..701f4e4f 100644
--- a/crates/gateway/src/adapters/response/native_passthrough_evidence.rs
+++ b/crates/gateway/src/adapters/response/native_passthrough_evidence.rs
@@ -259,7 +259,7 @@ impl ProjectionState {
.and_then(Value::as_str);
let expected_type = match kind {
"output_text" | "refusal" => "message",
- "reasoning_summary_text" => "reasoning",
+ "reasoning_summary_text" | "reasoning_text" => "reasoning",
"function_call_arguments" => "function_call",
"custom_tool_call_input" => "custom_tool_call",
_ => unreachable!("tracked delta kind"),
@@ -309,7 +309,12 @@ fn added_text_prefixes(item: &Map<String, Value>) -> Vec<AddedTextPrefix> {
if let Some(content) = item.get("content").and_then(Value::as_array) {
for (position, block) in content.iter().enumerate() {
if let Some(text) = block.get("text").and_then(Value::as_str) {
- capture(position, "output_text", text);
+ let kind = if item.get("type").and_then(Value::as_str) == Some("reasoning") {
+ "reasoning_text"
+ } else {
+ "output_text"
+ };
+ capture(position, kind, text);
}
if let Some(refusal) = block.get("refusal").and_then(Value::as_str) {
capture(position, "refusal", refusal);
@@ -341,6 +346,8 @@ fn delta_kind(event: &str) -> Option<(&'static str, &'static str, Option<&'stati
Some(("refusal", "refusal", Some("content_index")))
} else if event.starts_with("response.reasoning_summary_text.") {
Some(("reasoning_summary_text", "text", Some("summary_index")))
+ } else if event.starts_with("response.reasoning_text.") {
+ Some(("reasoning_text", "text", Some("content_index")))
} else if event.starts_with("response.function_call_arguments.") {
Some(("function_call_arguments", "arguments", None))
} else if event.starts_with("response.custom_tool_call_input.") {
@@ -352,15 +359,11 @@ fn delta_kind(event: &str) -> Option<(&'static str, &'static str, Option<&'stati
fn final_text<'a>(item: &'a Map<String, Value>, position: usize, kind: &str) -> Option<&'a str> {
match kind {
- "output_text" | "refusal" => item
+ "output_text" | "refusal" | "reasoning_text" => item
.get("content")?
.as_array()?
.get(position)?
- .get(if kind == "output_text" {
- "text"
- } else {
- "refusal"
- })?
+ .get(if kind != "refusal" { "text" } else { "refusal" })?
.as_str(),
"reasoning_summary_text" => item
.get("summary")?
diff --git a/crates/gateway/src/adapters/response/native_passthrough_helpers.rs b/crates/gateway/src/adapters/response/native_passthrough_helpers.rs
index 80fa82e0..57f1b566 100644
--- a/crates/gateway/src/adapters/response/native_passthrough_helpers.rs
+++ b/crates/gateway/src/adapters/response/native_passthrough_helpers.rs
@@ -57,6 +57,8 @@ pub(super) fn responses_semantic_event(event: &str) -> bool {
| "response.refusal.done"
| "response.reasoning_summary_text.delta"
| "response.reasoning_summary_text.done"
+ | "response.reasoning_text.delta"
+ | "response.reasoning_text.done"
| "response.function_call_arguments.delta"
| "response.function_call_arguments.done"
| "response.custom_tool_call_input.delta"
@@ -142,6 +144,16 @@ pub(super) fn response_item_is_semantic(item: &Map<String, Value>) -> bool {
item.get("encrypted_content")
.and_then(Value::as_str)
.is_some_and(|value| !value.is_empty())
+ || item
+ .get("content")
+ .and_then(Value::as_array)
+ .is_some_and(|content| {
+ content.iter().any(|part| {
+ part.get("text")
+ .and_then(Value::as_str)
+ .is_some_and(|text| !text.is_empty())
+ })
+ })
|| item
.get("summary")
.and_then(Value::as_array)
diff --git a/crates/gateway/src/adapters/response/native_passthrough_sse.rs b/crates/gateway/src/adapters/response/native_passthrough_sse.rs
index 4bb0d2d1..b4354fa6 100644
--- a/crates/gateway/src/adapters/response/native_passthrough_sse.rs
+++ b/crates/gateway/src/adapters/response/native_passthrough_sse.rs
@@ -22,7 +22,7 @@ pub(super) fn feed_projected_sse(
end_stream: bool,
) -> Result<Vec<NativeProjectedUnit>, ProtocolAdapterError> {
let charged = ChargedBytes::copy_from_opaque(budget, MemoryRole::TransportInflight, bytes)
- .map_err(|error| ModelIrError::InvalidSse(error.to_string()))?;
+ .map_err(|_| ModelIrError::BufferLimit(bytes.len()))?;
let mut metadata = VecDeque::new();
let mut sink = ProjectedSink::default();
let mut visitor = ProjectionVisitor {
@@ -41,7 +41,10 @@ pub(super) fn feed_projected_sse(
)
.into());
}
- outcome.map_err(|error| ModelIrError::InvalidSse(error.to_string()))?;
+ outcome.map_err(|error| match error {
+ SseError::BudgetExceeded => ModelIrError::BufferLimit(bytes.len()),
+ other => ModelIrError::InvalidSse(other.to_string()),
+ })?;
if sink.output.len() != metadata.len() {
return Err(
ModelIrError::InvalidSse("native projector lost SSE output metadata".into()).into(),
diff --git a/crates/gateway/src/adapters/response/native_reasoning_tests.rs b/crates/gateway/src/adapters/response/native_reasoning_tests.rs
new file mode 100644
index 00000000..1c691ee8
--- /dev/null
+++ b/crates/gateway/src/adapters/response/native_reasoning_tests.rs
@@ -0,0 +1,103 @@
+use super::native_passthrough::{NativeResponseProjector, NativeTerminalOutcome};
+use crate::server::{
+ core_runtime::profiles::{CandidateProtocolProfile, fixed_reasoning},
+ request_plan::IngressProtocol,
+};
+use hiroute_gateway_core::runtime::body::BudgetTree;
+use serde_json::{Value, json};
+
+fn wire(value: Value) -> Vec<u8> {
+ format!("data: {value}\n\n").into_bytes()
+}
+
+#[test]
+fn long_plain_reasoning_commits_on_first_delta_and_keeps_terminal_usage() {
+ let budget = BudgetTree::new(8 * 1024 * 1024, 8 * 1024 * 1024)
+ .unwrap()
+ .stream(8 * 1024 * 1024)
+ .unwrap();
+ let profile = CandidateProtocolProfile::exact_portable_path(
+ IngressProtocol::Responses,
+ IngressProtocol::Responses,
+ "physical",
+ fixed_reasoning("fixed"),
+ );
+ let mut projector =
+ NativeResponseProjector::new_for_attempt(&profile, true, "alias".into(), None, budget)
+ .unwrap();
+ let reasoning = |text: &str| {
+ json!({"type":"reasoning","id":"r","summary":[],
+ "content":[{"type":"reasoning_text","text":text}]})
+ };
+ projector.feed(&wire(json!({"type":"response.created","response":{"id":"resp","model":"physical","status":"in_progress"}})), false).unwrap();
+ projector
+ .feed(
+ &wire(
+ json!({"type":"response.output_item.added","output_index":0,"item":reasoning("")}),
+ ),
+ false,
+ )
+ .unwrap();
+ let chunk = "think ".repeat(512);
+ let mut text = String::new();
+ for _ in 0..800 {
+ text.push_str(&chunk);
+ let event = wire(
+ json!({"type":"response.reasoning_text.delta","item_id":"r","output_index":0,"content_index":0,"delta":chunk}),
+ );
+ let units = projector.feed(&event, false).unwrap();
+ assert!(
+ units.iter().any(|unit| unit.semantic),
+ "plain reasoning is output, not an indefinitely buffered opaque prefix"
+ );
+ assert_eq!(
+ units
+ .iter()
+ .flat_map(|u| u.bytes.iter().copied())
+ .collect::<Vec<_>>(),
+ event
+ );
+ }
+ assert!(text.len() > 2 * 1024 * 1024);
+ let terminal = wire(json!({"type":"response.completed","response":{
+ "id":"resp","model":"physical","status":"completed","output":[reasoning(&text)],
+ "usage":{"input_tokens":100,"output_tokens":800,"total_tokens":900}}}));
+ for chunk in terminal.chunks(16 * 1024) {
+ projector.feed(chunk, false).unwrap();
+ }
+ let last = projector.feed(&[], true).unwrap();
+ assert_eq!(
+ last.last().unwrap().terminal,
+ Some(NativeTerminalOutcome::Complete)
+ );
+ assert_eq!(projector.usage().output_tokens, Some(800));
+}
+
+#[test]
+fn failed_response_preserves_reported_usage() {
+ let budget = BudgetTree::new(1024 * 1024, 1024 * 1024)
+ .unwrap()
+ .stream(1024 * 1024)
+ .unwrap();
+ let profile = CandidateProtocolProfile::exact_portable_path(
+ IngressProtocol::Responses,
+ IngressProtocol::Responses,
+ "physical",
+ fixed_reasoning("fixed"),
+ );
+ let mut projector =
+ NativeResponseProjector::new_for_attempt(&profile, true, "alias".into(), None, budget)
+ .unwrap();
+ let units = projector
+ .feed(
+ &wire(json!({"type":"response.failed","response":{
+ "id":"resp","status":"failed","model":"physical","output":[],
+ "error":{"code":"server_error","message":"private body"},
+ "usage":{"input_tokens":100,"output_tokens":2}}})),
+ true,
+ )
+ .unwrap();
+ assert!(units.iter().any(|unit| unit.failure.is_some()));
+ assert_eq!(projector.usage().input_tokens, Some(100));
+ assert_eq!(projector.usage().output_tokens, Some(2));
+}
diff --git a/crates/gateway/src/observation/request.rs b/crates/gateway/src/observation/request.rs
index e054cd61..50d2525c 100644
--- a/crates/gateway/src/observation/request.rs
+++ b/crates/gateway/src/observation/request.rs
@@ -243,6 +243,32 @@ impl RequestObservation {
.token(CorrelationDomain::Attempt, attempt_id)
}
+ pub(crate) fn response_failure(
+ &self,
+ stage: hiroute_diagnostics::event::ResponseFailureStage,
+ reason: hiroute_diagnostics::event::ResponseFailureReason,
+ ) {
+ let state = self.inner.state.lock().unwrap_or_else(|e| e.into_inner());
+ let attempt = state
+ .accepted_attempt
+ .as_ref()
+ .or(state.current_attempt.as_ref())
+ .or(state.pending_attempt.as_ref());
+ let ordinal = attempt
+ .map(|a| a.ordinal)
+ .filter(|n| *n != 0)
+ .unwrap_or(state.next_attempt_ordinal);
+ drop(state);
+ self.emit_diagnostic(DiagnosticEvent::ResponseFailure(
+ hiroute_diagnostics::event::ResponseFailure {
+ request_token: self.inner.request_token,
+ attempt_index: u64::from(ordinal),
+ stage,
+ reason,
+ },
+ ));
+ }
+
pub(crate) fn reasoning_cleanup(
&self,
reason: hiroute_diagnostics::event::ReasoningCleanupReason,
diff --git a/crates/gateway/src/observation/tests.rs b/crates/gateway/src/observation/tests.rs
index a2086f07..d852fbd3 100644
--- a/crates/gateway/src/observation/tests.rs
+++ b/crates/gateway/src/observation/tests.rs
@@ -189,6 +189,10 @@ fn request_lifecycle_records_typed_diagnostics_without_raw_identities() {
assert!(request.accept_current("frame:test", 128).is_some());
request.reasoning_cleanup(ReasoningCleanupReason::ResponseProtocolProjection, 1, None);
request.reasoning_cleanup(ReasoningCleanupReason::ResponseProtocolProjection, 3, None);
+ request.response_failure(
+ hiroute_diagnostics::event::ResponseFailureStage::NativeProjection,
+ hiroute_diagnostics::event::ResponseFailureReason::ResourceLimit,
+ );
assert_eq!(
request
.lock_state()
@@ -208,6 +212,7 @@ fn request_lifecycle_records_typed_diagnostics_without_raw_identities() {
"semantic_commit",
"request_end",
"model_stage",
+ "response_failure",
] {
assert!(
log.contains(&format!("\"{kind}\":")),
diff --git a/crates/gateway/src/publication/tests.rs b/crates/gateway/src/publication/tests.rs
index 45272d16..f27e9dd7 100644
--- a/crates/gateway/src/publication/tests.rs
+++ b/crates/gateway/src/publication/tests.rs
@@ -37,9 +37,10 @@ fn domain_gateway_profile_roundtrip_preserves_model_evidence() {
fn published_attempt_phases_default_to_ten_minutes() {
let aggregate = super::compiler::compile(&snapshot(1, "renderer")).unwrap();
#[cfg(feature = "e2e-test-control")]
- let override_timeout = crate::server::test_control::attempt_timeout_override();
+ let expected_timeout = crate::server::test_control::attempt_timeout_override()
+ .unwrap_or(Duration::from_secs(10 * 60));
#[cfg(not(feature = "e2e-test-control"))]
- let override_timeout: Option<Duration> = None;
+ let expected_timeout = Duration::from_secs(10 * 60);
let plans = aggregate
.envelope
.attempt_plan_index_handle
@@ -47,18 +48,9 @@ fn published_attempt_phases_default_to_ten_minutes() {
.collect::<Vec<_>>();
assert_eq!(plans.len(), 2);
for (_, plan) in plans {
- assert_eq!(
- plan.timeouts.request_write,
- override_timeout.unwrap_or(Duration::from_secs(10 * 60))
- );
- assert_eq!(
- plan.timeouts.first_byte,
- override_timeout.unwrap_or(Duration::from_secs(10 * 60))
- );
- assert_eq!(
- plan.timeouts.stream_idle,
- override_timeout.unwrap_or(Duration::from_secs(10 * 60))
- );
+ assert_eq!(plan.timeouts.request_write, expected_timeout);
+ assert_eq!(plan.timeouts.first_byte, expected_timeout);
+ assert_eq!(plan.timeouts.stream_idle, expected_timeout);
}
}
diff --git a/crates/gateway/src/runtime/driver.rs b/crates/gateway/src/runtime/driver.rs
index 9128174f..04bdbdde 100644
--- a/crates/gateway/src/runtime/driver.rs
+++ b/crates/gateway/src/runtime/driver.rs
@@ -40,6 +40,7 @@ use super::state::{
mod materialization;
mod response;
+mod response_diagnostics;
mod selection;
pub(crate) use materialization::resolve_target;
@@ -181,10 +182,8 @@ pub struct ProductionAttemptState {
projector: Option<Box<adapters::NativeResponseProjector>>,
prefix: Option<ChargedBodyQueue>,
prefix_terminal_chunks: Option<usize>,
- decoder_budget: Option<response::PrecommitDecoderBudget>,
budget: StreamBudget,
semantic_seen: bool,
- terminal_seen: bool,
semantic_terminal: Option<SemanticTerminalOutcome>,
retry_after: Option<std::time::Duration>,
runtime_state_authority: RuntimeStateAuthoritySignal,
@@ -206,7 +205,6 @@ pub struct ProductionReadiness {
decoder: Option<adapters::NativeResponseDecoder>,
renderer: Option<adapters::IncrementalClientSseRenderer>,
projector: Option<Box<adapters::NativeResponseProjector>>,
- _decoder_budget: Option<response::PrecommitDecoderBudget>,
_chat_tool_projection_budget: Option<Reservation>,
budget: StreamBudget,
streaming: bool,
diff --git a/crates/gateway/src/runtime/driver/materialization.rs b/crates/gateway/src/runtime/driver/materialization.rs
index 244ff9db..3c620f20 100644
--- a/crates/gateway/src/runtime/driver/materialization.rs
+++ b/crates/gateway/src/runtime/driver/materialization.rs
@@ -34,7 +34,7 @@ use crate::server::core_runtime::profiles::{
PlannerCandidateFactsV1,
};
-use super::response::{PrecommitDecoderBudget, connector_error_profile, failure_facts};
+use super::response::{connector_error_profile, failure_facts};
#[path = "materialization/lease_target.rs"]
mod lease_target;
@@ -296,7 +296,6 @@ pub(super) async fn materialize_attempt(
plan.precommit_event_capacity,
)
.map_err(safe_error)?;
- let decoder_budget = PrecommitDecoderBudget::new(context.budget)?;
let lease_target = lease_target::for_request(&execution, profile)?;
let lease_target_digest = hiroute_domain::CanonicalDigest::of(&lease_target)
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
@@ -564,10 +563,8 @@ pub(super) async fn materialize_attempt(
projector: None,
prefix: Some(prefix),
prefix_terminal_chunks: None,
- decoder_budget: Some(decoder_budget),
budget: context.budget.clone(),
semantic_seen: false,
- terminal_seen: false,
semantic_terminal: None,
retry_after: None,
runtime_state_authority,
diff --git a/crates/gateway/src/runtime/driver/response.rs b/crates/gateway/src/runtime/driver/response.rs
index 33149cd7..7596feac 100644
--- a/crates/gateway/src/runtime/driver/response.rs
+++ b/crates/gateway/src/runtime/driver/response.rs
@@ -1,3 +1,7 @@
+use super::response_diagnostics;
+use hiroute_diagnostics::event::{
+ ResponseFailureReason as FailureReason, ResponseFailureStage as FailureStage,
+};
use std::sync::Arc;
use hiroute_gateway_core::core::filter::LocalReply;
@@ -5,7 +9,7 @@ use hiroute_gateway_core::runtime::attempt::{
ChargedResponseHead, Disposition, PrecommitEvent, PublishedDisposition,
};
use hiroute_gateway_core::runtime::body::{
- BodyMetadataOwner, ChargedBodyQueue, ChargedBytes, MemoryRole, Reservation, StreamBudget,
+ BodyMetadataOwner, ChargedBodyQueue, ChargedBytes, MemoryRole, StreamBudget,
};
use hiroute_gateway_core::runtime::driver::{
AcceptedBodyFrame, ClassifiedAttemptResult, NormalizedAttemptLocalReply,
@@ -34,14 +38,6 @@ use super::{
ProductionReadiness, SemanticTerminalOutcome, label, safe_error,
};
-pub(super) struct PrecommitDecoderBudget {
- budget: StreamBudget,
- raw_bytes: usize,
- charged_bytes: usize,
- charges: Vec<Reservation>,
- _metadata: Reservation,
-}
-
fn note_response_reasoning_loss(removed: usize) {
if removed > 0
&& let Some(observation) = crate::server::core_runtime::observation::active_request()
@@ -54,62 +50,6 @@ fn note_response_reasoning_loss(removed: usize) {
}
}
-impl PrecommitDecoderBudget {
- const MAX_CHARGE_STEPS: usize = usize::BITS as usize + 1;
-
- pub(super) fn new(budget: &StreamBudget) -> Result<Self, Arc<str>> {
- let metadata_bytes = Self::MAX_CHARGE_STEPS
- .checked_mul(std::mem::size_of::<Reservation>())
- .ok_or_else(|| Arc::from("native decoder metadata budget overflow"))?;
- let metadata = budget
- .reserve(MemoryRole::ResponsePrefix, metadata_bytes)
- .map_err(safe_error)?;
- let charges = Vec::with_capacity(Self::MAX_CHARGE_STEPS);
- Ok(Self {
- budget: budget.clone(),
- raw_bytes: 0,
- charged_bytes: 0,
- charges,
- _metadata: metadata,
- })
- }
-
- fn charge_frame(&mut self, bytes: usize) -> Result<(), Arc<str>> {
- let raw_bytes = self
- .raw_bytes
- .checked_add(bytes)
- .ok_or_else(|| Arc::from("native precommit decoder byte limit overflow"))?;
- // Vec's amortized retained capacity is strictly below twice the live
- // byte length after its minimum allocation. Shadow-charge that upper
- // bound before the native decoder is allowed to copy this frame.
- let target_charge = if raw_bytes == 0 {
- 0
- } else {
- raw_bytes
- .checked_mul(2)
- .and_then(|bytes| bytes.checked_add(8))
- .ok_or_else(|| Arc::from("native precommit decoder charge overflow"))?
- };
- if target_charge > self.charged_bytes {
- // Transport chunk boundaries are arbitrary and must not become a
- // correctness limit. Grow the accounting reservation geometrically
- // so a long sequence of tiny chunks uses bounded metadata while the
- // request memory budget remains enforced.
- let next_charge = target_charge
- .checked_next_power_of_two()
- .unwrap_or(target_charge);
- self.charges.push(
- self.budget
- .reserve(MemoryRole::ResponsePrefix, next_charge - self.charged_bytes)
- .map_err(safe_error)?,
- );
- self.charged_bytes = next_charge;
- }
- self.raw_bytes = raw_bytes;
- Ok(())
- }
-}
-
pub(super) fn classify_precommit(
state: &mut ProductionAttemptState,
event: PrecommitEvent,
@@ -166,78 +106,29 @@ pub(super) fn classify_precommit(
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?,
);
}
+ if state.streaming && status.is_success() {
+ // HTTP acceptance, not the arrival of a recognized semantic event,
+ // closes the transparent retry window. Body parsing stays bounded
+ // and executes only on the accepted response path.
+ return Ok(PrecommitClassification::classified(
+ ClassifiedAttemptResult {
+ facts: ProviderClassificationFacts {
+ http_status: Some(status),
+ readiness: label("response_headers"),
+ retryability: RetryabilityFact::NonRetryable,
+ ..ProviderClassificationFacts::default()
+ },
+ readiness: take_readiness(state, status, "text/event-stream", false)?,
+ },
+ ));
+ }
Ok(PrecommitClassification::pending())
}
PrecommitEvent::Body(bytes) => {
- state
- .decoder_budget
- .as_mut()
- .ok_or_else(|| Arc::from("native decoder budget owner is unavailable"))?
- .charge_frame(bytes.retained_capacity())?;
let status = state
.response_status
.ok_or_else(|| Arc::from("provider body arrived before final response head"))?;
- if state.streaming && status.is_success() {
- let outcome = match if state.native_output {
- project_native_chunk_state(state, bytes.bytes(), false)
- } else {
- decode_stream_chunk_state(state, bytes.bytes(), false)
- } {
- Ok(outcome) => outcome,
- Err(_) => {
- drop(bytes);
- return classify_state_failure(
- state,
- RawAttemptFailure::Protocol,
- StatusCode::BAD_GATEWAY,
- );
- }
- };
- drop(bytes);
- if let Some(error) = outcome.failure {
- return classify_model_error(state, error, status, true);
- }
- if !state.native_output && outcome.terminal && outcome.semantic {
- // Chat/Messages have already supplied their native terminal.
- // Validate this fragment's tail before publishing the
- // translated terminal, then close the downstream stream
- // with the last queued output rather than waiting for the
- // upstream transport to close as a separate event.
- if finish_native_stream_on_terminal(&mut state.decoder).is_err() {
- return classify_state_failure(
- state,
- RawAttemptFailure::Protocol,
- StatusCode::BAD_GATEWAY,
- );
- }
- let terminal_chunks = state.prefix.as_ref().map_or(0, ChargedBodyQueue::len);
- if terminal_chunks == 0 {
- return classify_state_failure(
- state,
- RawAttemptFailure::Protocol,
- StatusCode::BAD_GATEWAY,
- );
- }
- state.prefix_terminal_chunks = Some(terminal_chunks);
- }
- if outcome.semantic {
- return Ok(PrecommitClassification::classified(
- ClassifiedAttemptResult {
- facts: semantic_response_facts(status),
- readiness: take_readiness(
- state,
- StatusCode::OK,
- "text/event-stream",
- // Native protocol terminal metadata, rather
- // than upstream transport EOF, marks the
- // eventual downstream EOS frame.
- false,
- )?,
- },
- ));
- }
- Ok(PrecommitClassification::pending())
- } else if state.native_output && status.is_success() {
+ if state.native_output && status.is_success() {
let projected = state
.projector
.as_mut()
@@ -278,51 +169,7 @@ pub(super) fn classify_precommit(
let status = state
.response_status
.ok_or_else(|| Arc::from("provider ended before final response head"))?;
- if state.streaming && status.is_success() {
- let outcome = match if state.native_output {
- project_native_chunk_state(state, &[], true)
- } else {
- decode_stream_chunk_state(state, &[], true)
- } {
- Ok(outcome) => outcome,
- Err(_) => {
- return classify_state_failure(
- state,
- RawAttemptFailure::Protocol,
- StatusCode::BAD_GATEWAY,
- );
- }
- };
- if let Some(error) = outcome.failure {
- return classify_model_error(state, error, status, true);
- }
- if !state.semantic_seen {
- return classify_state_failure(
- state,
- RawAttemptFailure::Protocol,
- StatusCode::BAD_GATEWAY,
- );
- }
- if !state.native_output {
- state
- .decoder
- .take()
- .ok_or_else(|| Arc::from("native response decoder is unavailable"))?
- .finish()
- .map_err(|_| Arc::from("native response terminal is malformed"))?;
- }
- Ok(PrecommitClassification::classified(
- ClassifiedAttemptResult {
- facts: semantic_response_facts(status),
- readiness: take_readiness(
- state,
- StatusCode::OK,
- "text/event-stream",
- true,
- )?,
- },
- ))
- } else if state.native_output && status.is_success() {
+ if state.native_output && status.is_success() {
let units = match state
.projector
.as_mut()
@@ -348,7 +195,7 @@ pub(super) fn classify_precommit(
push_native_prefix_unit(state, unit.bytes, unit.terminal.is_some())?;
}
if let Some(error) = failure {
- return classify_model_error(state, error, status, false);
+ return classify_model_error(state, error, status);
}
if !state.semantic_seen || state.semantic_terminal.is_none() {
return classify_state_failure(
@@ -393,7 +240,7 @@ pub(super) fn classify_precommit(
};
state.semantic_terminal = semantic_terminal_for_response(&decoded.response);
if let Some(error) = decoded.response.error.clone() {
- return classify_model_error(state, error, status, false);
+ return classify_model_error(state, error, status);
}
if !decoded
.events
@@ -488,7 +335,13 @@ pub(super) fn finalize_attempt_facts(
let usage = readiness
.and_then(|readiness| readiness.projector.as_ref())
.map(|projector| projector.usage())
- .or_else(|| state.projector.as_ref().map(|projector| projector.usage()));
+ .or_else(|| state.projector.as_ref().map(|projector| projector.usage()))
+ .or_else(|| {
+ readiness
+ .and_then(|r| r.renderer.as_ref())
+ .map(|r| r.usage())
+ })
+ .or_else(|| state.renderer.as_ref().map(|r| r.usage()));
if let Some(usage) = usage.filter(|usage| !usage.is_empty()) {
facts.usage = Some(model_usage_fact(usage));
}
@@ -806,21 +659,13 @@ fn classify_model_error(
state: &mut ProductionAttemptState,
error: ModelError,
response_status: StatusCode,
- stream: bool,
) -> Result<PrecommitClassification<ProductionReadiness, ProductionDecodedSse>, Arc<str>> {
let kind = sanitized_provider_kind(&state.profile, &error);
let status = error.status.unwrap_or(response_status.as_u16());
- let raw = if stream && response_status.is_success() {
- RawAttemptFailure::Stream {
- kind,
- retry_after: state.retry_after,
- }
- } else {
- RawAttemptFailure::Http {
- status,
- kind,
- retry_after: state.retry_after,
- }
+ let raw = RawAttemptFailure::Http {
+ status,
+ kind,
+ retry_after: state.retry_after,
};
let local_status = StatusCode::from_u16(status)
.ok()
@@ -885,7 +730,6 @@ fn take_readiness(
decoder: state.decoder.take(),
renderer: state.renderer.take(),
projector: state.projector.take(),
- _decoder_budget: state.decoder_budget.take(),
_chat_tool_projection_budget: state.chat_tool_projection_budget.take(),
budget: state.budget.clone(),
streaming: state.streaming,
@@ -895,43 +739,6 @@ fn take_readiness(
})
}
-#[derive(Default)]
-struct StreamDecodeOutcome {
- semantic: bool,
- terminal: bool,
- failure: Option<ModelError>,
-}
-
-fn project_native_chunk_state(
- state: &mut ProductionAttemptState,
- bytes: &[u8],
- end_stream: bool,
-) -> Result<StreamDecodeOutcome, Arc<str>> {
- let units = state
- .projector
- .as_mut()
- .ok_or_else(|| Arc::from("native response projector is unavailable"))?
- .feed(bytes, end_stream)
- .map_err(|_| Arc::from("native stream response is malformed"))?;
- let mut outcome = StreamDecodeOutcome::default();
- for unit in units {
- let semantic_before = state.semantic_seen || outcome.semantic;
- if let Some(terminal) = unit.terminal {
- let terminal = native_terminal(terminal);
- state.semantic_terminal = Some(terminal);
- state.terminal_seen = true;
- outcome.terminal = true;
- }
- if unit.failure.is_some() && !semantic_before {
- outcome.failure = unit.failure;
- }
- outcome.semantic |= unit.semantic;
- push_native_prefix_unit(state, unit.bytes, unit.terminal.is_some())?;
- }
- state.semantic_seen |= outcome.semantic;
- Ok(outcome)
-}
-
fn project_native_chunk_readiness(
readiness: &mut ProductionReadiness,
bytes: &[u8],
@@ -942,13 +749,19 @@ fn project_native_chunk_readiness(
.as_mut()
.ok_or_else(|| Arc::from("accepted native projector is unavailable"))?
.feed(bytes, end_stream)
- .map_err(|_| Arc::from("accepted native stream is malformed"))?;
+ .map_err(|error| response_diagnostics::adapter(FailureStage::NativeProjection, error))?;
let mut output = Vec::new();
let mut terminal = false;
for unit in units {
if terminal {
return Err(Arc::from("native stream emitted bytes after terminal"));
}
+ if unit.failure.is_some() {
+ response_diagnostics::note(
+ FailureStage::ProviderStream,
+ FailureReason::ProviderRejected,
+ );
+ }
if let Some(outcome) = unit.terminal {
readiness.semantic_terminal = Some(native_terminal(outcome));
terminal = true;
@@ -964,6 +777,7 @@ fn push_native_prefix_unit(
terminal: bool,
) -> Result<(), Arc<str>> {
if state.prefix_terminal_chunks.is_some() {
+ response_diagnostics::note(FailureStage::PrefixBuffer, FailureReason::InvalidLifecycle);
return Err(Arc::from("native response emitted output after terminal"));
}
push_prefix_bytes(state, bytes)?;
@@ -974,6 +788,7 @@ fn push_native_prefix_unit(
.ok_or_else(|| Arc::from("response prefix owner is unavailable"))?
.len();
if chunks == 0 {
+ response_diagnostics::note(FailureStage::PrefixBuffer, FailureReason::InvalidLifecycle);
return Err(Arc::from("native terminal produced no wire bytes"));
}
state.prefix_terminal_chunks = Some(chunks);
@@ -1004,70 +819,6 @@ fn model_usage_fact(usage: &ModelUsage) -> UsageFact {
}
}
-fn decode_stream_chunk_state(
- state: &mut ProductionAttemptState,
- bytes: &[u8],
- end_stream: bool,
-) -> Result<StreamDecodeOutcome, Arc<str>> {
- let decoder = state
- .decoder
- .as_mut()
- .ok_or_else(|| Arc::from("native stream decoder is unavailable"))?;
- let renderer = state
- .renderer
- .as_mut()
- .ok_or_else(|| Arc::from("client stream renderer is unavailable"))?;
- let prefix = state
- .prefix
- .as_mut()
- .ok_or_else(|| Arc::from("response prefix owner is unavailable"))?;
- let mut status = decoder
- .feed(bytes, end_stream)
- .map_err(|_| Arc::from("native stream response is malformed"))?;
- let mut outcome = StreamDecodeOutcome::default();
- loop {
- for event in decoder.take_events() {
- observe_semantic_terminal(&mut state.semantic_terminal, &event.event);
- let semantic = event.is_semantic_output();
- if let ModelEvent::ResponseFailed { error } = &event.event
- && !state.semantic_seen
- && !outcome.semantic
- {
- outcome.failure = Some(error.clone());
- }
- let rendered_events = renderer
- .push(&event)
- .map_err(|_| Arc::from("client stream event is not representable"))?;
- let emitted = !rendered_events.is_empty();
- for rendered in rendered_events {
- push_queue_bytes(
- prefix,
- &state.budget,
- rendered
- .wire_bytes()
- .map_err(|_| Arc::from("client stream event serialization failed"))?,
- )?;
- }
- note_response_reasoning_loss(renderer.take_reasoning_loss());
- outcome.semantic |= semantic && emitted;
- let terminal = matches!(
- event.event,
- ModelEvent::ResponseCompleted { .. } | ModelEvent::ResponseFailed { .. }
- );
- outcome.terminal |= terminal;
- state.terminal_seen |= terminal;
- }
- if status != adapters::ResponseDecodeStatus::NeedDrain {
- break;
- }
- status = decoder
- .resume()
- .map_err(|_| Arc::from("native stream response drain failed"))?;
- }
- state.semantic_seen |= outcome.semantic;
- Ok(outcome)
-}
-
fn decode_stream_chunk_readiness(
readiness: &mut ProductionReadiness,
bytes: &[u8],
@@ -1083,14 +834,20 @@ fn decode_stream_chunk_readiness(
.ok_or_else(|| Arc::from("accepted client renderer is unavailable"))?;
let mut status = decoder
.feed(bytes, end_stream)
- .map_err(|_| Arc::from("accepted native stream is malformed"))?;
+ .map_err(|error| response_diagnostics::adapter(FailureStage::Decode, error))?;
let mut output = Vec::new();
loop {
for event in decoder.take_events() {
observe_semantic_terminal(&mut readiness.semantic_terminal, &event.event);
+ if matches!(event.event, ModelEvent::ResponseFailed { .. }) {
+ response_diagnostics::note(
+ FailureStage::ProviderStream,
+ FailureReason::ProviderRejected,
+ );
+ }
for rendered in renderer
.push(&event)
- .map_err(|_| Arc::from("accepted client stream event is not representable"))?
+ .map_err(|error| response_diagnostics::adapter(FailureStage::Render, error))?
{
output.extend_from_slice(
&rendered
@@ -1174,7 +931,9 @@ fn push_prefix_bytes(state: &mut ProductionAttemptState, bytes: Vec<u8>) -> Resu
.prefix
.as_mut()
.ok_or_else(|| Arc::from("response prefix owner is unavailable"))?;
- push_queue_bytes(prefix, &state.budget, bytes)
+ push_queue_bytes(prefix, &state.budget, bytes).inspect_err(|_| {
+ response_diagnostics::note(FailureStage::PrefixBuffer, FailureReason::ResourceLimit);
+ })
}
fn push_queue_bytes(
diff --git a/crates/gateway/src/runtime/driver/response_budget_tests.rs b/crates/gateway/src/runtime/driver/response_budget_tests.rs
index 7c4963cb..5989bacf 100644
--- a/crates/gateway/src/runtime/driver/response_budget_tests.rs
+++ b/crates/gateway/src/runtime/driver/response_budget_tests.rs
@@ -79,20 +79,22 @@ fn protocol_prefix_grows_beyond_initial_queue_capacity_and_releases_memory() {
}
#[test]
-fn precommit_decoder_is_bounded_by_bytes_not_transport_chunk_count() {
+fn drained_output_does_not_charge_cumulative_stream_bytes() {
let budget = BudgetTree::new(4 * 1024 * 1024, 4 * 1024 * 1024)
.unwrap()
.stream(4 * 1024 * 1024)
.unwrap();
- let mut decoder = PrecommitDecoderBudget::new(&budget).unwrap();
+ let plan = hiroute_gateway_core::runtime::body::BodyPlan::PassThrough {
+ max_chunk_bytes: 64 * 1024,
+ };
+ let mut queue =
+ ChargedBodyQueue::new(&budget, MemoryRole::ResponsePrefix, &plan, usize::MAX, 1).unwrap();
for _ in 0..1_000 {
- decoder.charge_frame(128).unwrap();
+ push_queue_bytes(&mut queue, &budget, vec![b'x'; 64 * 1024]).unwrap();
+ drop(queue.pop_front().unwrap());
}
- decoder.charge_frame(256 * 1024).unwrap();
- assert!(
- decoder.charge_frame(4 * 1024 * 1024).is_err(),
- "the actual memory budget remains authoritative"
- );
+ queue.clear_and_release();
+ assert_eq!(budget.snapshot().unwrap().live, 0);
}
#[test]
diff --git a/crates/gateway/src/runtime/driver/response_diagnostics.rs b/crates/gateway/src/runtime/driver/response_diagnostics.rs
new file mode 100644
index 00000000..eaecf4d2
--- /dev/null
+++ b/crates/gateway/src/runtime/driver/response_diagnostics.rs
@@ -0,0 +1,60 @@
+use crate::server::core_runtime::{adapters::ProtocolAdapterError, model_ir::ModelIrError};
+use hiroute_diagnostics::event::{ResponseFailureReason as Reason, ResponseFailureStage as Stage};
+use std::sync::Arc;
+
+pub(super) fn note(stage: Stage, reason: Reason) {
+ if let Some(observation) = crate::server::core_runtime::observation::active_request() {
+ observation.response_failure(stage, reason);
+ }
+}
+
+pub(super) fn adapter(stage: Stage, error: ProtocolAdapterError) -> Arc<str> {
+ note(stage, category(&error));
+ Arc::from("provider response processing failed")
+}
+
+fn category(error: &ProtocolAdapterError) -> Reason {
+ match error {
+ ProtocolAdapterError::ModelIr(error) => match error {
+ ModelIrError::BufferLimit(_) => Reason::ResourceLimit,
+ ModelIrError::InvalidJson(_) => Reason::InvalidJson,
+ ModelIrError::InvalidSse(_) => Reason::InvalidSse,
+ ModelIrError::ExpectedObject | ModelIrError::InvalidField(_) => Reason::InvalidField,
+ ModelIrError::InvalidResponseLifecycle(_) => Reason::InvalidLifecycle,
+ ModelIrError::MissingTerminalEvent => Reason::MissingTerminal,
+ ModelIrError::DuplicateTerminalEvent => Reason::DuplicateTerminal,
+ _ => Reason::OtherProtocol,
+ },
+ ProtocolAdapterError::ClientUnrepresentable(_) => Reason::Unrepresentable,
+ _ => Reason::OtherProtocol,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn diagnostic_categories_distinguish_local_limits_and_protocol_failures_without_payloads() {
+ for (error, expected) in [
+ (ModelIrError::BufferLimit(100), Reason::ResourceLimit),
+ (
+ ModelIrError::InvalidJson("secret provider body".into()),
+ Reason::InvalidJson,
+ ),
+ (
+ ModelIrError::InvalidSse("secret event".into()),
+ Reason::InvalidSse,
+ ),
+ (ModelIrError::MissingTerminalEvent, Reason::MissingTerminal),
+ (
+ ModelIrError::DuplicateTerminalEvent,
+ Reason::DuplicateTerminal,
+ ),
+ ] {
+ let category = category(&error.into());
+ assert_eq!(category, expected);
+ assert!(!serde_json::to_string(&category).unwrap().contains("secret"));
+ }
+ }
+}
diff --git a/crates/gateway/src/runtime/driver/response_terminal_tests.rs b/crates/gateway/src/runtime/driver/response_terminal_tests.rs
index 1ae344af..9ab98e58 100644
--- a/crates/gateway/src/runtime/driver/response_terminal_tests.rs
+++ b/crates/gateway/src/runtime/driver/response_terminal_tests.rs
@@ -36,7 +36,6 @@ fn accepted_codex_terminal_event_crosses_transport_frames_and_keeps_eos() {
decoder: None,
renderer: None,
projector: Some(Box::new(projector)),
- _decoder_budget: None,
_chat_tool_projection_budget: None,
budget: budget.clone(),
streaming: true,
@@ -150,7 +149,6 @@ fn accepted_chat_terminal_closes_responses_stream_without_waiting_for_transport_
),
renderer: Some(adapters::IncrementalClientSseRenderer::new(client, "alias").unwrap()),
projector: None,
- _decoder_budget: None,
_chat_tool_projection_budget: None,
budget: budget.clone(),
streaming: true,
diff --git a/crates/observation/src/receipt/completeness.rs b/crates/observation/src/receipt/completeness.rs
new file mode 100644
index 00000000..6e81fc76
--- /dev/null
+++ b/crates/observation/src/receipt/completeness.rs
@@ -0,0 +1,45 @@
+//! Incremental projection over the append-only fact log. Gap repair and retention
+//! can remove evidence, so those exceptional paths still rebuild from surviving facts.
+use super::*;
+
+pub(super) fn append(
+ transaction: &Transaction<'_>,
+ envelope: &ExecutionFactEnvelopeV1,
+) -> Result<(), FactProjectionError> {
+ let workspace = envelope.correlation.workspace_id.as_str();
+ let session = envelope.correlation.conversation_id.as_str();
+ let (previous, has_prior, gap): (String, bool, bool) = transaction.query_row(
+ "SELECT facts_completeness,
+ EXISTS(SELECT 1 FROM execution_fact_events WHERE workspace_id=?1 AND session_id=?2
+ AND NOT (producer_id=?3 AND producer_epoch=?4 AND stream_id=?5 AND sequence=?6)),
+ EXISTS(SELECT 1 FROM observation_gaps WHERE channel='fact' AND workspace_id=?1 AND session_id=?2)
+ FROM sessions WHERE workspace_id=?1 AND session_id=?2",
+ params![workspace, session, envelope.producer.stream.producer_id.as_str(),
+ envelope.producer.stream.producer_epoch.as_str(), envelope.producer.stream.stream_id.as_str(),
+ i64::try_from(envelope.sequence).map_err(|_| FactProjectionError::Invalid)?],
+ |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
+ ).map_err(|_| FactProjectionError::Storage)?;
+ let current = completeness_from_facts(std::slice::from_ref(envelope));
+ // A content/gap-created placeholder or a fully retired session has no prior
+ // fact contribution. Its initial "unknown" must not taint the first fact.
+ let result = if gap {
+ FactsCompleteness::Partial
+ } else if !has_prior {
+ current
+ } else {
+ let previous = match previous.as_str() {
+ "complete" => FactsCompleteness::Complete,
+ "partial" => FactsCompleteness::Partial,
+ "unknown" => FactsCompleteness::Unknown,
+ _ => return Err(FactProjectionError::Corrupt),
+ };
+ least_complete(previous, current)
+ };
+ transaction
+ .execute(
+ "UPDATE sessions SET facts_completeness=?3 WHERE workspace_id=?1 AND session_id=?2",
+ params![workspace, session, facts_completeness_str(result)],
+ )
+ .map_err(|_| FactProjectionError::Storage)?;
+ Ok(())
+}
diff --git a/crates/observation/src/receipt/mod.rs b/crates/observation/src/receipt/mod.rs
index c8a3dfb6..2ec75d74 100644
--- a/crates/observation/src/receipt/mod.rs
+++ b/crates/observation/src/receipt/mod.rs
@@ -1,5 +1,6 @@
//! Immutable Receipt and Value projections derived from the durable execution-fact log.
+mod completeness;
mod plan_quality;
mod validation;
@@ -140,7 +141,7 @@ fn ensure_scope(
if existing_correlation == "unproven" && correlation_name != "unproven" {
transaction
.execute(
- "UPDATE sessions SET correlation=?3, facts_completeness='complete'
+ "UPDATE sessions SET correlation=?3
WHERE workspace_id=?1 AND session_id=?2",
params![
correlation.workspace_id.as_str(),
@@ -269,7 +270,7 @@ fn ensure_scope(
)
.map_err(|_| FactProjectionError::Storage)?;
}
- update_completeness(transaction, facts)
+ completeness::append(transaction, envelope)
}
fn classified_traffic(
@@ -294,32 +295,6 @@ fn classified_traffic(
Ok("unknown")
}
-fn update_completeness(
- transaction: &Transaction<'_>,
- facts: &[ExecutionFactEnvelopeV1],
-) -> Result<(), FactProjectionError> {
- let first = facts
- .first()
- .ok_or(FactProjectionError::MissingPrerequisite)?;
- let completeness = session_facts_completeness(
- transaction,
- &first.correlation.workspace_id,
- &first.correlation.conversation_id,
- )?;
- transaction
- .execute(
- "UPDATE sessions SET facts_completeness=?3
- WHERE workspace_id=?1 AND session_id=?2",
- params![
- first.correlation.workspace_id.as_str(),
- first.correlation.conversation_id.as_str(),
- facts_completeness_str(completeness),
- ],
- )
- .map_err(|_| FactProjectionError::Storage)?;
- Ok(())
-}
-
fn build_receipt(
connection: &Connection,
facts: &[ExecutionFactEnvelopeV1],
diff --git a/crates/observation/src/store/fact_log.rs b/crates/observation/src/store/fact_log.rs
index 5e3c6543..210be784 100644
--- a/crates/observation/src/store/fact_log.rs
+++ b/crates/observation/src/store/fact_log.rs
@@ -114,6 +114,8 @@ pub(crate) fn load_session(
workspace_id: &WorkspaceId,
session_id: &SessionId,
) -> Result<Vec<ExecutionFactEnvelopeV1>, ObservationStoreError> {
+ #[cfg(test)]
+ SESSION_LOADS.with(|count| count.set(count.get() + 1));
let request_ids = {
let mut statement = connection
.prepare(
@@ -145,6 +147,11 @@ pub(crate) fn load_session(
Ok(facts)
}
+#[cfg(test)]
+thread_local! {
+ pub(crate) static SESSION_LOADS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
+}
+
fn sql_u64(value: u64) -> Result<i64, ObservationStoreError> {
value.try_into().map_err(|_| ObservationStoreError::Corrupt)
}
diff --git a/crates/observation/src/store/request_retention.rs b/crates/observation/src/store/request_retention.rs
index 55126d84..067b7dba 100644
--- a/crates/observation/src/store/request_retention.rs
+++ b/crates/observation/src/store/request_retention.rs
@@ -77,6 +77,27 @@ impl LocalObservationStore {
transaction.execute("DELETE FROM turns WHERE workspace_id=?1 AND session_id=?2 AND NOT EXISTS(SELECT 1 FROM logical_requests r WHERE r.workspace_id=?1 AND r.turn_id=turns.turn_id)",params![workspace,session]).map_err(|_|Error::Unavailable)?;
transaction.execute("UPDATE sessions SET content_completeness='expired',facts_completeness='unknown',tombstone_reason='retention_expired',agent_id='',correlation='unproven' WHERE workspace_id=?1 AND session_id=?2 AND NOT EXISTS(SELECT 1 FROM logical_requests WHERE workspace_id=?1 AND session_id=?2)",params![workspace,session]).map_err(|_|Error::Unavailable)?;
}
+ // Removing the only Partial/Unknown request may improve completeness.
+ // Rebuild once per affected session, never once per newly ingested fact.
+ let sessions: std::collections::BTreeSet<_> = requests
+ .iter()
+ .map(|(workspace, session, _)| (workspace, session))
+ .collect();
+ for (workspace, session) in sessions {
+ let completeness = crate::receipt::session_facts_completeness(
+ &transaction,
+ &WorkspaceId::parse(workspace).map_err(|_| Error::Corrupt)?,
+ &SessionId::parse(session).map_err(|_| Error::Corrupt)?,
+ )
+ .map_err(|_| Error::Unavailable)?;
+ let value = match completeness {
+ hiroute_domain::FactsCompleteness::Complete => "complete",
+ hiroute_domain::FactsCompleteness::Partial => "partial",
+ hiroute_domain::FactsCompleteness::Unknown => "unknown",
+ };
+ transaction.execute("UPDATE sessions SET facts_completeness=?3 WHERE workspace_id=?1 AND session_id=?2",
+ params![workspace, session, value]).map_err(|_| Error::Unavailable)?;
+ }
if !requests.is_empty() || quality_changed {
// Index cleanup is part of the same visibility barrier. Physical
// blob GC is resumable and can follow this committed logical erase.
diff --git a/crates/observation/src/store/tests/incremental_completeness.rs b/crates/observation/src/store/tests/incremental_completeness.rs
new file mode 100644
index 00000000..9055bac7
--- /dev/null
+++ b/crates/observation/src/store/tests/incremental_completeness.rs
@@ -0,0 +1,102 @@
+use super::support::*;
+use crate::WriterCycleOutcome;
+use hiroute_domain::{CompletenessDeltaV1, ExecutionFactV1, FactsCompleteness};
+
+#[test]
+fn long_session_appends_match_full_projection_without_reloading_history() {
+ for delta in [None, Some(CompletenessDeltaV1::Unknown)] {
+ let temp = tempfile::tempdir().unwrap();
+ let store = open_store(temp.path());
+ let writer = writer(&store);
+ let session = Fixture::new("long-session").session;
+ for index in 0..40 {
+ let mut fixture = Fixture::new(&format!("long-{index}"));
+ fixture.session = session.clone();
+ let channel = fact_channel(&fixture, 512 * 1024);
+ let attempt = format!("attempt-long-{index}");
+ let mut facts = request_facts(&fixture, &attempt, finished(&attempt), 100 + index);
+ // Exercise both envelope Unknown and terminal Partial contributions.
+ if index == 3 {
+ facts[0].completeness_delta = delta;
+ }
+ if index == 25
+ && let ExecutionFactV1::RequestFinished {
+ facts_completeness, ..
+ } = &mut facts.last_mut().unwrap().fact
+ {
+ *facts_completeness = FactsCompleteness::Partial;
+ }
+ let before = crate::store::fact_log::SESSION_LOADS.with(|n| n.get());
+ for fact in facts {
+ let outcome = offer_fact(&writer, &channel, fact);
+ assert!(matches!(outcome, WriterCycleOutcome::Ack(_)), "{outcome:?}");
+ }
+ assert_eq!(
+ crate::store::fact_log::SESSION_LOADS.with(|n| n.get()),
+ before,
+ "ordinary ingestion must not hydrate older requests"
+ );
+ let connection = store.connection.lock();
+ let rebuilt = crate::receipt::session_facts_completeness(
+ &connection,
+ &fixture.workspace,
+ &session,
+ )
+ .unwrap();
+ let projected: String = connection.query_row(
+ "SELECT facts_completeness FROM sessions WHERE workspace_id=?1 AND session_id=?2",
+ rusqlite::params![fixture.workspace.as_str(), session.as_str()], |row| row.get(0),
+ ).unwrap();
+ assert_eq!(
+ serde_json::to_value(rebuilt).unwrap().as_str().unwrap(),
+ projected
+ );
+ }
+ }
+}
+
+#[test]
+fn retention_removes_old_partial_contribution_before_next_append() {
+ let temp = tempfile::tempdir().unwrap();
+ let store = open_store(temp.path());
+ let writer = writer(&store);
+ let session = Fixture::new("retained-session").session;
+ for (name, timestamp) in [
+ ("old-partial", 100),
+ ("fresh-complete", hiroute_domain::SEVEN_DAYS_MILLIS),
+ ] {
+ let mut fixture = Fixture::new(name);
+ fixture.session = session.clone();
+ let channel = fact_channel(&fixture, 512 * 1024);
+ let mut facts = request_facts(&fixture, name, finished(name), timestamp);
+ if name == "old-partial"
+ && let ExecutionFactV1::RequestFinished {
+ facts_completeness, ..
+ } = &mut facts.last_mut().unwrap().fact
+ {
+ *facts_completeness = FactsCompleteness::Partial;
+ }
+ for fact in facts {
+ assert!(matches!(
+ offer_fact(&writer, &channel, fact),
+ WriterCycleOutcome::Ack(_)
+ ));
+ }
+ }
+ assert_eq!(
+ store
+ .expire_request_details(hiroute_domain::SEVEN_DAYS_MILLIS + 1000, 100)
+ .unwrap(),
+ 1
+ );
+ let actual: String = store
+ .connection
+ .lock()
+ .query_row(
+ "SELECT facts_completeness FROM sessions WHERE session_id=?1",
+ [session.as_str()],
+ |row| row.get(0),
+ )
+ .unwrap();
+ assert_eq!(actual, "complete");
+}
diff --git a/crates/observation/src/store/tests/mod.rs b/crates/observation/src/store/tests/mod.rs
index 98b07f3a..7c3174a6 100644
--- a/crates/observation/src/store/tests/mod.rs
+++ b/crates/observation/src/store/tests/mod.rs
@@ -2,6 +2,7 @@ mod blob_gc;
mod content_v2;
mod fact_regression;
mod failure_isolation;
+mod incremental_completeness;
mod plan_quality;
mod retention;
mod retention_matrix;
diff --git a/tools/e2e-harness/tests/p0_gateway_runtime.rs b/tools/e2e-harness/tests/p0_gateway_runtime.rs
index 289851b3..7ffd968a 100644
--- a/tools/e2e-harness/tests/p0_gateway_runtime.rs
+++ b/tools/e2e-harness/tests/p0_gateway_runtime.rs
@@ -294,9 +294,9 @@ fn forged_generic_error_header_cannot_change_connector_classification() {
}
#[test]
-fn preoutput_native_sse_error_falls_back_but_semantic_output_is_rendered() {
+fn successful_sse_headers_commit_even_without_semantic_output_and_never_fallback() {
for first_body in [
- b"event: response.failed\ndata: {\"type\":\"response.failed\",\"error\":{\"type\":\"server_error\",\"message\":\"retry\"}}\n\n".as_slice(),
+ b"event: response.failed\ndata: {\"type\":\"response.failed\",\"response\":{\"id\":\"failed-stream\",\"model\":\"runtime-native\",\"status\":\"failed\",\"output\":[],\"error\":{\"type\":\"server_error\",\"message\":\"retry\"},\"usage\":{\"input_tokens\":7,\"output_tokens\":2}}}\n\n".as_slice(),
b"event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"empty-stream\",\"model\":\"runtime-native\"}}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"empty-stream\",\"model\":\"runtime-native\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":1,\"output_tokens\":0,\"total_tokens\":1}}}\n\n".as_slice(),
b": keepalive\n\nevent: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"control-only\",\"model\":\"runtime-native\"}}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"control-only\",\"model\":\"runtime-native\",\"status\":\"completed\",\"output\":[]}}\n\n".as_slice(),
] {
@@ -308,15 +308,64 @@ fn preoutput_native_sse_error_falls_back_but_semantic_output_is_rendered() {
status: 200,
body: RESPONSES_STREAM_OK,
}]);
- let fixture = RuntimeFixture::launch(&[&first, &second], 2);
+ let fixture = RuntimeFixture::launch_with_observation(
+ &[&first, &second], 2, ObservationFaults::healthy());
let response =
fixture.request_body(br#"{"model":"runtime-model","input":"hello","stream":true}"#);
assert_eq!(response.status, 200);
assert_eq!(first.calls(), 1);
- assert_eq!(second.calls(), 1);
- assert!(String::from_utf8_lossy(&response.body).contains("response.output_text.delta"));
+ assert_eq!(second.calls(), 0);
+ assert!(!String::from_utf8_lossy(&response.body).contains("response.output_text.delta"));
+ assert!(String::from_utf8_lossy(&response.body).contains(
+ if first_body.starts_with(b"event: response.failed") {
+ "response.failed"
+ } else {
+ "response.completed"
+ }
+ ));
+ if first_body.starts_with(b"event: response.failed") {
+ let facts = wait_execution_facts(&fixture, 1);
+ assert!(facts.iter().any(|row|
+ row.pointer("/fact/kind").and_then(|v| v.as_str()) == Some("usage_and_cache")
+ && row.pointer("/fact/input_tokens").and_then(|v| v.as_u64()) == Some(7)
+ && row.pointer("/fact/output_tokens").and_then(|v| v.as_u64()) == Some(2)
+ ), "failed stream must retain reported usage: {facts:?}");
+ }
+ }
+}
+
+#[test]
+fn successful_stream_forwards_unknown_events_before_any_semantic_output() {
+ use std::io::Read;
+ let unknown = b"event: response.vendor_extension\ndata: {\"opaque\":\"early-event\"}\n\n";
+ let provider = NativeProvider::start(vec![ProviderReply::StreamDrip {
+ status: 200,
+ chunks: vec![unknown.to_vec(), RESPONSES_STREAM_OK.to_vec()],
+ interval: Duration::from_secs(5),
+ }]);
+ let fixture = RuntimeFixture::launch(&[&provider], 1);
+ let mut stream = open_request(
+ fixture.address,
+ "POST",
+ "/v1/responses",
+ &[("X-HiRoute-Token", "runtime-token")],
+ br#"{"model":"runtime-model","input":"hello","stream":true}"#,
+ );
+ stream
+ .set_read_timeout(Some(Duration::from_secs(2)))
+ .unwrap();
+ let mut received = Vec::new();
+ let mut chunk = [0u8; 4096];
+ while !received.windows(unknown.len()).any(|part| part == unknown) {
+ let count = stream
+ .read(&mut chunk)
+ .expect("unknown event must arrive before the provider resumes");
+ assert!(count > 0, "stream ended before its unknown event");
+ received.extend_from_slice(&chunk[..count]);
}
+ assert!(received.starts_with(b"HTTP/1.1 200"));
+ assert_eq!(provider.calls(), 1);
}
#[test]