Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

fix(gateway): add precise response diagnostics and bounded replay (#62)

澄潭122c9ca4e6ca
diff --git a/contracts/compatibility-support.v1.json b/contracts/compatibility-support.v1.json
index a104b89d..4bd1ef2f 100644
--- a/contracts/compatibility-support.v1.json
+++ b/contracts/compatibility-support.v1.json
@@ -1259,6 +1259,17 @@
"crates/diagnostics/src/runtime.rs"
],
"field": "optional model, reasoning, HTTP protocol, provider error and model terminal result, target and commit facts in retained V1 logs"
+ },
+ {
+ "contract": "hiroute.diagnostic-event/v1",
+ "field": "ResponseFailure payload before detailed response reason, field and decoder position extension",
+ "mode": "frozen_fixture",
+ "owner": "diagnostics",
+ "reason": "The current diagnostic reader accepts supported v0.2.0 ResponseFailure bytes from released producer d625e653048b2306c88da07826d52027e9891675, defaulting absent optional attempt correlation, field and position. New writers emit the current closed event contract; old readers are not promised forward compatibility.",
+ "removal_condition": "Retire only when v0.2.0 diagnostic records are outside the supported source and retention policy.",
+ "allowed_paths": [
+ "crates/diagnostics/tests/response_failure_compatibility.rs"
+ ]
}
]
}
diff --git a/crates/diagnostics/src/event/model.rs b/crates/diagnostics/src/event/model.rs
index d7662284..e58e657f 100644
--- a/crates/diagnostics/src/event/model.rs
+++ b/crates/diagnostics/src/event/model.rs
@@ -113,6 +113,42 @@ pub struct ResponseFailure {
pub attempt_index: u64,
pub stage: ResponseFailureStage,
pub reason: ResponseFailureReason,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub attempt_token: Option<CorrelationToken>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub field: Option<ResponseFailureField>,
+ /// One-based SSE event visited, including the event rejected by the decoder.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub frame_index: Option<u64>,
+ /// Bytes supplied to the decoder, an upper bound rather than a token offset.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub received_bytes: Option<u64>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ResponseFailureField {
+ Object,
+ Choices,
+ Delta,
+ Content,
+ ToolCalls,
+ ToolId,
+ ToolName,
+ ToolArguments,
+ FinishReason,
+ Usage,
+ SystemFingerprint,
+ ServiceTier,
+ Logprobs,
+ Model,
+ Id,
+ Type,
+ Index,
+ Role,
+ Output,
+ Status,
+ Unknown,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -140,6 +176,16 @@ pub enum ResponseFailureReason {
NoSemanticOutput,
ProviderRejected,
OtherProtocol,
+ UnsupportedField,
+ UnsupportedValue,
+ MissingToolIdentity,
+ InvalidToolArguments,
+ ProviderStateNotPortable,
+ ToolIdBindingRequired,
+ ToolContinuationConflict,
+ Capability,
+ ContextProjection,
+ Serialization,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
diff --git a/crates/diagnostics/tests/response_failure_compatibility.rs b/crates/diagnostics/tests/response_failure_compatibility.rs
new file mode 100644
index 00000000..4951c488
--- /dev/null
+++ b/crates/diagnostics/tests/response_failure_compatibility.rs
@@ -0,0 +1,19 @@
+use hiroute_diagnostics::event::{ResponseFailure, ResponseFailureReason, ResponseFailureStage};
+
+#[test]
+fn supported_release_response_failure_retains_meaning_and_defaults_extensions() {
+ // Frozen from the serializer at supported release d625e653048b2306c88da07826d52027e9891675:
+ // crates/diagnostics/src/event/model.rs::ResponseFailure. Registered in compatibility support.
+ let released =
+ r#"{"request_token":null,"attempt_index":3,"stage":"decode","reason":"other_protocol"}"#;
+ let current: ResponseFailure = serde_json::from_str(released).unwrap();
+ assert_eq!(current.attempt_index, 3);
+ assert_eq!(current.stage, ResponseFailureStage::Decode);
+ assert_eq!(current.reason, ResponseFailureReason::OtherProtocol);
+ assert!(current.request_token.is_none());
+ assert!(current.attempt_token.is_none());
+ assert!(current.field.is_none());
+ assert!(current.frame_index.is_none());
+ assert!(current.received_bytes.is_none());
+ assert_eq!(serde_json::to_string(&current).unwrap(), released);
+}
diff --git a/crates/gateway/README.md b/crates/gateway/README.md
index 3c282ccc..7b62f883 100644
--- a/crates/gateway/README.md
+++ b/crates/gateway/README.md
@@ -85,6 +85,7 @@ provider references, observation and the future tool-selection boundary.
| Decision diagnostics use the production transport/parser without running a business model | `classifier_diagnostic_uses_the_production_transport_and_exact_protocol` in [classification.rs](src/core_runtime/classification.rs) |
| Accepted execution history survives client rewrite/compaction with observation disabled | [accepted_history.rs](../../tools/e2e-harness/tests/p0_gateway_runtime/accepted_history.rs) |
| Tool continuation survives restart with authentication | [continuation.rs](../../tools/e2e-harness/tests/p0_gateway_protocol/continuation.rs) |
+| Claude direct tool callers preserve native history and portable call/result pairing | `messages_direct_tool_caller_preserves_native_history_and_portable_tool_pairing` in [adapter tests](src/adapters/tests.rs); ingress rejects programmatic execution authority |
| Live publication cutover pins each request and preserves the last good version | [p0_gateway_request_authority.rs](../../tools/e2e-harness/tests/p0_gateway_request_authority.rs) |
Use the repository [test planner](../../scripts/test-plan.py) and
@@ -107,6 +108,60 @@ request and Agent turn observations even when its native bytes were delivered.
The decision diagnostic uses closed error codes; provider text is not a public
error message or a substitute for evidence of credential failure.
+## Response failure evidence
+
+`runtime/driver/response_diagnostics.rs` owns the closed, payload-free failure
+classification; `adapters/response/decoder.rs` reports SSE event ordinal and
+received-byte upper bound without per-token logs. Unix Debug-only
+`runtime/stream_capture.rs` can capture an explicitly enabled private session;
+its `stream_replay` example restores the actual attempt profile and tool mapping.
+This is offline decoder evidence, not proof of network timing or task completion.
+After successful provider materialization, private capture registers its existing
+weak handle once a pending attempt is staged, including explicitly unauthenticated
+profiles whose staging occurs after materialization rather than a credential read.
+Private capture binds that request-owned weak handle at formal attempt promotion,
+after validating the pending binding and credential. A prebody failure closes
+body capture and seals the file before `stopped`; later promotion may replace
+the sample atomically with an identical content prefix, one typed correlation
+record and a new durable seal. A private temporary copy is synced before rename,
+so termination always leaves either complete version; temporary bytes count toward
+the session quota. If that quota cannot hold both copies, the original survives. It
+rechecks the private directory, file identity/permissions/owner, expiry and
+original file/session/record budgets. Content capture never reopens, and the
+weak handle cannot retain a writer or bind another request or attempt.
+
+The helper requires Unix Python with non-reaping `os.waitid`/`os.WNOWAIT`
+(Linux, or Python 3.13+ on macOS). It checks support before creating a session.
+From a clean checkout of the exact candidate, wrap the managed Debug daemon and
+its isolated settings arguments with:
+
+```sh
+python3 scripts/private-stream-capture.py run \
+ --root /absolute/new-private-session --source-sha FULL_CANDIDATE_SHA \
+ --client isolated-client-version --seconds 300 --attempts 3 \
+ -- /absolute/managed-debug-hirouted ISOLATED_DAEMON_ARGUMENTS
+```
+
+The helper issues no requests. Disable client retries, use a separate API-key
+context, and stop on the first failure. Each private file is limited to 8 MiB,
+the session to 32 MiB and four underlying attempts, and capture expires within
+one hour. Credentials in transport headers are excluded. Never commit or upload
+the files. Run the same candidate's Debug `stream_replay` executable with the
+private `attempt-N.capture` path to check original, one-byte and 4096-byte chunks;
+it prints only safe results. Incomplete or unsealed samples are rejected.
+Replay supports successful cross-protocol decoder paths, skips informational
+HTTP heads, and rejects native projector or non-success paths as
+`unsupported_capture_path` without a decoder verdict.
+A separate, bounded retention owner stops the isolated candidate's entire process
+group, including descendants after the leader exits, and reaps the leader. It
+deletes the raw session within 24 hours, even after the CLI returns. Its PID is printed;
+SIGTERM also stops the candidate and deletes the session immediately. Keep the
+owner running until deletion. Host shutdown or SIGKILL of that owner cannot run
+cleanup; after recovery, delete the exact expired session with
+`python3 scripts/private-stream-capture.py cleanup /absolute/new-private-session`.
+Kernel locks release on candidate crashes, so stale lock files do not prevent
+recovery cleanup. No shared scheduler or daily daemon is modified.
+
Known incomplete nonstream JSON responses follow the same prebody relay boundary
as streaming failures, for both native and converted protocols. Reported usage
survives fallback; partial response content is not delivered. Attempt/deadline
diff --git a/crates/gateway/examples/stream_replay.rs b/crates/gateway/examples/stream_replay.rs
new file mode 100644
index 00000000..74d460f7
--- /dev/null
+++ b/crates/gateway/examples/stream_replay.rs
@@ -0,0 +1,29 @@
+//! Private offline replay of a Debug capture; never prints bodies or error text.
+#[cfg(all(unix, debug_assertions))]
+#[tokio::main]
+async fn main() {
+ let args: Vec<_> = std::env::args_os().collect();
+ if args.len() != 2 {
+ eprintln!("usage: stream_replay PRIVATE_CAPTURE");
+ std::process::exit(2);
+ }
+ for chunk in [0, 1, 4096] {
+ match hiroute_gateway::runtime::stream_capture::replay_capture(
+ std::path::Path::new(&args[1]),
+ chunk,
+ )
+ .await
+ {
+ Ok(summary) => println!("{summary}"),
+ Err(reason) => {
+ eprintln!("{reason}");
+ std::process::exit(1);
+ }
+ }
+ }
+}
+#[cfg(not(all(unix, debug_assertions)))]
+fn main() {
+ eprintln!("stream replay requires Unix Debug build");
+ std::process::exit(2);
+}
diff --git a/crates/gateway/src/adapters/response/decoder.rs b/crates/gateway/src/adapters/response/decoder.rs
index 1ff3b7b5..c068e61e 100644
--- a/crates/gateway/src/adapters/response/decoder.rs
+++ b/crates/gateway/src/adapters/response/decoder.rs
@@ -43,6 +43,8 @@ pub struct NativeResponseDecoder {
framer: Option<SseFramer>,
ended: bool,
end_stream_pending: bool,
+ received_bytes: u64,
+ frame_index: u64,
}
impl NativeResponseDecoder {
@@ -186,6 +188,8 @@ impl NativeResponseDecoder {
framer,
ended: false,
end_stream_pending: false,
+ received_bytes: 0,
+ frame_index: 0,
})
}
@@ -205,6 +209,7 @@ impl NativeResponseDecoder {
if self.events.len() >= MAX_QUEUED_EVENTS {
return Ok(ResponseDecodeStatus::NeedDrain);
}
+ self.received_bytes = self.received_bytes.saturating_add(bytes.len() as u64);
let need_drain = if self.streaming {
if end_stream {
self.end_stream_pending = true;
@@ -260,6 +265,7 @@ impl NativeResponseDecoder {
&self.budget,
&mut self.state,
&mut self.events,
+ &mut self.frame_index,
)?;
if matches!(outcome, SseFeedOutcome::Complete) && self.end_stream_pending {
self.finish_stream_eof()?;
@@ -277,6 +283,11 @@ impl NativeResponseDecoder {
self.events.drain(..).collect()
}
+ /// Safe failure position; received bytes are an upper bound for a multi-event read.
+ pub fn diagnostic_position(&self) -> (u64, u64) {
+ (self.frame_index, self.received_bytes)
+ }
+
pub fn sse_complexity(&self) -> Option<SseComplexity> {
self.framer.as_ref().map(SseFramer::complexity)
}
@@ -310,13 +321,14 @@ impl NativeResponseDecoder {
) -> Result<SseFeedOutcome, ProtocolAdapterError> {
let charged =
ChargedBytes::copy_from_opaque(&self.budget, MemoryRole::TransportInflight, bytes)
- .map_err(|error| ModelIrError::InvalidSse(error.to_string()))?;
+ .map_err(|_| ModelIrError::BufferLimit(bytes.len()))?;
let outcome = run_sse(
self.framer.as_mut().expect("streaming decoder has framer"),
Some((charged, end_stream)),
&self.budget,
&mut self.state,
&mut self.events,
+ &mut self.frame_index,
)?;
Ok(outcome)
}
@@ -337,12 +349,14 @@ fn run_sse(
budget: &StreamBudget,
state: &mut ProtocolState,
output: &mut VecDeque<ModelStreamEventV1>,
+ frame_index: &mut u64,
) -> Result<SseFeedOutcome, ProtocolAdapterError> {
let mut visitor = NativeVisitor {
budget,
state,
output,
protocol_error: None,
+ frame_index,
};
let mut sink = DiscardOutput;
let framed = match input {
@@ -352,7 +366,16 @@ fn run_sse(
if let Some(error) = visitor.protocol_error {
return Err(error);
}
- framed.map_err(|error| ModelIrError::InvalidSse(error.to_string()).into())
+ framed.map_err(|error| {
+ match error {
+ SseError::BudgetExceeded
+ | SseError::EventLimit
+ | SseError::PendingLimit
+ | SseError::OutputLimit => ModelIrError::BufferLimit(usize::MAX),
+ other => ModelIrError::InvalidSse(other.to_string()),
+ }
+ .into()
+ })
}
struct NativeVisitor<'a> {
@@ -360,6 +383,7 @@ struct NativeVisitor<'a> {
state: &'a mut ProtocolState,
output: &'a mut VecDeque<ModelStreamEventV1>,
protocol_error: Option<ProtocolAdapterError>,
+ frame_index: &'a mut u64,
}
impl SseVisitor for NativeVisitor<'_> {
@@ -371,6 +395,7 @@ impl SseVisitor for NativeVisitor<'_> {
if self.output.len() >= MAX_QUEUED_EVENTS {
return Err(SseError::NeedDrain);
}
+ *self.frame_index = self.frame_index.saturating_add(1);
if event
.fields()
.any(|field| !field.comment && field.name.starts_with(b"data") && field.name != b"data")
@@ -415,3 +440,46 @@ impl BoundedOutputSink for DiscardOutput {
Ok(())
}
}
+
+#[cfg(test)]
+mod diagnostic_tests {
+ use super::*;
+ use crate::server::core_runtime::profiles::fixed_reasoning;
+
+ #[test]
+ fn actual_sse_budget_rejection_is_distinct_from_malformed_sse() {
+ let profile = CandidateProtocolProfile::exact_portable_path(
+ IngressProtocol::Messages,
+ IngressProtocol::ChatCompletions,
+ "physical",
+ fixed_reasoning("fixed"),
+ );
+ for fragmented in [false, true] {
+ let budget = BudgetTree::new(1024, 1024).unwrap().stream(1024).unwrap();
+ let mut decoder =
+ NativeResponseDecoder::new_for_attempt(&profile, 200, true, None, budget.clone())
+ .unwrap();
+ let error = if fragmented {
+ (0..16)
+ .find_map(|_| decoder.feed(&[b'x'; 256], false).err())
+ .expect("retained frame must exhaust the owner")
+ } else {
+ decoder.feed(&[b'x'; 2048], false).unwrap_err()
+ };
+ assert!(
+ matches!(
+ error,
+ ProtocolAdapterError::ModelIr(ModelIrError::BufferLimit(_))
+ ),
+ "{error:?}"
+ );
+ drop(decoder);
+ assert_eq!(budget.snapshot().unwrap().live, 0);
+ }
+ let mut decoder = NativeResponseDecoder::new(&profile, 200, true).unwrap();
+ assert!(matches!(
+ decoder.feed(b"data-invalid: {}\n\n", false),
+ Err(ProtocolAdapterError::ModelIr(ModelIrError::InvalidSse(_)))
+ ));
+ }
+}
diff --git a/crates/gateway/src/adapters/tool_projection.rs b/crates/gateway/src/adapters/tool_projection.rs
index 7ba1c2be..71d3e37d 100644
--- a/crates/gateway/src/adapters/tool_projection.rs
+++ b/crates/gateway/src/adapters/tool_projection.rs
@@ -11,7 +11,7 @@ use super::ProtocolAdapterError;
const MAX_CHAT_TOOL_NAME_BYTES: usize = 64;
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
pub(crate) struct ChatToolIdentity {
pub(crate) emitted_name: String,
pub(crate) kind: ToolKindV1,
@@ -19,7 +19,7 @@ pub(crate) struct ChatToolIdentity {
pub(crate) local_name: String,
}
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
pub(crate) struct ChatToolProjection {
entries: Vec<ChatToolIdentity>,
}
diff --git a/crates/gateway/src/core_runtime.rs b/crates/gateway/src/core_runtime.rs
index db468918..e515e581 100644
--- a/crates/gateway/src/core_runtime.rs
+++ b/crates/gateway/src/core_runtime.rs
@@ -1733,3 +1733,7 @@ mod inbound_auth_tests;
#[cfg(test)]
#[path = "core_runtime/acceptance_tests.rs"]
mod acceptance_tests;
+
+#[cfg(all(test, unix, debug_assertions, feature = "e2e-test-control"))]
+#[path = "core_runtime/response_diagnostics_tests.rs"]
+mod response_diagnostics_tests;
diff --git a/crates/gateway/src/core_runtime/response_diagnostics_tests.rs b/crates/gateway/src/core_runtime/response_diagnostics_tests.rs
new file mode 100644
index 00000000..68161f05
--- /dev/null
+++ b/crates/gateway/src/core_runtime/response_diagnostics_tests.rs
@@ -0,0 +1,1034 @@
+//! Deterministic upstream injections through the production HTTPS provider and listener.
+//! Each child owns its environment, diagnostics, Replay and optional private capture.
+use std::fs;
+use std::io::{Read, Write};
+use std::net::{TcpListener, TcpStream};
+use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt};
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
+
+use hiroute_diagnostics::event::ProcessRole;
+use hiroute_diagnostics::level::DiagnosticLevel;
+use hiroute_diagnostics::record::Component;
+use hiroute_diagnostics::runtime::{DiagnosticRuntime, RuntimeConfig};
+use serde_json::{Value, json};
+
+use super::*;
+use crate::server::core_runtime::observation::{
+ GatewayObservationSinks, ObservationAck, ObservationNack, ObservationRecord,
+ ObservationRecordSink, accounted_acknowledgement,
+};
+use crate::server::publication::{
+ AliasPlanV1, GatewayPublicationInstaller, GatewayPublicationSnapshotV3, GrantV1, ModelRouteV2,
+ token_sha256,
+};
+use crate::server::test_control::{
+ E2E_DIAL_CONFIG_ENV, E2E_DIAL_CONFIG_FILE, TestTlsListener, sealed_native_candidate,
+ write_dial_config,
+};
+
+const CHILD_CASE: &str = "HIROUTE_STREAM_DIAGNOSTIC_CASE";
+const CHILD_ROOT: &str = "HIROUTE_STREAM_DIAGNOSTIC_ROOT";
+const TOKEN: &str = "SENTINEL_CLIENT_CREDENTIAL";
+const TOOL: &str = "SENTINEL_DYNAMIC_TOOL";
+const TEST: &str = "server::core_runtime::response_diagnostics_tests::production_listener_stream_diagnostics_and_private_capture";
+
+struct PrivateRoot(PathBuf);
+impl Drop for PrivateRoot {
+ fn drop(&mut self) {
+ let _ = fs::remove_dir_all(&self.0);
+ }
+}
+
+#[test]
+fn production_listener_stream_diagnostics_and_private_capture() {
+ if let Ok(case) = std::env::var(CHILD_CASE) {
+ run_case(&case, &PathBuf::from(std::env::var_os(CHILD_ROOT).unwrap()));
+ return;
+ }
+ let mut failures = Vec::new();
+ for case in [
+ "unsupported_known",
+ "unknown_metadata",
+ "unsupported_value",
+ "invalid_type",
+ "invalid_json",
+ "prebody_invalid_json",
+ "prebody_capture_invalid_json",
+ "prebody_capture_relay",
+ "prebody_capture_no_credential",
+ "prebody_capture_no_credential_relay",
+ "invalid_sse",
+ "missing_tool",
+ "invalid_arguments",
+ "invalid_lifecycle",
+ "missing_terminal",
+ "duplicate_terminal",
+ "resource_limit",
+ "drain_failure",
+ "capture_known",
+ "capture_unknown",
+ "capture_disabled",
+ ] {
+ if let Err(failure) = run_child_case(case) {
+ failures.push(failure);
+ }
+ }
+ assert!(failures.is_empty(), "{}", failures.join("\n"));
+}
+
+#[test]
+fn production_completed_prefix_capture_keeps_formal_attempt_correlation() {
+ run_child_case("capture_known").unwrap();
+}
+
+#[test]
+fn production_prebody_failed_capture_keeps_promoted_attempt_correlation() {
+ let failures: Vec<_> = ["prebody_capture_invalid_json", "prebody_capture_relay"]
+ .into_iter()
+ .filter_map(|case| run_child_case(case).err())
+ .collect();
+ assert!(failures.is_empty(), "{}", failures.join("\n"));
+}
+
+#[test]
+fn production_no_credential_prebody_capture_keeps_promoted_attempt_correlation() {
+ let failures: Vec<_> = [
+ "prebody_capture_no_credential",
+ "prebody_capture_no_credential_relay",
+ ]
+ .into_iter()
+ .filter_map(|case| run_child_case(case).err())
+ .collect();
+ assert!(failures.is_empty(), "{}", failures.join("\n"));
+}
+
+fn run_child_case(case: &str) -> Result<(), String> {
+ let mut random = [0u8; 16];
+ getrandom::fill(&mut random).unwrap();
+ let root = PrivateRoot(
+ fs::canonicalize(std::env::temp_dir())
+ .unwrap()
+ .join(format!(
+ "hiroute-stream-acceptance-{:x}",
+ u128::from_le_bytes(random)
+ )),
+ );
+ fs::DirBuilder::new().mode(0o700).create(&root.0).unwrap();
+ let mut child = Command::new(std::env::current_exe().unwrap());
+ child
+ .args(["--exact", TEST, "--nocapture"])
+ .env(CHILD_CASE, case)
+ .env(CHILD_ROOT, &root.0)
+ .env(E2E_DIAL_CONFIG_ENV, root.0.join(E2E_DIAL_CONFIG_FILE))
+ .env("HIROUTE_REPLAY_ROOT", root.0.join("replay"))
+ .env_remove("HIROUTE_PRIVATE_STREAM_CAPTURE");
+ if matches!(
+ case,
+ "capture_known"
+ | "capture_unknown"
+ | "prebody_capture_invalid_json"
+ | "prebody_capture_relay"
+ | "prebody_capture_no_credential"
+ | "prebody_capture_no_credential_relay"
+ ) {
+ child.env("HIROUTE_PRIVATE_STREAM_CAPTURE", root.0.join("capture"));
+ }
+ let output = child.output().unwrap();
+ if output.status.success() && String::from_utf8_lossy(&output.stdout).contains("1 passed") {
+ println!("stream injection {case}: green");
+ Ok(())
+ } else {
+ Err(format!(
+ "{case}: {}\n{}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ ))
+ }
+}
+
+#[derive(Clone, Default)]
+struct Facts(Arc<Mutex<Vec<Value>>>);
+impl ObservationRecordSink for Facts {
+ fn deliver(&self, record: &ObservationRecord) -> Result<ObservationAck, ObservationNack> {
+ self.0
+ .lock()
+ .unwrap()
+ .push(serde_json::from_slice(record.payload()).unwrap());
+ Ok(accounted_acknowledgement(record))
+ }
+}
+
+fn chat(delta: Value, finish: Value) -> Value {
+ json!({"id":"SENTINEL_RESPONSE_BODY","model":"physical",
+ "choices":[{"index":0,"delta":delta,"finish_reason":finish}]})
+}
+fn frame(value: Value) -> Vec<u8> {
+ format!("data: {value}\n\n").into_bytes()
+}
+fn tool(name: Option<&str>, arguments: &str) -> Value {
+ let mut call = json!({"index":0,"type":"function","function":{"arguments":arguments}});
+ if let Some(name) = name {
+ call["id"] = "SENTINEL_NATIVE_CALL".into();
+ call["function"]["name"] = name.into();
+ }
+ json!({"tool_calls":[call]})
+}
+
+fn sample(case: &str) -> (Vec<u8>, Option<&'static str>, Option<&'static str>, u64) {
+ let (mut wire, reason, field, index) = match case {
+ "unsupported_known" => {
+ let mut value = chat(json!({}), Value::Null);
+ value["system_fingerprint"] = "SENTINEL_PROVIDER_TEXT".into();
+ (
+ frame(value),
+ Some("unsupported_field"),
+ Some("system_fingerprint"),
+ 1,
+ )
+ }
+ "unknown_metadata" => {
+ // Unknown sibling metadata is already tolerated by the production
+ // decoder. Keep that semantic policy and verify it never enters logs.
+ let mut value = chat(json!({"content":"SENTINEL_PROVIDER_TEXT"}), json!("stop"));
+ value["SENTINEL_UNKNOWN_FIELD"] = "SENTINEL_PROVIDER_TEXT".into();
+ value["usage"] = json!({"prompt_tokens":11,"completion_tokens":7,"total_tokens":18});
+ (frame(value), None, None, 2)
+ }
+ "invalid_type" => (
+ frame(chat(
+ json!({"content":{"SENTINEL_PROVIDER_TEXT":true}}),
+ Value::Null,
+ )),
+ Some("invalid_field"),
+ Some("content"),
+ 1,
+ ),
+ "unsupported_value" => (
+ frame(chat(json!({"role":"SENTINEL_PROVIDER_ROLE"}), Value::Null)),
+ Some("unsupported_value"),
+ None,
+ 1,
+ ),
+ "invalid_json"
+ | "prebody_invalid_json"
+ | "prebody_capture_invalid_json"
+ | "prebody_capture_relay"
+ | "prebody_capture_no_credential"
+ | "prebody_capture_no_credential_relay" => (
+ b"data: {\"SENTINEL_PROVIDER_TEXT\":\n\n".to_vec(),
+ Some("invalid_json"),
+ None,
+ 1,
+ ),
+ "invalid_sse" => (
+ b"data-invalid: SENTINEL_PROVIDER_TEXT\n\n".to_vec(),
+ Some("invalid_sse"),
+ None,
+ 1,
+ ),
+ "missing_tool" => (
+ frame(chat(tool(None, "{}"), Value::Null)),
+ Some("missing_tool_identity"),
+ None,
+ 1,
+ ),
+ "invalid_arguments" => (
+ frame(chat(
+ tool(Some(TOOL), "{\"SENTINEL_ARGUMENT\":"),
+ json!("tool_calls"),
+ )),
+ Some("invalid_tool_arguments"),
+ None,
+ 1,
+ ),
+ "invalid_lifecycle" => {
+ let mut wire = frame(chat(tool(Some(TOOL), "{"), Value::Null));
+ let mut changed = tool(Some(TOOL), "}");
+ changed["tool_calls"][0]["id"] = "SENTINEL_CHANGED_CALL".into();
+ wire.extend(frame(chat(changed, Value::Null)));
+ (wire, Some("invalid_lifecycle"), None, 2)
+ }
+ "missing_terminal" => (
+ frame(chat(
+ json!({"content":"SENTINEL_PROVIDER_TEXT"}),
+ Value::Null,
+ )),
+ Some("missing_terminal"),
+ None,
+ 1,
+ ),
+ "duplicate_terminal" => {
+ let mut wire = frame(chat(
+ json!({"content":"SENTINEL_PROVIDER_TEXT"}),
+ json!("stop"),
+ ));
+ wire.extend_from_slice(b"data: [DONE]\n\ndata: [DONE]\n\n");
+ (wire, Some("duplicate_terminal"), None, 3)
+ }
+ "resource_limit" => (
+ frame(chat(
+ json!({"content":"x".repeat(6 * 1024 * 1024)}),
+ Value::Null,
+ )),
+ Some("resource_limit"),
+ None,
+ 0,
+ ),
+ "drain_failure" => {
+ let mut wire = Vec::new();
+ for _ in 0..40 {
+ wire.extend(frame(chat(
+ json!({"content":"SENTINEL_PROVIDER_TEXT"}),
+ Value::Null,
+ )));
+ }
+ let mut value = chat(json!({}), Value::Null);
+ value["system_fingerprint"] = "SENTINEL_PROVIDER_TEXT".into();
+ wire.extend(frame(value));
+ (
+ wire,
+ Some("unsupported_field"),
+ Some("system_fingerprint"),
+ 41,
+ )
+ }
+ "capture_unknown" => (
+ frame(chat(
+ tool(Some("SENTINEL_UNKNOWN_TOOL"), "{}"),
+ json!("tool_calls"),
+ )),
+ Some("missing_tool_identity"),
+ None,
+ 1,
+ ),
+ "capture_known" => {
+ let mut value = chat(tool(Some(TOOL), "{}"), json!("tool_calls"));
+ value["usage"] = json!({"prompt_tokens":11,"completion_tokens":7,"total_tokens":18});
+ (frame(value), None, None, 2)
+ }
+ "capture_disabled" => {
+ let mut value = chat(json!({"content":"SENTINEL_PROVIDER_TEXT"}), json!("stop"));
+ value["usage"] = json!({"prompt_tokens":11,"completion_tokens":7,"total_tokens":18});
+ (frame(value), None, None, 2)
+ }
+ _ => panic!("unknown test case"),
+ };
+ if matches!(
+ case,
+ "invalid_arguments"
+ | "unknown_metadata"
+ | "capture_known"
+ | "capture_unknown"
+ | "capture_disabled"
+ ) {
+ wire.extend_from_slice(b"data: [DONE]\n\n");
+ }
+ (wire, reason, field, index)
+}
+
+fn run_case(case: &str, root: &Path) {
+ let capture_enabled = std::env::var_os("HIROUTE_PRIVATE_STREAM_CAPTURE").is_some();
+ if capture_enabled {
+ create_capture_session(&root.join("capture"));
+ }
+ let (tail, reason, field, frame_index) = sample(case);
+ let prebody = case.starts_with("prebody_");
+ let relay = case.ends_with("_relay");
+ let no_credential = case.contains("no_credential");
+ let after_body = reason.is_some() && !prebody;
+ let mut wire = if after_body {
+ frame(chat(
+ json!({"content":"SENTINEL_INITIAL_BODY"}),
+ Value::Null,
+ ))
+ } else {
+ Vec::new()
+ };
+ let prefix_len = wire.len();
+ wire.extend(tail);
+ let frame_index = if after_body && frame_index != 0 {
+ frame_index + 1
+ } else {
+ frame_index
+ };
+ let (body_delivered, wait_for_body) = std::sync::mpsc::sync_channel(0);
+ let listener = TestTlsListener::bind("stream-diagnostic.invalid").unwrap();
+ write_dial_config(root, &[&listener]).unwrap();
+ let peer = listener.try_clone().unwrap();
+ let sent = wire.clone();
+ let upstream = std::thread::spawn(move || {
+ let (mut stream, _) = peer.accept().unwrap();
+ stream
+ .set_read_timeout(Some(Duration::from_secs(10)))
+ .unwrap();
+ let request = read_request(&mut stream);
+ write!(stream, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", sent.len()).unwrap();
+ if after_body {
+ stream.write_all(&sent[..prefix_len]).unwrap();
+ stream.flush().unwrap();
+ // No error bytes exist on the wire until the client has received
+ // the first real body unit. This proves the afterbody boundary.
+ wait_for_body.recv_timeout(Duration::from_secs(3)).unwrap();
+ let _ = stream.write_all(&sent[prefix_len..]);
+ } else {
+ let _ = stream.write_all(&sent);
+ }
+ let _ = stream.finish();
+ if relay {
+ let (mut stream, _) = peer.accept().unwrap();
+ let _second_request = read_request(&mut stream);
+ let mut value = chat(json!({"content":"SENTINEL_RECOVERY_BODY"}), json!("stop"));
+ value["usage"] = json!({"prompt_tokens":11,"completion_tokens":7,"total_tokens":18});
+ let mut recovery = frame(value);
+ recovery.extend_from_slice(b"data: [DONE]\n\n");
+ write!(stream, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", recovery.len()).unwrap();
+ stream.write_all(&recovery).unwrap();
+ let _ = stream.finish();
+ }
+ request
+ });
+ let publications = Arc::new(GatewayPublicationInstaller::open(root.join("lkg.json")).unwrap());
+ let publication = publication(&listener, prebody && !relay, no_credential);
+ if let crate::server::publication::GatewayPrepareOutcome::Prepared(prepared) =
+ publications.prepare(publication).unwrap()
+ {
+ publications.publish(prepared).unwrap();
+ }
+ let credential = json!({"schema_version":"hiroute.gateway.credential-leases/v1",
+ "credential_ref":"stream-credential","keys":[{"key_id":"stream-key","generation":1,
+ "authorization":"Bearer SENTINEL_UPSTREAM_CREDENTIAL"}]});
+ fs::write(
+ root.join("lease.json"),
+ serde_json::to_vec(&credential).unwrap(),
+ )
+ .unwrap();
+ fs::write(root.join("credentials.json"), serde_json::to_vec(&json!({
+ "schema_version":"hiroute.gateway.credentials/v1","credentials":{"stream-credential":"lease.json"}
+ })).unwrap()).unwrap();
+ let mut ports = ProductionPorts::fail_closed(publications);
+ ports.credentials = Arc::new(
+ crate::server::composition::FileCredentialResolver::open(&root.join("credentials.json"))
+ .unwrap(),
+ );
+ ports.runtime_state = Arc::new(crate::ports::InMemoryRuntimeStateStore::default());
+ let diagnostics = DiagnosticRuntime::start(RuntimeConfig {
+ root: root.join("diagnostics"),
+ role: ProcessRole::Daemon,
+ component: Component::Gateway,
+ parent_session_id: None,
+ level_override: Some(DiagnosticLevel::Debug),
+ });
+ let facts = Facts::default();
+ let mut sinks = GatewayObservationSinks::discard();
+ sinks.execution_fact = Arc::new(facts.clone());
+ let observation =
+ GatewayObservation::with_sinks(512 * 1024, sinks).with_diagnostics(diagnostics.port());
+ let runtime = ProductionGatewayRuntime::compose_with_planner_and_observation(
+ ports,
+ Arc::new(PublicationPlannerInputAuthority),
+ Arc::new(observation),
+ );
+ let address = TcpListener::bind("127.0.0.1:0")
+ .unwrap()
+ .local_addr()
+ .unwrap();
+ let mut gateway = crate::server::GatewayLauncher::from_runtime(address, Arc::new(runtime))
+ .unwrap()
+ .start_managed()
+ .unwrap();
+ let body = serde_json::to_vec(&json!({"model":"stream-alias","max_tokens":64,
+ "messages":[{"role":"user","content":"SENTINEL_REQUEST_BODY"}],"stream":true,
+ "tools":[{"name":TOOL,"description":"SENTINEL_TOOL_DESCRIPTION","input_schema":{"type":"object"}}]})).unwrap();
+ let mut client = TcpStream::connect(address).unwrap();
+ client
+ .set_read_timeout(Some(Duration::from_secs(15)))
+ .unwrap();
+ write!(client, "POST /v1/messages HTTP/1.1\r\nHost: {address}\r\nX-HiRoute-Token: {TOKEN}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).unwrap();
+ client.write_all(&body).unwrap();
+ let mut response = Vec::new();
+ if after_body {
+ client
+ .set_read_timeout(Some(Duration::from_secs(2)))
+ .unwrap();
+ while !response
+ .windows(b"SENTINEL_INITIAL_BODY".len())
+ .any(|w| w == b"SENTINEL_INITIAL_BODY")
+ {
+ let mut chunk = [0u8; 4096];
+ let n = client
+ .read(&mut chunk)
+ .expect("initial body was not delivered before the error");
+ assert!(n != 0, "response ended before initial body delivery");
+ response.extend_from_slice(&chunk[..n]);
+ }
+ body_delivered.send(()).unwrap();
+ client
+ .set_read_timeout(Some(Duration::from_secs(15)))
+ .unwrap();
+ }
+ // A failed committed stream may end with a reset. The formal execution fact is the oracle.
+ let _ = client.read_to_end(&mut response);
+ assert!(
+ response.starts_with(if prebody && !relay {
+ b"HTTP/1.1 502"
+ } else {
+ b"HTTP/1.1 200"
+ }),
+ "{}",
+ String::from_utf8_lossy(&response)
+ );
+ if case == "capture_known" {
+ let response = String::from_utf8_lossy(&response);
+ assert!(response.contains("\"type\":\"tool_use\""));
+ assert!(response.contains(&format!("\"name\":\"{TOOL}\"")));
+ assert!(response.contains("\"partial_json\":\"{}\""));
+ assert!(response.contains("event: message_stop"));
+ }
+ if relay {
+ assert!(
+ response
+ .windows(b"SENTINEL_RECOVERY_BODY".len())
+ .any(|w| w == b"SENTINEL_RECOVERY_BODY")
+ );
+ }
+ let upstream_request = upstream.join().unwrap();
+ assert!(!upstream_request.is_empty());
+ if no_credential {
+ assert!(
+ !String::from_utf8_lossy(&upstream_request)
+ .to_ascii_lowercase()
+ .contains("authorization:")
+ );
+ }
+ wait_finished(&facts, reason.is_some() && !relay);
+ gateway.shutdown();
+ gateway.join(Duration::from_secs(5)).unwrap();
+ let status = diagnostics.status();
+ assert!(status.armed);
+ assert!(status.unavailable.is_none());
+ assert!(status.settings_error.is_none());
+ diagnostics.shutdown();
+ // Keep the collector live through Gateway shutdown; request_finished is
+ // a terminal business fact, not a barrier for every observation producer.
+ let facts = facts.0.lock().unwrap().clone();
+ if no_credential {
+ assert!(
+ !facts.iter().any(
+ |r| r.pointer("/fact/kind").and_then(Value::as_str) == Some("credential_lease")
+ )
+ );
+ }
+ let log = fs::read_to_string(root.join("diagnostics/daemon/current.jsonl")).unwrap();
+ assert!(
+ !log.contains("SENTINEL"),
+ "untrusted body/tool/header leaked into product diagnostics"
+ );
+ let records: Vec<Value> = log
+ .lines()
+ .map(|line| serde_json::from_str(line).unwrap())
+ .collect();
+ assert!(
+ records
+ .iter()
+ .any(|r| payload(r, "level_applied").is_some_and(|v| v["level"] == "debug")),
+ "actual Debug level missing: {log}"
+ );
+ if relay {
+ assert_prebody_relay_business(&facts);
+ } else {
+ assert_business(&facts, reason.is_some(), after_body);
+ }
+ if let Some(reason) = reason {
+ let failures: Vec<_> = records
+ .iter()
+ .filter_map(|r| payload(r, "response_failure"))
+ .collect();
+ assert_eq!(failures.len(), 1, "{case}: {log}");
+ let failure = failures[0];
+ assert_eq!(failure["reason"], reason, "{case}");
+ assert_eq!(
+ failure.get("field").and_then(Value::as_str),
+ field,
+ "{case}"
+ );
+ assert_eq!(failure["stage"], "decode", "{case}");
+ if frame_index != 0 {
+ assert_eq!(failure["frame_index"], frame_index, "{case}");
+ }
+ assert!(
+ failure["received_bytes"]
+ .as_u64()
+ .is_some_and(|n| n > 0 && n <= wire.len() as u64)
+ );
+ assert_eq!(failure["attempt_index"], 1);
+ let begin = records
+ .iter()
+ .find_map(|r| payload(r, "attempt_begin"))
+ .unwrap();
+ let end = records
+ .iter()
+ .find_map(|r| payload(r, "request_end"))
+ .unwrap();
+ if after_body {
+ assert!(!failure["attempt_token"].is_null());
+ assert_eq!(failure["attempt_token"], begin["attempt_token"]);
+ } else {
+ // Prebody classification precedes formal attempt promotion. The
+ // diagnostic must retain the request and ordinal, without inventing
+ // an unavailable attempt token.
+ assert!(failure["attempt_token"].is_null());
+ }
+ assert!(!failure["request_token"].is_null());
+ assert_eq!(failure["request_token"], end["request_token"]);
+ } else {
+ assert!(
+ records
+ .iter()
+ .all(|r| payload(r, "response_failure").is_none())
+ );
+ }
+ wait_replay_clean(&root.join("replay"));
+ if capture_enabled {
+ assert_capture(
+ &root.join("capture"),
+ &upstream_request,
+ &wire,
+ reason,
+ &records,
+ &facts,
+ );
+ } else {
+ assert!(
+ !root.join("capture").exists(),
+ "default-disabled path created capture state"
+ );
+ }
+}
+
+fn publication(
+ provider: &TestTlsListener,
+ limit_prebody_attempts: bool,
+ no_credential: bool,
+) -> GatewayPublicationSnapshotV3 {
+ let protocol = IngressProtocol::Messages;
+ GatewayPublicationSnapshotV3::seal(
+ "personal/default",
+ "stream-authority",
+ 1,
+ 1,
+ "stream-renderer/v1",
+ vec![AliasPlanV1 {
+ served_model_id: "stream-alias".into(),
+ purpose: "stream diagnostics acceptance".into(),
+ agent_plan_revision: 1,
+ protocols: vec![protocol],
+ overall_timeout_ms: 30_000,
+ // The initial malformed entity is tested with an explicit one-attempt
+ // budget. Other cases retain both candidates and prove no replay once
+ // the client has actually received the first body unit.
+ max_attempts: if limit_prebody_attempts { 1 } else { 2 },
+ routing: None,
+ candidates: (1..=2)
+ .map(|i| {
+ let mut candidate = sealed_native_candidate(
+ i,
+ &format!("stream-target-{i}"),
+ &["stream-credential".into()],
+ provider.authority(),
+ "physical",
+ &[(protocol, IngressProtocol::ChatCompletions)],
+ );
+ if no_credential {
+ candidate.credential_refs = vec![format!("credential/none/stream-{i}")];
+ for profile in &mut candidate.protocol_profiles {
+ profile.connector.authentication =
+ hiroute_domain::GatewayCriticalFactV1::Exact(
+ hiroute_domain::GatewayAuthenticationSemanticsV1::None,
+ );
+ }
+ candidate.protocol_profile_digest =
+ hiroute_domain::CanonicalDigest::of(&candidate.protocol_profiles)
+ .unwrap();
+ }
+ candidate
+ })
+ .collect(),
+ }],
+ vec![GrantV1 {
+ grant_id: "stream-grant".into(),
+ generation: 1,
+ bearer_token_sha256: token_sha256(TOKEN),
+ protocol,
+ route_protocols: Default::default(),
+ routes: [(
+ "stream-alias".into(),
+ ModelRouteV2::Plan {
+ plan_id: "legacy/stream-alias".into(),
+ alias: "stream-alias".into(),
+ revision: 1,
+ semantic_digest: hiroute_domain::CanonicalDigest::of_bytes(b"stream-alias"),
+ },
+ )]
+ .into_iter()
+ .collect(),
+ }],
+ )
+ .unwrap()
+}
+
+fn read_request(stream: &mut impl Read) -> Vec<u8> {
+ let mut bytes = Vec::new();
+ let mut expected = None;
+ loop {
+ let mut chunk = [0u8; 4096];
+ let len = stream.read(&mut chunk).unwrap();
+ assert!(len > 0, "upstream request ended before its body");
+ bytes.extend_from_slice(&chunk[..len]);
+ if expected.is_none()
+ && let Some(end) = bytes.windows(4).position(|w| w == b"\r\n\r\n")
+ {
+ let head = String::from_utf8_lossy(&bytes[..end]);
+ let length = head
+ .lines()
+ .find_map(|line| {
+ line.to_ascii_lowercase()
+ .strip_prefix("content-length: ")
+ .map(str::to_owned)
+ })
+ .unwrap()
+ .parse::<usize>()
+ .unwrap();
+ expected = Some(end + 4 + length);
+ }
+ if expected.is_some_and(|expected| bytes.len() == expected) {
+ return bytes;
+ }
+ assert!(bytes.len() < 1024 * 1024);
+ }
+}
+
+fn payload<'a>(record: &'a Value, name: &str) -> Option<&'a Value> {
+ record
+ .get(name)
+ .or_else(|| record.as_object()?.values().find_map(|v| payload(v, name)))
+}
+fn wait_finished(facts: &Facts, failed: bool) {
+ let deadline = Instant::now() + Duration::from_secs(5);
+ loop {
+ let records = facts.0.lock().unwrap().clone();
+ let finished = records
+ .iter()
+ .any(|r| r.pointer("/fact/kind").and_then(Value::as_str) == Some("request_finished"));
+ // The native completion producer can deliver its usage after the
+ // request's terminal fact. Successful fixtures require both known
+ // usage sources; assert_business still checks the exact count/content.
+ let usage_complete = failed
+ || records
+ .iter()
+ .filter(|r| {
+ r.pointer("/fact/kind").and_then(Value::as_str) == Some("usage_and_cache")
+ })
+ .count()
+ >= 2;
+ if finished && usage_complete {
+ return;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "incomplete terminal/usage facts: {records:?}"
+ );
+ std::thread::sleep(Duration::from_millis(5));
+ }
+}
+fn assert_prebody_relay_business(facts: &[Value]) {
+ let named = |name| {
+ facts
+ .iter()
+ .filter(|r| r.pointer("/fact/kind").and_then(Value::as_str) == Some(name))
+ .collect::<Vec<_>>()
+ };
+ let starts = named("attempt_started");
+ assert_eq!(starts.len(), 2);
+ let finished = named("attempt_finished");
+ assert_eq!(finished.len(), 2);
+ let first = finished.iter().find(|r| r["fact"]["ordinal"] == 1).unwrap();
+ let second = finished.iter().find(|r| r["fact"]["ordinal"] == 2).unwrap();
+ assert_eq!(first["fact"]["outcome"], "rejected");
+ assert_eq!(first["fact"]["disposition"], "continue");
+ assert_eq!(first["fact"]["retryable"], true);
+ assert_eq!(first["fact"]["commits"]["downstream_headers"], "clear");
+ assert_eq!(first["fact"]["commits"]["downstream_semantic"], "clear");
+ assert_eq!(second["fact"]["outcome"], "accepted");
+ assert_eq!(
+ second["fact"]["commits"]["downstream_headers"],
+ "write_confirmed"
+ );
+ let requests = named("request_finished");
+ assert_eq!(requests.len(), 1);
+ assert_eq!(requests[0]["fact"]["outcome"], "accepted");
+ assert_eq!(requests[0]["fact"]["attempts_started"], 2);
+ assert_eq!(requests[0]["fact"]["attempts_finished"], 2);
+ assert_eq!(named("semantic_commit").len(), 1);
+ let usage = named("usage_and_cache");
+ assert_eq!(usage.len(), 2);
+ for record in usage {
+ assert_eq!(record["fact"]["ordinal"], 2);
+ assert_eq!(record["fact"]["input_tokens"], 11);
+ assert_eq!(record["fact"]["output_tokens"], 7);
+ }
+}
+
+fn assert_business(facts: &[Value], failed: bool, after_body: bool) {
+ let named = |name| {
+ facts
+ .iter()
+ .filter(|r| r.pointer("/fact/kind").and_then(Value::as_str) == Some(name))
+ .collect::<Vec<_>>()
+ };
+ assert_eq!(
+ named("attempt_started").len(),
+ 1,
+ "only one actual attempt may run: bounded prebody or committed afterbody"
+ );
+ let finished = named("attempt_finished");
+ assert_eq!(finished.len(), 1);
+ assert_eq!(finished[0]["fact"]["ordinal"], 1);
+ assert_eq!(
+ finished[0]["fact"]["outcome"],
+ if after_body {
+ "postcommit_transport_failed"
+ } else if failed {
+ "rejected"
+ } else {
+ "accepted"
+ }
+ );
+ if failed && !after_body {
+ // Relay eligibility remains Continue; the separately frozen one-attempt
+ // budget ends the request without promoting another candidate.
+ assert_eq!(finished[0]["fact"]["disposition"], "continue");
+ assert_eq!(finished[0]["fact"]["retryable"], true);
+ assert_eq!(
+ finished[0]["fact"]["commits"]["downstream_semantic"],
+ "clear"
+ );
+ }
+ if after_body {
+ let semantic_started = finished[0]["fact"]["commits"]["downstream_semantic"] != "clear";
+ assert_eq!(
+ finished[0]["fact"]["termination_reason"],
+ if semantic_started {
+ "stream_started_no_retry"
+ } else {
+ "provider_encoder_failure"
+ }
+ );
+ assert_eq!(
+ finished[0]["fact"]["stream_outcome"],
+ if semantic_started {
+ "stream_started_no_retry"
+ } else {
+ "aborted_before_semantic_commit"
+ }
+ );
+ }
+ assert_eq!(
+ finished[0]["fact"]["commits"]["upstream_request"],
+ "write_confirmed"
+ );
+ assert_eq!(
+ finished[0]["fact"]["commits"]["downstream_headers"],
+ if failed && !after_body {
+ "clear"
+ } else {
+ "write_confirmed"
+ }
+ );
+ let requests = named("request_finished");
+ assert_eq!(finished[0]["fact"]["cleanup_outcome"], "completed");
+ assert_eq!(requests.len(), 1);
+ assert_eq!(
+ requests[0]["fact"]["outcome"],
+ if failed {
+ if finished[0]["fact"]["commits"]["downstream_semantic"] != "clear" {
+ "postcommit_transport_failed"
+ } else {
+ "failed"
+ }
+ } else {
+ "accepted"
+ }
+ );
+ assert_eq!(requests[0]["fact"]["attempts_started"], 1);
+ assert_eq!(requests[0]["fact"]["attempts_finished"], 1);
+ let semantic_commits = named("semantic_commit").len();
+ assert_eq!(semantic_commits, if failed && !after_body { 0 } else { 1 });
+ let usage = named("usage_and_cache");
+ // These failed entities report no usage: preserve absence rather than
+ // synthesizing zero tokens from an incomplete response.
+ assert_eq!(usage.len(), if failed { 0 } else { 2 });
+ if !failed {
+ assert_eq!(
+ usage
+ .iter()
+ .filter(|r| r["fact"]["source"] == "provider_completion")
+ .count(),
+ 1
+ );
+ }
+ for record in usage {
+ assert_eq!(record["fact"]["ordinal"], 1);
+ assert!(matches!(
+ record["fact"]["source"].as_str(),
+ Some("provider_completion" | "accepted_canonical_model_event")
+ ));
+ assert_eq!(record["fact"]["input_tokens"], 11);
+ assert_eq!(record["fact"]["output_tokens"], 7);
+ }
+}
+fn wait_replay_clean(root: &Path) {
+ let deadline = Instant::now() + Duration::from_secs(5);
+ loop {
+ if !root.exists() || fs::read_dir(root).unwrap().next().is_none() {
+ return;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "Replay resources survived request cleanup"
+ );
+ std::thread::sleep(Duration::from_millis(5));
+ }
+}
+fn create_capture_session(root: &Path) {
+ fs::DirBuilder::new().mode(0o700).create(root).unwrap();
+ let now = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap()
+ .as_secs();
+ let mut file = fs::OpenOptions::new()
+ .create_new(true)
+ .write(true)
+ .mode(0o600)
+ .open(root.join("session.json"))
+ .unwrap();
+ file.write_all(serde_json::to_string(&json!({"source_sha":"a".repeat(40),"binary_sha256":"b".repeat(64),
+ "client":"deterministic-production-listener-fixture","expires_at":now+60,"delete_after":now+120})).unwrap().as_bytes()).unwrap();
+}
+
+fn assert_capture(
+ root: &Path,
+ upstream: &[u8],
+ entity: &[u8],
+ reason: Option<&str>,
+ diagnostics: &[Value],
+ facts: &[Value],
+) {
+ let lock = fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)
+ .open(root.join("active.lock"))
+ .unwrap();
+ // The stable inode survives both clean exit and crashes; kernel ownership
+ // must be released before cleanup can use this same lock.
+ rustix::fs::flock(&lock, rustix::fs::FlockOperation::NonBlockingLockExclusive)
+ .expect("writer ownership was not released");
+ let files: Vec<_> = fs::read_dir(root)
+ .unwrap()
+ .map(|e| e.unwrap().path())
+ .filter(|p| p.extension().is_some_and(|e| e == "capture"))
+ .collect();
+ assert_eq!(files.len(), 1);
+ let path = &files[0];
+ assert_eq!(
+ fs::metadata(path).unwrap().permissions().mode() & 0o777,
+ 0o600
+ );
+ let bytes = fs::read(path).unwrap();
+ assert!(
+ !bytes
+ .windows(b"SENTINEL_UPSTREAM_CREDENTIAL".len())
+ .any(|w| w == b"SENTINEL_UPSTREAM_CREDENTIAL")
+ );
+ let mut remaining = bytes.as_slice();
+ let mut records = Vec::new();
+ while !remaining.is_empty() {
+ assert!(remaining.len() >= 9);
+ let size = u64::from_le_bytes(remaining[1..9].try_into().unwrap()) as usize;
+ assert!(size <= remaining.len() - 9);
+ records.push((remaining[0], &remaining[9..9 + size]));
+ remaining = &remaining[9 + size..];
+ }
+ assert_eq!(records.last().unwrap().0, 8);
+ assert!(records.iter().any(|r| r.0 == 2));
+ let captured = |kind| {
+ records
+ .iter()
+ .filter(|r| r.0 == kind)
+ .flat_map(|r| r.1.iter().copied())
+ .collect::<Vec<_>>()
+ };
+ let body_offset = upstream.windows(4).position(|w| w == b"\r\n\r\n").unwrap() + 4;
+ assert_eq!(
+ captured(1),
+ upstream[body_offset..],
+ "capture must use actual prepared request reader"
+ );
+ assert_eq!(
+ captured(4),
+ entity,
+ "ordered read blocks must reproduce the exact entity"
+ );
+ let context: Value = serde_json::from_slice(records[0].1).unwrap();
+ assert_eq!(
+ context["request_bytes"],
+ (upstream.len() - body_offset) as u64
+ );
+ assert_eq!(context["profile"]["ingress_protocol"], "messages");
+ assert_eq!(
+ context["profile"]["capability"]["upstream_protocol"],
+ "chat_completions"
+ );
+ assert!(!context["chat_tools"].is_null());
+ assert_eq!(records.iter().filter(|r| r.0 == 7).count(), 1);
+ let correlation: Value =
+ serde_json::from_slice(records.iter().find(|r| r.0 == 7).unwrap().1).unwrap();
+ let attempt = diagnostics
+ .iter()
+ .find_map(|r| payload(r, "attempt_begin"))
+ .unwrap();
+ assert_eq!(correlation["request_token"], attempt["request_token"]);
+ assert_eq!(correlation["attempt_token"], attempt["attempt_token"]);
+ assert_eq!(correlation["attempt_index"], attempt["attempt_index"]);
+ assert_eq!(
+ correlation["request_id"],
+ facts
+ .iter()
+ .find_map(|r| r.pointer("/correlation/request_id"))
+ .unwrap()
+ .clone()
+ );
+ let rt = tokio::runtime::Runtime::new().unwrap();
+ for chunks in [0, 1, 4096] {
+ let replay = rt
+ .block_on(crate::runtime::stream_capture::replay_capture(path, chunks))
+ .unwrap();
+ assert_eq!(
+ replay["decoder"]["result"],
+ if reason.is_some() {
+ "rejected"
+ } else {
+ "accepted"
+ }
+ );
+ if let Some(reason) = reason {
+ assert_eq!(replay["decoder"]["reason"], reason);
+ } else {
+ assert_eq!(replay["decoder"]["tool_calls"], 1);
+ }
+ assert!(!replay.to_string().contains("SENTINEL"));
+ }
+}
diff --git a/crates/gateway/src/observation/provider.rs b/crates/gateway/src/observation/provider.rs
index ced83905..9fe97c3a 100644
--- a/crates/gateway/src/observation/provider.rs
+++ b/crates/gateway/src/observation/provider.rs
@@ -111,6 +111,12 @@ impl ProviderRuntimePort for ObservedProductionProvider {
if let (Some(observation), Some(credential_ref)) = (&observation, no_credential_ref) {
observation.no_credential_materialized(&stable_binding_id, &credential_ref);
}
+ // The unauthenticated profile has no credential-read staging hook.
+ // Bind only after authoritative materialization and its None staging.
+ #[cfg(all(unix, debug_assertions))]
+ if let Some(observation) = &observation {
+ inner.register_private_capture(observation);
+ }
let tracker = observation.as_ref().and_then(|observation| {
observation.begin_response_capture(
&stable_binding_id,
diff --git a/crates/gateway/src/observation/request.rs b/crates/gateway/src/observation/request.rs
index 8fb0a3b7..9b4921bd 100644
--- a/crates/gateway/src/observation/request.rs
+++ b/crates/gateway/src/observation/request.rs
@@ -82,6 +82,12 @@ pub(super) struct RequestObservationInner {
#[derive(Default)]
pub(super) struct RequestObservationState {
+ #[cfg(all(unix, debug_assertions))]
+ pub(super) private_capture: Option<(
+ String,
+ String,
+ crate::runtime::stream_capture::PendingCapture,
+ )>,
pub(super) agent_plan_id: Option<String>,
pub(super) candidates: BTreeMap<String, CandidateObservation>,
pub(super) pending_attempt: Option<AttemptObservation>,
@@ -243,32 +249,6 @@ impl RequestObservation {
.token(CorrelationDomain::Attempt, attempt_id)
}
- pub(crate) fn response_failure(
- &self,
- stage: hiroute_diagnostics::event::ResponseFailureStage,
- reason: hiroute_diagnostics::event::ResponseFailureReason,
- ) {
- let state = self.inner.state.lock().unwrap_or_else(|e| e.into_inner());
- let attempt = state
- .accepted_attempt
- .as_ref()
- .or(state.current_attempt.as_ref())
- .or(state.pending_attempt.as_ref());
- let ordinal = attempt
- .map(|a| a.ordinal)
- .filter(|n| *n != 0)
- .unwrap_or(state.next_attempt_ordinal);
- drop(state);
- self.emit_diagnostic(DiagnosticEvent::ResponseFailure(
- hiroute_diagnostics::event::ResponseFailure {
- request_token: self.inner.request_token,
- attempt_index: u64::from(ordinal),
- stage,
- reason,
- },
- ));
- }
-
pub(crate) fn reasoning_cleanup(
&self,
reason: hiroute_diagnostics::event::ReasoningCleanupReason,
diff --git a/crates/gateway/src/observation/request/completion.rs b/crates/gateway/src/observation/request/completion.rs
index eab33381..6d4c3609 100644
--- a/crates/gateway/src/observation/request/completion.rs
+++ b/crates/gateway/src/observation/request/completion.rs
@@ -98,6 +98,10 @@ impl RequestObservation {
.unwrap_or_else(|| failure.termination_reason.as_str());
let mut state = self.lock_state();
let pending = state.pending_attempt.take();
+ #[cfg(all(unix, debug_assertions))]
+ {
+ state.private_capture = None;
+ }
if disposition == Disposition::Continue {
state.previous_attempt_id = pending.and_then(|attempt| attempt.previous_attempt_id);
state.next_attempt_reason = Some(format!("precommit_fallback_after_{error_class}"));
@@ -302,6 +306,23 @@ impl RequestObservation {
],
);
state.current_attempt = Some(attempt.clone());
+ #[cfg(all(unix, debug_assertions))]
+ let capture = state
+ .private_capture
+ .take()
+ .filter(|(binding, credential, _)| {
+ *binding == attempt.stable_binding_id && *credential == attempt.credential_ref
+ });
+ drop(state);
+ #[cfg(all(unix, debug_assertions))]
+ if let Some((_, _, capture)) = capture {
+ capture.promoted(crate::runtime::stream_capture::CaptureCorrelation {
+ request_token: self.inner.request_token,
+ request_id: self.inner.metadata.request_id.clone(),
+ attempt_index: attempt.ordinal,
+ attempt_token: self.attempt_token(&attempt.attempt_id),
+ });
+ }
Some(attempt)
}
diff --git a/crates/gateway/src/observation/request/diagnostic.rs b/crates/gateway/src/observation/request/diagnostic.rs
index e2f3e634..631422fc 100644
--- a/crates/gateway/src/observation/request/diagnostic.rs
+++ b/crates/gateway/src/observation/request/diagnostic.rs
@@ -5,6 +5,88 @@ use crate::server::core_runtime::observation::wire_diagnostic;
use hiroute_diagnostics::{correlation::CorrelationDomain, event::DiagnosticEvent};
impl RequestObservation {
+ #[cfg(test)]
+ pub(crate) fn response_failure(
+ &self,
+ stage: hiroute_diagnostics::event::ResponseFailureStage,
+ reason: hiroute_diagnostics::event::ResponseFailureReason,
+ ) {
+ self.response_failure_detail(stage, reason, None, None);
+ }
+
+ #[cfg(all(unix, debug_assertions))]
+ pub(crate) fn register_private_capture(
+ &self,
+ capture: crate::runtime::stream_capture::PendingCapture,
+ ) {
+ let mut state = self.lock_state();
+ if let Some(pending) = state.pending_attempt.as_ref() {
+ state.private_capture = Some((
+ pending.stable_binding_id.clone(),
+ pending.credential_ref.clone(),
+ capture,
+ ));
+ }
+ }
+
+ #[cfg(all(unix, debug_assertions))]
+ pub(crate) fn private_capture_correlation(
+ &self,
+ ) -> crate::runtime::stream_capture::CaptureCorrelation {
+ let state = self.inner.state.lock().unwrap_or_else(|e| e.into_inner());
+ let attempt = state
+ .accepted_attempt
+ .as_ref()
+ .or(state.current_attempt.as_ref())
+ .or(state.pending_attempt.as_ref());
+ crate::runtime::stream_capture::CaptureCorrelation {
+ request_token: self.inner.request_token,
+ request_id: self.inner.metadata.request_id.clone(),
+ attempt_index: attempt
+ .map(|a| a.ordinal)
+ .filter(|n| *n != 0)
+ .unwrap_or(state.next_attempt_ordinal),
+ attempt_token: attempt
+ .filter(|a| !a.attempt_id.is_empty())
+ .and_then(|a| self.attempt_token(&a.attempt_id)),
+ }
+ }
+
+ pub(crate) fn response_failure_detail(
+ &self,
+ stage: hiroute_diagnostics::event::ResponseFailureStage,
+ reason: hiroute_diagnostics::event::ResponseFailureReason,
+ field: Option<hiroute_diagnostics::event::ResponseFailureField>,
+ position: Option<(u64, u64)>,
+ ) {
+ let state = self.inner.state.lock().unwrap_or_else(|e| e.into_inner());
+ let attempt = state
+ .accepted_attempt
+ .as_ref()
+ .or(state.current_attempt.as_ref())
+ .or(state.pending_attempt.as_ref());
+ let ordinal = attempt
+ .map(|a| a.ordinal)
+ .filter(|n| *n != 0)
+ .unwrap_or(state.next_attempt_ordinal);
+ let attempt_token = attempt
+ .filter(|a| !a.attempt_id.is_empty())
+ .and_then(|a| self.attempt_token(&a.attempt_id));
+ drop(state);
+ self.emit_diagnostic(DiagnosticEvent::ResponseFailure(
+ hiroute_diagnostics::event::ResponseFailure {
+ request_token: self.inner.request_token,
+ attempt_index: u64::from(ordinal),
+ stage,
+ reason,
+ attempt_token,
+ field,
+ frame_index: position.map(|p| p.0).filter(|n| *n != 0),
+ received_bytes: position.map(|p| p.1),
+ },
+ ));
+ }
+
pub(in crate::server::core_runtime::observation) fn wire_diagnostic(
&self,
mut event: hiroute_diagnostics::event::UpstreamWire,
diff --git a/crates/gateway/src/runtime.rs b/crates/gateway/src/runtime.rs
index 26a7de71..a2ed7071 100644
--- a/crates/gateway/src/runtime.rs
+++ b/crates/gateway/src/runtime.rs
@@ -6,6 +6,8 @@
mod driver;
mod state;
+#[cfg(all(unix, debug_assertions))]
+pub mod stream_capture;
pub use driver::*;
pub use state::StateAccessError;
diff --git a/crates/gateway/src/runtime/driver.rs b/crates/gateway/src/runtime/driver.rs
index 240c30cc..2a55b11b 100644
--- a/crates/gateway/src/runtime/driver.rs
+++ b/crates/gateway/src/runtime/driver.rs
@@ -40,7 +40,7 @@ use super::state::{
mod materialization;
mod response;
-mod response_diagnostics;
+pub(crate) mod response_diagnostics;
mod selection;
pub(crate) use materialization::TargetResolver;
@@ -165,6 +165,8 @@ impl ProductionReplaySeedEnvelope {
}
pub struct ProductionAttemptState {
+ #[cfg(all(unix, debug_assertions))]
+ capture: Option<super::stream_capture::Capture>,
response_status: Option<http::StatusCode>,
resolved_target: TransportTarget,
permits: OwnedAttemptStatePermits,
@@ -190,6 +192,16 @@ pub struct ProductionAttemptState {
}
impl ProductionAttemptState {
+ #[cfg(all(unix, debug_assertions))]
+ pub(crate) fn register_private_capture(
+ &self,
+ request: &crate::server::core_runtime::observation::RequestObservation,
+ ) {
+ if let Some(capture) = &self.capture {
+ capture.register(request);
+ }
+ }
+
pub(crate) fn chat_tool_projection_for_observation(
&self,
) -> Option<&adapters::ChatToolProjection> {
@@ -198,6 +210,8 @@ impl ProductionAttemptState {
}
pub struct ProductionReadiness {
+ #[cfg(all(unix, debug_assertions))]
+ capture: Option<super::stream_capture::Capture>,
response_status: http::StatusCode,
content_type: &'static str,
prefix: ChargedBodyQueue,
diff --git a/crates/gateway/src/runtime/driver/materialization.rs b/crates/gateway/src/runtime/driver/materialization.rs
index baa91b2d..66d10fd9 100644
--- a/crates/gateway/src/runtime/driver/materialization.rs
+++ b/crates/gateway/src/runtime/driver/materialization.rs
@@ -459,6 +459,13 @@ pub(super) async fn materialize_attempt(
),
Some(_) | None => None,
};
+ #[cfg(all(unix, debug_assertions))]
+ let capture = crate::runtime::stream_capture::Capture::begin(
+ profile,
+ prepared.chat_tool_projection.as_ref(),
+ prepared.wire_len,
+ streaming,
+ );
let chat_tool_projection = prepared.chat_tool_projection.take();
let body = adapters::sequential_attempt_body(
prepared.clone(),
@@ -467,6 +474,11 @@ pub(super) async fn materialize_attempt(
quantum,
)
.map_err(safe_error)?;
+ #[cfg(all(unix, debug_assertions))]
+ let body = match capture.as_ref() {
+ Some(capture) => capture.wrap(body),
+ None => body,
+ };
let lease = context.leases.acquire().map_err(safe_error)?;
let mut headers = HeaderMap::new();
let authority = resolved_target.authority.as_ref();
@@ -548,6 +560,8 @@ pub(super) async fn materialize_attempt(
body,
},
ProductionAttemptState {
+ #[cfg(all(unix, debug_assertions))]
+ capture,
response_status: None,
resolved_target,
permits,
diff --git a/crates/gateway/src/runtime/driver/response.rs b/crates/gateway/src/runtime/driver/response.rs
index 3bdfa70f..381f6636 100644
--- a/crates/gateway/src/runtime/driver/response.rs
+++ b/crates/gateway/src/runtime/driver/response.rs
@@ -57,6 +57,10 @@ pub(super) fn classify_precommit(
state: &mut ProductionAttemptState,
event: PrecommitEvent,
) -> Result<PrecommitClassification<ProductionReadiness, ProductionDecodedSse>, Arc<str>> {
+ #[cfg(all(unix, debug_assertions))]
+ if let Some(capture) = &state.capture {
+ capture.response(&event);
+ }
match event {
PrecommitEvent::ResponseHead(head) if head.status().is_informational() => {
Ok(PrecommitClassification::pending())
@@ -126,7 +130,8 @@ pub(super) fn classify_precommit(
.ok_or_else(|| Arc::from("native response projector is unavailable"))?
.feed(bytes.bytes(), false);
drop(bytes);
- if projected.is_err() {
+ if let Err(error) = projected {
+ response_diagnostics::adapter(FailureStage::NativeProjection, error);
let raw = response_decode_failure(status, state.retry_after);
return classify_state_failure(
state,
@@ -142,7 +147,8 @@ pub(super) fn classify_precommit(
.ok_or_else(|| Arc::from("native response decoder is unavailable"))?
.feed(bytes.bytes(), false);
drop(bytes);
- if decoded.is_err() {
+ if let Err(error) = decoded {
+ response_diagnostics::adapter(FailureStage::Decode, error);
let raw = response_decode_failure(status, state.retry_after);
return classify_state_failure(
state,
@@ -380,6 +386,10 @@ pub(super) fn accepted_response_head(
readiness: &ProductionReadiness,
published: &PublishedDisposition,
) -> Result<GatewayResponseHead, Arc<str>> {
+ #[cfg(all(unix, debug_assertions))]
+ if let Some(capture) = &readiness.capture {
+ capture.record_attempt_correlation();
+ }
match published.disposition {
Disposition::Accept | Disposition::Terminate => {
let mut headers = HeaderMap::new();
@@ -399,6 +409,24 @@ pub(super) fn accepted_response_head(
pub(super) fn encode_accepted_event(
readiness: &mut ProductionReadiness,
event: ProviderAcceptedEvent<ProductionDecodedSse>,
+) -> Result<Option<AcceptedBodyFrame>, Arc<str>> {
+ #[cfg(all(unix, debug_assertions))]
+ if let (Some(capture), ProviderAcceptedEvent::Raw(raw)) = (&readiness.capture, &event) {
+ capture.response(raw);
+ }
+ let result = encode_accepted_event_inner(readiness, event);
+ #[cfg(all(unix, debug_assertions))]
+ if result.is_err()
+ && let Some(capture) = &readiness.capture
+ {
+ capture.failed();
+ }
+ result
+}
+
+fn encode_accepted_event_inner(
+ readiness: &mut ProductionReadiness,
+ event: ProviderAcceptedEvent<ProductionDecodedSse>,
) -> Result<Option<AcceptedBodyFrame>, Arc<str>> {
let event = match event {
ProviderAcceptedEvent::Terminal {
@@ -470,7 +498,9 @@ pub(super) fn encode_accepted_event(
.take()
.ok_or_else(|| Arc::from("accepted native decoder is unavailable"))?
.finish()
- .map_err(|_| Arc::from("accepted native stream ended without terminal"))?;
+ .map_err(|error| {
+ response_diagnostics::adapter(FailureStage::EndOfStream, error)
+ })?;
(rendered, true)
};
queue_accepted_stream_output(readiness, rendered, projected_terminal)
@@ -743,6 +773,10 @@ fn classify_state_failure(
raw: RawAttemptFailure,
local_status: StatusCode,
) -> Result<PrecommitClassification<ProductionReadiness, ProductionDecodedSse>, Arc<str>> {
+ #[cfg(all(unix, debug_assertions))]
+ if let Some(capture) = &state.capture {
+ capture.failed();
+ }
let failure = classify_failure(&raw, connector_error_profile(&state.profile)?);
let facts = failure_facts(&failure);
state.classified_failure = Some(failure);
@@ -784,6 +818,8 @@ fn take_readiness(
prefix_eos_pending: bool,
) -> Result<ProductionReadiness, Arc<str>> {
Ok(ProductionReadiness {
+ #[cfg(all(unix, debug_assertions))]
+ capture: state.capture.take(),
response_status,
content_type,
prefix: state
@@ -897,9 +933,13 @@ fn decode_stream_chunk_readiness(
.renderer
.as_mut()
.ok_or_else(|| Arc::from("accepted client renderer is unavailable"))?;
- let mut status = decoder
- .feed(bytes, end_stream)
- .map_err(|error| response_diagnostics::adapter(FailureStage::Decode, error))?;
+ let mut status = decoder.feed(bytes, end_stream).map_err(|error| {
+ response_diagnostics::adapter_at(
+ FailureStage::Decode,
+ error,
+ Some(decoder.diagnostic_position()),
+ )
+ })?;
let mut output = Vec::new();
loop {
for event in decoder.take_events() {
@@ -926,9 +966,13 @@ fn decode_stream_chunk_readiness(
if status != adapters::ResponseDecodeStatus::NeedDrain {
break;
}
- status = decoder
- .resume()
- .map_err(|_| Arc::from("accepted native stream drain failed"))?;
+ status = decoder.resume().map_err(|error| {
+ response_diagnostics::adapter_at(
+ FailureStage::Decode,
+ error,
+ Some(decoder.diagnostic_position()),
+ )
+ })?;
}
let terminal = status == adapters::ResponseDecodeStatus::Terminal;
if terminal && !end_stream {
@@ -943,10 +987,13 @@ fn finish_native_stream_on_terminal(
let active = decoder
.as_mut()
.ok_or_else(|| Arc::from("native stream decoder is unavailable"))?;
- if active
- .feed(&[], true)
- .map_err(|_| Arc::from("native stream has a malformed terminal tail"))?
- != adapters::ResponseDecodeStatus::Terminal
+ if active.feed(&[], true).map_err(|error| {
+ response_diagnostics::adapter_at(
+ FailureStage::EndOfStream,
+ error,
+ Some(active.diagnostic_position()),
+ )
+ })? != adapters::ResponseDecodeStatus::Terminal
|| !active.take_events().is_empty()
{
return Err(Arc::from("native stream has events after terminal"));
@@ -955,7 +1002,7 @@ fn finish_native_stream_on_terminal(
.take()
.expect("terminal decoder is present")
.finish()
- .map_err(|_| Arc::from("native stream terminal is incomplete"))?;
+ .map_err(|error| response_diagnostics::adapter(FailureStage::EndOfStream, error))?;
Ok(())
}
diff --git a/crates/gateway/src/runtime/driver/response_diagnostics.rs b/crates/gateway/src/runtime/driver/response_diagnostics.rs
index 583223dc..61950487 100644
--- a/crates/gateway/src/runtime/driver/response_diagnostics.rs
+++ b/crates/gateway/src/runtime/driver/response_diagnostics.rs
@@ -3,13 +3,18 @@ use crate::server::core_runtime::{
model_ir::{ModelError, ModelIrError},
};
use hiroute_diagnostics::event::{
- ResponseFailureReason as Reason, ResponseFailureStage as Stage, WireProviderError,
+ ResponseFailureField as Field, ResponseFailureReason as Reason, ResponseFailureStage as Stage,
+ WireProviderError,
};
use std::sync::Arc;
pub(super) fn note(stage: Stage, reason: Reason) {
+ report(stage, reason, None, None);
+}
+
+fn report(stage: Stage, reason: Reason, field: Option<Field>, position: Option<(u64, u64)>) {
if let Some(observation) = crate::server::core_runtime::observation::active_request() {
- observation.response_failure(stage, reason);
+ observation.response_failure_detail(stage, reason, field, position);
}
}
@@ -64,31 +69,87 @@ fn model_error_code(error: &ModelError, response_status: Option<u16>) -> WirePro
}
pub(super) fn adapter(stage: Stage, error: ProtocolAdapterError) -> Arc<str> {
- note(stage, category(&error));
+ adapter_at(stage, error, None)
+}
+
+pub(super) fn adapter_at(
+ stage: Stage,
+ error: ProtocolAdapterError,
+ position: Option<(u64, u64)>,
+) -> Arc<str> {
+ let (reason, field) = category(&error);
+ report(stage, reason, field, position);
Arc::from("provider response processing failed")
}
-fn category(error: &ProtocolAdapterError) -> Reason {
- match error {
+pub(crate) fn category(error: &ProtocolAdapterError) -> (Reason, Option<Field>) {
+ let mut field = None;
+ let reason = match error {
ProtocolAdapterError::ModelIr(error) => match error {
ModelIrError::BufferLimit(_) => Reason::ResourceLimit,
ModelIrError::InvalidJson(_) => Reason::InvalidJson,
ModelIrError::InvalidSse(_) => Reason::InvalidSse,
- ModelIrError::ExpectedObject | ModelIrError::InvalidField(_) => Reason::InvalidField,
+ ModelIrError::ExpectedObject => {
+ field = Some(Field::Object);
+ Reason::InvalidField
+ }
+ ModelIrError::InvalidField(name) => {
+ field = Some(known_field(name));
+ Reason::InvalidField
+ }
+ ModelIrError::UnsupportedField(name) => {
+ field = Some(known_field(name));
+ Reason::UnsupportedField
+ }
+ ModelIrError::UnsupportedValue(_) => Reason::UnsupportedValue,
+ ModelIrError::InvalidToolArguments(_) => Reason::InvalidToolArguments,
+ ModelIrError::MissingToolIdentity(_) => Reason::MissingToolIdentity,
+ ModelIrError::ProviderStateNotPortable => Reason::ProviderStateNotPortable,
+ ModelIrError::ToolIdBindingRequired(_) => Reason::ToolIdBindingRequired,
+ ModelIrError::ToolContinuationConflict => Reason::ToolContinuationConflict,
ModelIrError::InvalidResponseLifecycle(_) => Reason::InvalidLifecycle,
ModelIrError::MissingTerminalEvent => Reason::MissingTerminal,
ModelIrError::DuplicateTerminalEvent => Reason::DuplicateTerminal,
- _ => Reason::OtherProtocol,
+ ModelIrError::ResponsesPreviousResponseIdUnsupported
+ | ModelIrError::ResponsesConversationUnsupported => Reason::UnsupportedValue,
},
ProtocolAdapterError::ClientUnrepresentable(_) => Reason::Unrepresentable,
- _ => Reason::OtherProtocol,
+ ProtocolAdapterError::Capability(_) => Reason::Capability,
+ ProtocolAdapterError::Context(_) => Reason::ContextProjection,
+ ProtocolAdapterError::Serialization(_) => Reason::Serialization,
+ };
+ (reason, field)
+}
+
+// Exact allowlist only: never parse an error message or echo an unknown field path.
+fn known_field(name: &str) -> Field {
+ match name {
+ "choices" => Field::Choices,
+ "delta" => Field::Delta,
+ "content" => Field::Content,
+ "tool_calls" => Field::ToolCalls,
+ "tool id" | "tool_call.id" => Field::ToolId,
+ "tool name" | "tool_call.function.name" => Field::ToolName,
+ "arguments" | "tool_call.function.arguments" => Field::ToolArguments,
+ "finish_reason" => Field::FinishReason,
+ "usage" => Field::Usage,
+ "system_fingerprint" => Field::SystemFingerprint,
+ "service_tier" => Field::ServiceTier,
+ "logprobs" => Field::Logprobs,
+ "model" => Field::Model,
+ "id" => Field::Id,
+ "type" => Field::Type,
+ "index" => Field::Index,
+ "role" => Field::Role,
+ "output" => Field::Output,
+ "status" => Field::Status,
+ _ => Field::Unknown,
}
}
#[cfg(test)]
mod tests {
use super::*;
-
#[test]
fn model_errors_keep_safe_attribution_without_provider_text() {
let known = ModelError { status: Some(400), code: Some("InvalidParameter".into()), message: Some("<400> InternalError.Algo.InvalidParameter: The thinking_budget parameter must be a positive integer and not greater than 81920".into()), retryable: None };
@@ -115,26 +176,52 @@ mod tests {
}
#[test]
- fn diagnostic_categories_distinguish_local_limits_and_protocol_failures_without_payloads() {
+ fn diagnostic_categories_are_closed_and_never_serialize_dynamic_messages() {
for (error, expected) in [
(ModelIrError::BufferLimit(100), Reason::ResourceLimit),
(
- ModelIrError::InvalidJson("secret provider body".into()),
+ ModelIrError::InvalidJson("SECRET_BODY".into()),
Reason::InvalidJson,
),
(
- ModelIrError::InvalidSse("secret event".into()),
+ ModelIrError::InvalidSse("SECRET_BODY".into()),
Reason::InvalidSse,
),
+ (
+ ModelIrError::UnsupportedField("SECRET_BODY".into()),
+ Reason::UnsupportedField,
+ ),
+ (
+ ModelIrError::UnsupportedValue("SECRET_BODY".into()),
+ Reason::UnsupportedValue,
+ ),
+ (
+ ModelIrError::MissingToolIdentity("SECRET_TOOL".into()),
+ Reason::MissingToolIdentity,
+ ),
+ (
+ ModelIrError::InvalidToolArguments("SECRET_KEY".into()),
+ Reason::InvalidToolArguments,
+ ),
+ (
+ ModelIrError::ProviderStateNotPortable,
+ Reason::ProviderStateNotPortable,
+ ),
+ (
+ ModelIrError::InvalidResponseLifecycle("SECRET_BODY".into()),
+ Reason::InvalidLifecycle,
+ ),
(ModelIrError::MissingTerminalEvent, Reason::MissingTerminal),
(
ModelIrError::DuplicateTerminalEvent,
Reason::DuplicateTerminal,
),
] {
- let category = category(&error.into());
- assert_eq!(category, expected);
- assert!(!serde_json::to_string(&category).unwrap().contains("secret"));
+ let result = category(&error.into());
+ assert_eq!(result.0, expected);
+ assert!(!serde_json::to_string(&result).unwrap().contains("SECRET"));
}
+ assert_eq!(known_field("system_fingerprint"), Field::SystemFingerprint);
+ assert_eq!(known_field("tool_calls.SECRET_TOOL"), Field::Unknown);
}
}
diff --git a/crates/gateway/src/runtime/driver/response_prefix.rs b/crates/gateway/src/runtime/driver/response_prefix.rs
index 3d059642..594e7973 100644
--- a/crates/gateway/src/runtime/driver/response_prefix.rs
+++ b/crates/gateway/src/runtime/driver/response_prefix.rs
@@ -92,9 +92,13 @@ fn decode_prefix(
.renderer
.as_mut()
.ok_or_else(|| Arc::from("client stream renderer is unavailable"))?;
- let mut status = decoder
- .feed(bytes, end_stream)
- .map_err(|error| response_diagnostics::adapter(FailureStage::Decode, error))?;
+ let mut status = decoder.feed(bytes, end_stream).map_err(|error| {
+ response_diagnostics::adapter_at(
+ FailureStage::Decode,
+ error,
+ Some(decoder.diagnostic_position()),
+ )
+ })?;
let mut output = Vec::new();
loop {
for event in decoder.take_events() {
@@ -120,9 +124,13 @@ fn decode_prefix(
if status != adapters::ResponseDecodeStatus::NeedDrain {
break;
}
- status = decoder
- .resume()
- .map_err(|_| Arc::from("stream prefix drain failed"))?;
+ status = decoder.resume().map_err(|error| {
+ response_diagnostics::adapter_at(
+ FailureStage::Decode,
+ error,
+ Some(decoder.diagnostic_position()),
+ )
+ })?;
}
let terminal = status == adapters::ResponseDecodeStatus::Terminal;
if terminal && !end_stream {
@@ -133,7 +141,7 @@ fn decode_prefix(
.take()
.expect("prefix decoder is present")
.finish()
- .map_err(|_| Arc::from("stream prefix ended without terminal"))?;
+ .map_err(|error| response_diagnostics::adapter(FailureStage::EndOfStream, error))?;
}
push_native_prefix_unit(state, output, terminal || end_stream)?;
Ok(None)
diff --git a/crates/gateway/src/runtime/driver/response_terminal_tests.rs b/crates/gateway/src/runtime/driver/response_terminal_tests.rs
index fd37b7c9..9d804e27 100644
--- a/crates/gateway/src/runtime/driver/response_terminal_tests.rs
+++ b/crates/gateway/src/runtime/driver/response_terminal_tests.rs
@@ -29,6 +29,8 @@ fn accepted_codex_terminal_event_crosses_transport_frames_and_keeps_eos() {
)
.unwrap();
let mut readiness = ProductionReadiness {
+ #[cfg(all(unix, debug_assertions))]
+ capture: None,
response_status: StatusCode::OK,
content_type: "text/event-stream",
prefix,
@@ -133,6 +135,8 @@ fn accepted_chat_terminal_closes_responses_stream_without_waiting_for_transport_
);
let client = ClientProtocolProfile::for_candidate(&profile).unwrap();
let mut readiness = ProductionReadiness {
+ #[cfg(all(unix, debug_assertions))]
+ capture: None,
response_status: StatusCode::OK,
content_type: "text/event-stream",
prefix,
diff --git a/crates/gateway/src/runtime/stream_capture.rs b/crates/gateway/src/runtime/stream_capture.rs
new file mode 100644
index 00000000..f6b35968
--- /dev/null
+++ b/crates/gateway/src/runtime/stream_capture.rs
@@ -0,0 +1,531 @@
+//! Opt-in Unix Debug-only forensic capture. Never feeds the product diagnostic log.
+//! A session owns a private directory and an exclusive capture lock. Failed/partial
+//! capture is not replay evidence; it never changes the business result.
+use std::fs::{self, File, OpenOptions};
+use std::io::{self, Read, Write};
+use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, Mutex, Weak};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use hiroute_gateway_core::runtime::attempt::{AttemptError, AttemptRequestBodyReader};
+use hiroute_gateway_core::runtime::body::ChargedBytes;
+use serde::{Deserialize, Serialize};
+
+use crate::server::core_runtime::{adapters, profiles::CandidateProtocolProfile};
+
+mod replay;
+pub use replay::replay_capture;
+#[cfg(test)]
+mod tests;
+
+const MAX_FILE: u64 = 8 * 1024 * 1024;
+const MAX_TOTAL: u64 = 32 * 1024 * 1024;
+const MAX_ATTEMPTS: usize = 4;
+const MAX_RECORDS: u64 = 65536;
+
+#[derive(Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct Session {
+ source_sha: String,
+ binary_sha256: String,
+ client: String,
+ expires_at: u64,
+ delete_after: u64,
+}
+
+#[derive(Serialize, Deserialize)]
+struct Context {
+ session: Session,
+ profile: CandidateProtocolProfile,
+ chat_tools: Option<adapters::ChatToolProjection>,
+ request_bytes: u64,
+ streaming: bool,
+ correlation: serde_json::Value,
+ // Values of transport headers have no role in the response decoder.
+ transport_header_values_redacted: bool,
+}
+
+#[derive(Clone)]
+pub(crate) struct Capture(Arc<Mutex<Writer>>);
+
+pub(crate) struct PendingCapture(Weak<Mutex<Writer>>);
+
+#[derive(Serialize)]
+pub(crate) struct CaptureCorrelation {
+ pub(crate) request_token: Option<hiroute_diagnostics::identity::CorrelationToken>,
+ pub(crate) request_id: String,
+ pub(crate) attempt_index: u32,
+ pub(crate) attempt_token: Option<hiroute_diagnostics::identity::CorrelationToken>,
+}
+
+impl PendingCapture {
+ pub(crate) fn promoted(self, correlation: CaptureCorrelation) {
+ if let Some(writer) = self.0.upgrade() {
+ Capture(writer).correlate(&correlation, true);
+ }
+ }
+}
+
+struct Writer {
+ file: File,
+ path: PathBuf,
+ _lock: File,
+ root: PathBuf,
+ written: u64,
+ records: u64,
+ limit: u64,
+ expires_at: u64,
+ active: bool,
+ attempt_correlated: bool,
+ failed: bool,
+ failed_seal_offset: Option<u64>,
+}
+
+fn now() -> u64 {
+ SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap_or_default()
+ .as_secs()
+}
+
+fn private_file(path: &Path) -> io::Result<File> {
+ OpenOptions::new()
+ .write(true)
+ .create_new(true)
+ .mode(0o600)
+ .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)
+ .open(path)
+}
+
+fn read_private(path: &Path, limit: u64) -> io::Result<Vec<u8>> {
+ let file = OpenOptions::new()
+ .read(true)
+ .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)
+ .open(path)?;
+ let metadata = file.metadata()?;
+ if !metadata.is_file()
+ || metadata.mode() & 0o777 != 0o600
+ || metadata.uid() != rustix::process::geteuid().as_raw()
+ || metadata.len() > limit
+ {
+ return Err(io::Error::other("unsafe or oversized capture file"));
+ }
+ let mut bytes = Vec::new();
+ file.take(limit + 1).read_to_end(&mut bytes)?;
+ if bytes.len() as u64 > limit {
+ return Err(io::Error::other("capture file grew"));
+ }
+ Ok(bytes)
+}
+
+impl Capture {
+ pub(crate) fn begin(
+ profile: &CandidateProtocolProfile,
+ chat_tools: Option<&adapters::ChatToolProjection>,
+ request_bytes: usize,
+ streaming: bool,
+ ) -> Option<Self> {
+ let root = PathBuf::from(std::env::var_os("HIROUTE_PRIVATE_STREAM_CAPTURE")?);
+ Self::open(&root, profile, chat_tools, request_bytes, streaming).ok()
+ }
+
+ pub(crate) fn register(
+ &self,
+ request: &crate::server::core_runtime::observation::RequestObservation,
+ ) {
+ request.register_private_capture(PendingCapture(Arc::downgrade(&self.0)));
+ }
+
+ fn open(
+ root: &Path,
+ profile: &CandidateProtocolProfile,
+ chat_tools: Option<&adapters::ChatToolProjection>,
+ request_bytes: usize,
+ streaming: bool,
+ ) -> io::Result<Self> {
+ // Canonical spelling rejects symlinks in every component, not just the leaf.
+ let metadata = fs::symlink_metadata(root)?;
+ if !root.is_absolute()
+ || fs::canonicalize(root)? != root
+ || !metadata.is_dir()
+ || metadata.mode() & 0o777 != 0o700
+ || metadata.uid() != rustix::process::geteuid().as_raw()
+ {
+ return Err(io::Error::other(
+ "capture requires a private canonical directory",
+ ));
+ }
+ let session: Session =
+ serde_json::from_slice(&read_private(&root.join("session.json"), 4096)?)?;
+ let time = now();
+ if session.expires_at <= time
+ || session.expires_at > time + 3600
+ || session.delete_after <= session.expires_at
+ || session.delete_after > time + 86400
+ || !hex(&session.source_sha, 40)
+ || !hex(&session.binary_sha256, 64)
+ || session.client.is_empty()
+ || session.client.len() > 128
+ || root.join("stopped").exists()
+ {
+ return Err(io::Error::other(
+ "invalid, expired or stopped capture session",
+ ));
+ }
+ let lock = OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .truncate(false)
+ .mode(0o600)
+ .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)
+ .open(root.join("active.lock"))?;
+ let metadata = lock.metadata()?;
+ if !metadata.is_file()
+ || metadata.mode() & 0o777 != 0o600
+ || metadata.uid() != rustix::process::geteuid().as_raw()
+ {
+ return Err(io::Error::other("unsafe capture lock"));
+ }
+ // Keep the inode stable. Kernel ownership disappears on crash/kill and
+ // cleanup acquires this same lock before removing an expired session.
+ rustix::fs::flock(&lock, rustix::fs::FlockOperation::NonBlockingLockExclusive)?;
+ if root.join("stopped").exists() || now() >= session.expires_at {
+ return Err(io::Error::other("capture session stopped or expired"));
+ }
+ Self::open_locked(
+ root,
+ lock,
+ session,
+ profile,
+ chat_tools,
+ request_bytes,
+ streaming,
+ )
+ }
+
+ fn open_locked(
+ root: &Path,
+ lock: File,
+ session: Session,
+ profile: &CandidateProtocolProfile,
+ chat_tools: Option<&adapters::ChatToolProjection>,
+ request_bytes: usize,
+ streaming: bool,
+ ) -> io::Result<Self> {
+ let mut total = 0u64;
+ let mut attempts = 0;
+ for entry in fs::read_dir(root)? {
+ let entry = entry?;
+ let metadata = fs::symlink_metadata(entry.path())?;
+ if !metadata.is_file() {
+ return Err(io::Error::other("unexpected capture directory entry"));
+ }
+ total = total.saturating_add(metadata.len());
+ if entry.path().extension().is_some_and(|e| e == "capture") {
+ attempts += 1;
+ }
+ }
+ if attempts >= MAX_ATTEMPTS || total >= MAX_TOTAL || request_bytes as u64 >= MAX_FILE {
+ return Err(io::Error::other("capture budget exhausted"));
+ }
+ let mut profile = profile.clone();
+ if let crate::server::core_runtime::profiles::CriticalFact::Exact(headers) =
+ &mut profile.connector.headers
+ {
+ for (_, value) in &mut headers.required_headers {
+ *value = "redacted".into();
+ }
+ }
+ let context = Context {
+ session,
+ profile,
+ chat_tools: chat_tools.cloned(),
+ request_bytes: request_bytes as u64,
+ streaming,
+ correlation: correlation(),
+ transport_header_values_redacted: true,
+ };
+ let path = root.join(format!("attempt-{}.capture", attempts + 1));
+ let file = private_file(&path)?;
+ let mut writer = Writer {
+ file,
+ path,
+ _lock: lock,
+ root: root.to_owned(),
+ written: 0,
+ records: 0,
+ limit: MAX_FILE.min(MAX_TOTAL - total),
+ expires_at: context.session.expires_at,
+ active: true,
+ attempt_correlated: false,
+ failed: false,
+ failed_seal_offset: None,
+ };
+ writer.write(0, &serde_json::to_vec(&context)?)?;
+ Ok(Self(Arc::new(Mutex::new(writer))))
+ }
+
+ fn record(&self, kind: u8, bytes: &[u8]) {
+ let mut writer = self.0.lock().unwrap_or_else(|p| p.into_inner());
+ if writer.active && writer.write(kind, bytes).is_err() {
+ writer.active = false;
+ }
+ }
+
+ pub(crate) fn response(&self, event: &hiroute_gateway_core::runtime::attempt::PrecommitEvent) {
+ use hiroute_gateway_core::runtime::attempt::PrecommitEvent;
+ self.record_attempt_correlation();
+ match event {
+ PrecommitEvent::ResponseHead(head) => {
+ self.record(3, &head.status().as_u16().to_le_bytes());
+ }
+ PrecommitEvent::Body(bytes) => self.record(4, bytes.bytes()),
+ PrecommitEvent::EndStream => self.record(5, &[]),
+ PrecommitEvent::SseEvent { .. } => {}
+ }
+ }
+
+ pub(crate) fn record_attempt_correlation(&self) {
+ // Snapshot observation before taking the writer lock. Formal promotion
+ // supplies its own validated snapshot and never calls back into state.
+ if let Some(request) = crate::server::core_runtime::observation::active_request() {
+ self.correlate(&request.private_capture_correlation(), false);
+ }
+ }
+
+ fn correlate(&self, correlation: &CaptureCorrelation, allow_sealed: bool) {
+ let mut writer = self.0.lock().unwrap_or_else(|p| p.into_inner());
+ if writer.attempt_correlated || correlation.attempt_token.is_none() {
+ return;
+ }
+ let Ok(bytes) = serde_json::to_vec(correlation) else {
+ return;
+ };
+ if writer.active {
+ writer.attempt_correlated = true;
+ if writer.write(7, &bytes).is_err() {
+ writer.active = false;
+ }
+ } else if allow_sealed && let Some(offset) = writer.failed_seal_offset.take() {
+ // Never overwrite the sample after publishing `stopped`: its owner
+ // may terminate us at any instruction. Publish a complete replacement.
+ writer.attempt_correlated = true;
+ let _ = writer.correlate_sealed(offset, &bytes);
+ }
+ }
+
+ pub(crate) fn failed(&self) {
+ let mut writer = self.0.lock().unwrap_or_else(|p| p.into_inner());
+ if writer.failed {
+ return;
+ }
+ writer.failed = true;
+ if writer.active && writer.write(6, &[]).is_err() {
+ writer.active = false;
+ }
+ // The supervisor can terminate the process immediately after `stopped`.
+ // Seal and flush this sample before publishing that signal, even while
+ // the request reader or response driver still owns another Capture.
+ writer.failed_seal_offset = writer.seal();
+ let _ = private_file(&writer.root.join("stopped"));
+ }
+
+ pub(crate) fn wrap(
+ &self,
+ body: Box<dyn AttemptRequestBodyReader>,
+ ) -> Box<dyn AttemptRequestBodyReader> {
+ Box::new(CapturedBody {
+ body,
+ capture: self.clone(),
+ ended: false,
+ })
+ }
+}
+
+fn correlation() -> serde_json::Value {
+ crate::server::core_runtime::observation::active_request()
+ .and_then(|r| serde_json::to_value(r.private_capture_correlation()).ok())
+ .unwrap_or(serde_json::Value::Null)
+}
+
+fn hex(value: &str, len: usize) -> bool {
+ value.len() == len && value.bytes().all(|c| c.is_ascii_hexdigit())
+}
+
+impl Writer {
+ fn seal(&mut self) -> Option<u64> {
+ // A missing seal is never replayable. Partial requests remain rejected
+ // by the independent request-completeness check during replay.
+ let seal_offset = self.written;
+ let sealed = self.active && self.write(8, &[]).is_ok();
+ self.active = false;
+ let flushed = self.file.sync_all().is_ok();
+ if !flushed && sealed {
+ self.invalidate_seal(seal_offset);
+ }
+ if !flushed || !sealed {
+ return None;
+ }
+ Some(seal_offset)
+ }
+
+ fn invalidate_seal(&self, offset: u64) {
+ // A partial write or flush failure must not advertise sealed evidence.
+ if self.file.set_len(offset).is_err() {
+ let _ = fs::remove_file(&self.path);
+ }
+ }
+
+ fn correlate_sealed(&mut self, offset: u64, bytes: &[u8]) -> io::Result<()> {
+ self.check_correlation_tail(offset, bytes.len(), true)?;
+ let replacement = CorrelationReplacement {
+ file: private_file(&self.path.with_extension("correlation"))?,
+ path: self.path.with_extension("correlation"),
+ };
+ let mut file = &replacement.file;
+ let source = OpenOptions::new()
+ .read(true)
+ .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)
+ .open(&self.path)?;
+ if io::copy(&mut source.take(offset), &mut file)? != offset {
+ return Err(io::Error::other("incomplete capture replacement"));
+ }
+ file.write_all(&[7])?;
+ #[cfg(test)]
+ tests::correlation_checkpoint(&self.root, "partial");
+ file.write_all(&(bytes.len() as u64).to_le_bytes())?;
+ file.write_all(bytes)?;
+ file.write_all(&[8])?;
+ file.write_all(&0u64.to_le_bytes())?;
+ file.sync_all()?;
+ // Recheck original identity and the now-present temporary copy's total.
+ self.check_correlation_tail(offset, bytes.len(), false)?;
+ let next_file = file.try_clone()?;
+ #[cfg(test)]
+ tests::correlation_checkpoint(&self.root, "ready");
+ fs::rename(&replacement.path, &self.path)?;
+ self.file = next_file;
+ self.written = offset + 18 + bytes.len() as u64;
+ self.records += 1;
+ #[cfg(test)]
+ tests::correlation_checkpoint(&self.root, "published");
+ File::open(&self.root)?.sync_all()?;
+ Ok(())
+ }
+
+ fn check_correlation_tail(
+ &self,
+ offset: u64,
+ bytes: usize,
+ reserve_copy: bool,
+ ) -> io::Result<()> {
+ let root = fs::symlink_metadata(&self.root)?;
+ let file = self.file.metadata()?;
+ let path = fs::symlink_metadata(&self.path)?;
+ let uid = rustix::process::geteuid().as_raw();
+ let new_size = offset.saturating_add(18).saturating_add(bytes as u64);
+ let mut total = 0u64;
+ for entry in fs::read_dir(&self.root)? {
+ let metadata = fs::symlink_metadata(entry?.path())?;
+ if !metadata.is_file() || metadata.uid() != uid {
+ return Err(io::Error::other("unsafe capture directory entry"));
+ }
+ total = total.saturating_add(metadata.len());
+ }
+ if !root.is_dir()
+ || root.mode() & 0o777 != 0o700
+ || root.uid() != uid
+ || fs::canonicalize(&self.root)? != self.root
+ || !file.is_file()
+ || file.mode() & 0o777 != 0o600
+ || file.uid() != uid
+ || !path.is_file()
+ || path.ino() != file.ino()
+ || path.dev() != file.dev()
+ || file.len() != self.written
+ || self.written != offset.saturating_add(9)
+ || now() >= self.expires_at
+ || new_size > self.limit
+ || total.saturating_add(if reserve_copy { new_size } else { 0 }) > MAX_TOTAL
+ || self.records.saturating_add(1) > MAX_RECORDS
+ {
+ return Err(io::Error::other(
+ "unsafe or over-limit capture correlation tail",
+ ));
+ }
+ Ok(())
+ }
+
+ // Binary records: kind:u8, length:u64 LE, exact bytes. File order is read order;
+ // offsets are cumulative per direction. These are NOT TCP/TLS packet boundaries.
+ fn write(&mut self, kind: u8, bytes: &[u8]) -> io::Result<()> {
+ let size = 9u64.saturating_add(bytes.len() as u64);
+ if now() >= self.expires_at
+ || self.written.saturating_add(size) > self.limit
+ || self.records >= MAX_RECORDS
+ {
+ self.active = false;
+ return Err(io::Error::other("capture bound reached"));
+ }
+ self.file.write_all(&[kind])?;
+ self.file.write_all(&(bytes.len() as u64).to_le_bytes())?;
+ self.file.write_all(bytes)?;
+ self.written += size;
+ self.records += 1;
+ Ok(())
+ }
+}
+// A killed process can leave a bounded private temporary file. The existing
+// session retention owner removes it together with the rest of the session.
+struct CorrelationReplacement {
+ file: File,
+ path: PathBuf,
+}
+impl Drop for CorrelationReplacement {
+ fn drop(&mut self) {
+ let _ = fs::remove_file(&self.path);
+ }
+}
+
+impl Drop for Writer {
+ fn drop(&mut self) {
+ let _ = self.seal();
+ }
+}
+
+struct CapturedBody {
+ body: Box<dyn AttemptRequestBodyReader>,
+ capture: Capture,
+ ended: bool,
+}
+impl std::fmt::Debug for CapturedBody {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str("PrivateCapturedBody")
+ }
+}
+impl AttemptRequestBodyReader for CapturedBody {
+ fn visible_bytes(&self) -> usize {
+ self.body.visible_bytes()
+ }
+ fn max_chunk_bytes(&self) -> usize {
+ self.body.max_chunk_bytes()
+ }
+ fn high_water_bytes(&self) -> usize {
+ self.body.high_water_bytes()
+ }
+ fn next_chunk(&mut self) -> Result<Option<ChargedBytes>, AttemptError> {
+ let chunk = self.body.next_chunk()?;
+ if let Some(bytes) = &chunk {
+ self.capture.record(1, bytes.bytes());
+ } else if !self.ended {
+ self.capture.record(2, &[]);
+ self.ended = true;
+ }
+ Ok(chunk)
+ }
+ fn release(&mut self) {
+ self.body.release();
+ }
+}
diff --git a/crates/gateway/src/runtime/stream_capture/replay.rs b/crates/gateway/src/runtime/stream_capture/replay.rs
new file mode 100644
index 00000000..70325de2
--- /dev/null
+++ b/crates/gateway/src/runtime/stream_capture/replay.rs
@@ -0,0 +1,142 @@
+use super::*;
+use crate::server::core_runtime::adapters::{
+ NativeResponseDecoder, ProtocolAdapterError, ResponseDecodeStatus,
+};
+use hiroute_gateway_core::runtime::body::BudgetTree;
+
+/// Offline only. `chunk_bytes == 0` preserves captured reader blocks; other values
+/// rechunk the same response entity. Never claims to reproduce socket timing.
+pub async fn replay_capture(
+ path: &Path,
+ chunk_bytes: usize,
+) -> Result<serde_json::Value, &'static str> {
+ let bytes = read_private(path, MAX_FILE).map_err(|_| "capture unavailable or unsafe")?;
+ let records = records(&bytes)?;
+ let context: Context = serde_json::from_slice(
+ records
+ .first()
+ .filter(|r| r.0 == 0)
+ .ok_or("missing context")?
+ .1,
+ )
+ .map_err(|_| "invalid context")?;
+ if now() >= context.session.delete_after {
+ return Err("capture retention expired");
+ }
+ if records.last().map(|r| r.0) != Some(8) {
+ return Err("capture incomplete or over limit");
+ }
+ let request_bytes: usize = records.iter().filter(|r| r.0 == 1).map(|r| r.1.len()).sum();
+ if request_bytes as u64 != context.request_bytes || !records.iter().any(|r| r.0 == 2) {
+ return Err("request body capture incomplete");
+ }
+ let mut status = None;
+ for (_, bytes) in records.iter().filter(|r| r.0 == 3) {
+ let head = u16::from_le_bytes((*bytes).try_into().map_err(|_| "invalid status")?);
+ if !(100..600).contains(&head) {
+ return Err("invalid status");
+ }
+ if (100..200).contains(&head) {
+ if status.is_some() {
+ return Err("informational head after final status");
+ }
+ continue;
+ }
+ if status.replace(head).is_some() {
+ return Err("multiple final statuses");
+ }
+ }
+ let status = status.ok_or("missing final status")?;
+ // Native successful delivery uses NativeResponseProjector, while retryable
+ // HTTP failures can bypass decoding entirely. Do not substitute a decoder
+ // verdict for a production path this offline reader cannot restore.
+ if !(200..300).contains(&status)
+ || context.profile.ingress_protocol == context.profile.capability.upstream_protocol
+ {
+ return Err("unsupported_capture_path");
+ }
+ let entity: Vec<u8> = records
+ .iter()
+ .filter(|r| r.0 == 4)
+ .flat_map(|r| r.1.iter().copied())
+ .collect();
+ let chunks: Box<dyn Iterator<Item = &[u8]>> = if chunk_bytes == 0 {
+ Box::new(records.iter().filter(|r| r.0 == 4).map(|r| r.1))
+ } else {
+ Box::new(entity.chunks(chunk_bytes))
+ };
+ let tree =
+ BudgetTree::new(8 * 1024 * 1024, 8 * 1024 * 1024).map_err(|_| "budget unavailable")?;
+ let budget = tree
+ .stream(8 * 1024 * 1024)
+ .map_err(|_| "budget unavailable")?;
+ let delivery =
+ adapters::ActiveResponseDelivery::new(context.profile.ingress_protocol, budget.clone());
+ let result = adapters::with_active_response_delivery(delivery, async {
+ let mut decoder = NativeResponseDecoder::new_for_attempt(
+ &context.profile,
+ status,
+ context.streaming && (200..300).contains(&status),
+ context.chat_tools,
+ budget,
+ )?;
+ let mut event_count = 0u64;
+ for chunk in chunks {
+ drain(&mut decoder, chunk, false, &mut event_count)?;
+ }
+ drain(&mut decoder, &[], true, &mut event_count)?;
+ let position = decoder.diagnostic_position();
+ let decoded = decoder.finish()?;
+ Ok::<_, ProtocolAdapterError>((event_count, position, decoded.response.tool_id_map.len()))
+ })
+ .await;
+ let result = match result {
+ Ok((events, (frame, received), tools)) => {
+ serde_json::json!({"result":"accepted", "events":events, "frame_index":frame, "received_bytes":received, "tool_calls":tools})
+ }
+ Err(error) => {
+ let (reason, field) = crate::runtime::driver::response_diagnostics::category(&error);
+ serde_json::json!({"result":"rejected", "reason":reason, "field":field})
+ }
+ };
+ Ok(
+ serde_json::json!({"source_sha": context.session.source_sha, "request_bytes": request_bytes,
+ "response_bytes": entity.len(), "chunk_bytes": chunk_bytes, "decoder": result,
+ "captured_gateway_failure": records.iter().any(|r| r.0 == 6)}),
+ )
+}
+
+fn drain(
+ decoder: &mut NativeResponseDecoder,
+ bytes: &[u8],
+ eof: bool,
+ events: &mut u64,
+) -> Result<(), ProtocolAdapterError> {
+ let mut status = decoder.feed(bytes, eof)?;
+ loop {
+ *events += decoder.take_events().len() as u64;
+ if status != ResponseDecodeStatus::NeedDrain {
+ break;
+ }
+ status = decoder.resume()?;
+ }
+ Ok(())
+}
+
+pub(super) fn records(mut bytes: &[u8]) -> Result<Vec<(u8, &[u8])>, &'static str> {
+ let mut records = Vec::new();
+ while !bytes.is_empty() {
+ if bytes.len() < 9 || records.len() as u64 >= MAX_RECORDS {
+ return Err("invalid record boundary");
+ }
+ let kind = bytes[0];
+ let len = u64::from_le_bytes(bytes[1..9].try_into().map_err(|_| "invalid length")?);
+ bytes = &bytes[9..];
+ if kind > 8 || len > bytes.len() as u64 {
+ return Err("invalid record");
+ }
+ records.push((kind, &bytes[..len as usize]));
+ bytes = &bytes[len as usize..];
+ }
+ Ok(records)
+}
diff --git a/crates/gateway/src/runtime/stream_capture/tests.rs b/crates/gateway/src/runtime/stream_capture/tests.rs
new file mode 100644
index 00000000..c640c6b9
--- /dev/null
+++ b/crates/gateway/src/runtime/stream_capture/tests.rs
@@ -0,0 +1,482 @@
+use super::*;
+use crate::server::core_runtime::profiles::fixed_reasoning;
+use crate::server::request_plan::IngressProtocol;
+use std::os::unix::fs::PermissionsExt;
+use std::time::{Duration, Instant};
+
+fn setup() -> PathBuf {
+ let mut random = [0u8; 16];
+ getrandom::fill(&mut random).unwrap();
+ let root = fs::canonicalize(std::env::temp_dir())
+ .unwrap()
+ .join(format!("hiroute-capture-{:x}", u128::from_le_bytes(random)));
+ fs::create_dir(&root).unwrap();
+ fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap();
+ let session = Session {
+ source_sha: "a".repeat(40),
+ binary_sha256: "b".repeat(64),
+ client: "fixture".into(),
+ expires_at: now() + 60,
+ delete_after: now() + 120,
+ };
+ private_file(&root.join("session.json"))
+ .unwrap()
+ .write_all(&serde_json::to_vec(&session).unwrap())
+ .unwrap();
+ root
+}
+fn profile() -> CandidateProtocolProfile {
+ CandidateProtocolProfile::exact_portable_path(
+ IngressProtocol::Messages,
+ IngressProtocol::ChatCompletions,
+ "physical",
+ fixed_reasoning("fixed"),
+ )
+}
+
+fn promoted_correlation(index: u32) -> CaptureCorrelation {
+ CaptureCorrelation {
+ request_token: Some(hiroute_diagnostics::identity::CorrelationToken::from_bytes(
+ [1; 32],
+ )),
+ request_id: "private-request".into(),
+ attempt_index: index,
+ attempt_token: Some(hiroute_diagnostics::identity::CorrelationToken::from_bytes(
+ [index as u8; 32],
+ )),
+ }
+}
+
+pub(super) fn correlation_checkpoint(root: &Path, stage: &str) {
+ if std::env::var_os("HIROUTE_CAPTURE_INTERRUPT_ROOT").as_deref() == Some(root.as_os_str())
+ && std::env::var("HIROUTE_CAPTURE_INTERRUPT_STAGE").as_deref() == Ok(stage)
+ {
+ private_file(&root.join("checkpoint"))
+ .unwrap()
+ .write_all(stage.as_bytes())
+ .unwrap();
+ loop {
+ std::thread::park();
+ }
+ }
+}
+
+#[tokio::test]
+async fn sealed_correlation_survives_process_termination() {
+ const TEST: &str =
+ "runtime::stream_capture::tests::sealed_correlation_survives_process_termination";
+ if let Some(root) = std::env::var_os("HIROUTE_CAPTURE_INTERRUPT_ROOT") {
+ let root = PathBuf::from(root);
+ let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();
+ capture.record(1, b"{}");
+ capture.record(2, &[]);
+ capture.record(3, &200u16.to_le_bytes());
+ capture.record(4, b"data: {invalid\n\n");
+ capture.failed();
+ PendingCapture(Arc::downgrade(&capture.0)).promoted(promoted_correlation(1));
+ panic!("missing interruption checkpoint");
+ }
+ for stage in ["partial", "ready", "published"] {
+ let root = setup();
+ let mut child = std::process::Command::new(std::env::current_exe().unwrap())
+ .args(["--exact", TEST, "--nocapture"])
+ .env("HIROUTE_CAPTURE_INTERRUPT_ROOT", &root)
+ .env("HIROUTE_CAPTURE_INTERRUPT_STAGE", stage)
+ .stdout(std::process::Stdio::null())
+ .spawn()
+ .unwrap();
+ let deadline = Instant::now() + Duration::from_secs(10);
+ while !root.join("checkpoint").exists() && Instant::now() < deadline {
+ if child.try_wait().unwrap().is_some() {
+ break;
+ }
+ std::thread::sleep(Duration::from_millis(5));
+ }
+ let reached = root.join("checkpoint").exists();
+ let stopped = root.join("stopped").exists();
+ // SIGKILL cannot execute a destructor or finish an in-progress write.
+ let _ = child.kill();
+ let status = child.wait().unwrap();
+ assert!(
+ reached && stopped,
+ "{stage}: checkpoint not reached: {status}"
+ );
+ let path = root.join("attempt-1.capture");
+ let bytes = read_private(&path, MAX_FILE).unwrap();
+ let records = replay::records(&bytes).unwrap();
+ assert_eq!(records.last().unwrap().0, 8, "{stage}");
+ assert_eq!(records.iter().filter(|r| r.0 == 8).count(), 1, "{stage}");
+ assert_eq!(
+ records.iter().filter(|r| r.0 == 7).count(),
+ usize::from(stage == "published"),
+ "{stage}"
+ );
+ for chunk in [0, 1, 4096] {
+ let replay = replay_capture(&path, chunk).await.unwrap();
+ assert_eq!(replay["decoder"]["reason"], "invalid_json", "{stage}");
+ assert_eq!(replay["captured_gateway_failure"], true, "{stage}");
+ }
+ fs::remove_dir_all(root).unwrap();
+ }
+}
+
+#[test]
+fn promoted_failure_appends_only_one_bounded_correlation_without_reopening_content() {
+ let root = setup();
+ let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();
+ capture.record(1, b"{}");
+ capture.record(2, &[]);
+ capture.record(3, &200u16.to_le_bytes());
+ capture.record(4, b"original response");
+ capture.failed();
+ let path = root.join("attempt-1.capture");
+ let initial = fs::read(&path).unwrap();
+ assert!(root.join("stopped").exists());
+ assert_eq!(replay::records(&initial).unwrap().last().unwrap().0, 8);
+ PendingCapture(Arc::downgrade(&capture.0)).promoted(promoted_correlation(1));
+ let bound = fs::read(&path).unwrap();
+ assert_eq!(&initial[..initial.len() - 9], &bound[..initial.len() - 9]);
+ let records = replay::records(&bound).unwrap();
+ assert_eq!(records.iter().filter(|r| r.0 == 7).count(), 1);
+ assert_eq!(records.iter().filter(|r| r.0 == 8).count(), 1);
+ assert_eq!(records.last().unwrap().0, 8);
+ let correlation: serde_json::Value =
+ serde_json::from_slice(records.iter().find(|r| r.0 == 7).unwrap().1).unwrap();
+ assert_eq!(correlation["attempt_index"], 1);
+ assert!(!capture.0.lock().unwrap().active);
+ PendingCapture(Arc::downgrade(&capture.0)).promoted(promoted_correlation(2));
+ capture.record(4, b"must not capture a later response");
+ capture.failed();
+ assert_eq!(fs::read(&path).unwrap(), bound);
+ let weak = PendingCapture(Arc::downgrade(&capture.0));
+ drop(capture);
+ weak.promoted(promoted_correlation(3));
+ assert_eq!(fs::read(&path).unwrap(), bound);
+ fs::remove_dir_all(root).unwrap();
+}
+
+#[test]
+fn sealed_correlation_respects_original_bounds_and_file_identity() {
+ for guard in [
+ "file_limit",
+ "session_limit",
+ "temporary_copy_limit",
+ "records",
+ "expiry",
+ "permissions",
+ "identity",
+ ] {
+ let root = setup();
+ let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();
+ capture.record(1, b"{}");
+ capture.record(2, &[]);
+ capture.failed();
+ let path = root.join("attempt-1.capture");
+ let initial = fs::read(&path).unwrap();
+ match guard {
+ "file_limit" => {
+ let mut w = capture.0.lock().unwrap();
+ w.limit = w.written;
+ }
+ "session_limit" => {
+ private_file(&root.join("other.capture"))
+ .unwrap()
+ .set_len(MAX_TOTAL - initial.len() as u64)
+ .unwrap();
+ }
+ "temporary_copy_limit" => {
+ // The final delta fits, but both complete copies cannot coexist.
+ let correlation_bytes = serde_json::to_vec(&promoted_correlation(1)).unwrap().len();
+ let existing_bytes: u64 = fs::read_dir(&root)
+ .unwrap()
+ .map(|entry| entry.unwrap().metadata().unwrap().len())
+ .sum();
+ private_file(&root.join("other.capture"))
+ .unwrap()
+ .set_len(MAX_TOTAL - existing_bytes - correlation_bytes as u64 - 9)
+ .unwrap();
+ }
+ "records" => capture.0.lock().unwrap().records = MAX_RECORDS,
+ "expiry" => capture.0.lock().unwrap().expires_at = now() - 1,
+ "permissions" => fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(),
+ "identity" => {
+ fs::rename(&path, root.join("original.capture")).unwrap();
+ private_file(&path).unwrap().write_all(&initial).unwrap();
+ }
+ _ => unreachable!(),
+ }
+ PendingCapture(Arc::downgrade(&capture.0)).promoted(promoted_correlation(1));
+ assert_eq!(fs::read(&path).unwrap(), initial, "{guard}");
+ assert!(!capture.0.lock().unwrap().active, "{guard}");
+ drop(capture);
+ fs::remove_dir_all(root).unwrap();
+ }
+}
+
+// Only after the test has dropped its previous writer. A concurrent process
+// spawn can inherit the open-file description until CLOEXEC closes it. Wait
+// for that actual kernel ownership to end; all other errors remain failures.
+fn open_after_capture_release(root: &Path, mut on_blocked: impl FnMut()) -> Capture {
+ let deadline = Instant::now() + Duration::from_secs(2);
+ loop {
+ match Capture::open(root, &profile(), None, 2, true) {
+ Ok(capture) => return capture,
+ Err(error)
+ if error.kind() == io::ErrorKind::WouldBlock && Instant::now() < deadline =>
+ {
+ on_blocked();
+ std::thread::sleep(Duration::from_millis(1));
+ }
+ Err(error) => panic!("capture ownership did not release: {error}"),
+ }
+ }
+}
+
+#[test]
+fn completed_capture_waits_for_inherited_lock_descriptor_release() {
+ let root = setup();
+ let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();
+ // A descriptor duplicate models the same open-file description inherited
+ // across fork, even when the originating writer has already been dropped.
+ let inherited = capture.0.lock().unwrap()._lock.try_clone().unwrap();
+ drop(capture);
+ assert_eq!(
+ Capture::open(&root, &profile(), None, 2, true)
+ .err()
+ .unwrap()
+ .kind(),
+ io::ErrorKind::WouldBlock,
+ "live descriptor ownership must still reject another writer"
+ );
+ let (release, observed) = std::sync::mpsc::channel();
+ let holder = std::thread::spawn(move || {
+ observed.recv_timeout(Duration::from_secs(2)).unwrap();
+ std::thread::sleep(Duration::from_millis(50));
+ drop(inherited);
+ });
+ let mut blocked = 0;
+ let next = open_after_capture_release(&root, || {
+ blocked += 1;
+ if blocked == 1 {
+ release.send(()).unwrap();
+ }
+ });
+ holder.join().unwrap();
+ assert!(
+ blocked > 0,
+ "re-admission must observe the retained kernel lock"
+ );
+ drop(next);
+ fs::remove_dir_all(root).unwrap();
+}
+
+#[tokio::test]
+async fn capture_preserves_bytes_and_replays_request_bound_tools_across_chunks() {
+ let root = setup();
+ let request = serde_json::json!({"model":"alias", "max_tokens":64,"messages":[{"role":"user","content":"hello"}], "tools":[{"name":"known","description":"tool", "input_schema":{"type":"object"}}]});
+ let ir = adapters::decode_ingress_request(IngressProtocol::Messages, &request).unwrap();
+ let tools = adapters::ChatToolProjection::for_request(&ir).unwrap();
+ let body = serde_json::to_vec(&request).unwrap();
+ let capture = Capture::open(&root, &profile(), Some(&tools), body.len(), true).unwrap();
+ capture.record(1, &body[..9]);
+ capture.record(1, &body[9..]);
+ capture.record(2, &[]);
+ capture.record(3, &103u16.to_le_bytes());
+ capture.record(3, &200u16.to_le_bytes());
+ let sse = b"data: {\"id\":\"resp\",\"model\":\"physical\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call\",\"type\":\"function\",\"function\":{\"name\":\"unknown_secret_tool\",\"arguments\":\"{}\"}}]},\"finish_reason\":null}]}\n\ndata: [DONE]\n\n";
+ capture.record(4, &sse[..17]);
+ capture.record(4, &sse[17..]);
+ capture.failed();
+ // The supervisor may stop the process as soon as this marker appears.
+ // A failed sample must already be durable and sealed while other owners still exist.
+ assert!(root.join("stopped").exists());
+ assert!(root.join("active.lock").exists());
+ let at_stop = read_private(&root.join("attempt-1.capture"), MAX_FILE).unwrap();
+ assert_eq!(replay::records(&at_stop).unwrap().last().unwrap().0, 8);
+ for size in [0, 1, 4096] {
+ let replay = replay_capture(&root.join("attempt-1.capture"), size)
+ .await
+ .unwrap();
+ assert_eq!(replay["decoder"]["result"], "rejected");
+ assert_eq!(replay["decoder"]["reason"], "missing_tool_identity");
+ }
+ capture.record(4, b"cannot append after seal");
+ drop(capture);
+ let path = root.join("attempt-1.capture");
+ assert_eq!(
+ fs::metadata(&path).unwrap().permissions().mode() & 0o777,
+ 0o600
+ );
+ let bytes = read_private(&path, MAX_FILE).unwrap();
+ let records = replay::records(&bytes).unwrap();
+ let actual: Vec<u8> = records
+ .iter()
+ .filter(|r| r.0 == 1)
+ .flat_map(|r| r.1.iter().copied())
+ .collect();
+ assert_eq!(actual, body);
+ let chunks: Vec<_> = records.iter().filter(|r| r.0 == 4).map(|r| r.1).collect();
+ assert_eq!(chunks, vec![&sse[..17], &sse[17..]]);
+ for size in [0, 1, 4096] {
+ let result = replay_capture(&path, size).await.unwrap();
+ assert_eq!(result["decoder"]["result"], "rejected");
+ assert!(!result.to_string().contains("unknown_secret_tool"));
+ }
+ assert!(
+ Capture::open(&root, &profile(), None, 2, true).is_err(),
+ "first failure stops further captures"
+ );
+ fs::remove_dir_all(root).unwrap();
+}
+
+#[tokio::test]
+async fn bounded_capture_is_fail_closed_without_mutating_business_input() {
+ let root = setup();
+ let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();
+ assert!(
+ Capture::open(&root, &profile(), None, 2, true).is_err(),
+ "one writer per session"
+ );
+ {
+ let mut writer = capture.0.lock().unwrap();
+ writer.limit = writer.written + 10;
+ }
+ capture.record(1, b"{}");
+ drop(capture);
+ assert!(
+ replay_capture(&root.join("attempt-1.capture"), 0)
+ .await
+ .is_err()
+ );
+ fs::set_permissions(root.join("session.json"), fs::Permissions::from_mode(0o644)).unwrap();
+ assert!(Capture::open(&root, &profile(), None, 2, true).is_err());
+ fs::remove_dir_all(root).unwrap();
+}
+
+#[test]
+fn capture_redacts_transport_headers_and_enforces_expiry() {
+ let root = setup();
+ let mut profile = profile();
+ if let crate::server::core_runtime::profiles::CriticalFact::Exact(headers) =
+ &mut profile.connector.headers
+ {
+ headers
+ .required_headers
+ .push(("x-api-key".into(), "SECRET_CREDENTIAL".into()));
+ }
+ let capture = Capture::open(&root, &profile, None, 2, true).unwrap();
+ {
+ let mut writer = capture.0.lock().unwrap();
+ writer.expires_at = now() - 1;
+ }
+ capture.record(1, b"{}");
+ drop(capture);
+ let bytes = read_private(&root.join("attempt-1.capture"), MAX_FILE).unwrap();
+ assert!(!String::from_utf8_lossy(&bytes).contains("SECRET_CREDENTIAL"));
+ assert_ne!(replay::records(&bytes).unwrap().last().unwrap().0, 8);
+ fs::remove_dir_all(root).unwrap();
+}
+
+#[tokio::test]
+async fn capture_limits_and_partial_request_cannot_produce_replay_evidence() {
+ let root = setup();
+ assert!(Capture::open(&root, &profile(), None, MAX_FILE as usize, true).is_err());
+ assert!(root.join("active.lock").exists());
+ let capture = open_after_capture_release(&root, || {});
+ capture.record(1, b"{");
+ capture.record(3, &200u16.to_le_bytes());
+ capture.failed();
+ drop(capture);
+ assert_eq!(
+ replay_capture(&root.join("attempt-1.capture"), 0)
+ .await
+ .unwrap_err(),
+ "request body capture incomplete"
+ );
+ fs::remove_dir_all(root).unwrap();
+
+ let root = setup();
+ for _ in 0..MAX_ATTEMPTS {
+ drop(open_after_capture_release(&root, || {}));
+ }
+ assert!(Capture::open(&root, &profile(), None, 2, true).is_err());
+ assert!(root.join("active.lock").exists());
+ fs::remove_dir_all(root).unwrap();
+
+ let root = setup();
+ private_file(&root.join("existing.capture"))
+ .unwrap()
+ .set_len(MAX_TOTAL)
+ .unwrap();
+ assert!(Capture::open(&root, &profile(), None, 2, true).is_err());
+ assert!(root.join("active.lock").exists());
+ fs::remove_dir_all(root).unwrap();
+
+ let root = setup();
+ let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();
+ capture.0.lock().unwrap().records = MAX_RECORDS;
+ capture.record(1, b"{}");
+ drop(capture);
+ assert!(
+ replay_capture(&root.join("attempt-1.capture"), 0)
+ .await
+ .is_err()
+ );
+ assert!(replay::records(&[4, 2, 0, 0, 0, 0, 0, 0, 0, 1]).is_err());
+ fs::remove_dir_all(root).unwrap();
+}
+
+#[tokio::test]
+async fn replay_uses_final_head_and_rejects_paths_without_decoder_equivalence() {
+ let body = b"{}";
+ let response = b"data: {\"id\":\"resp\",\"model\":\"physical\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"resp\",\"model\":\"physical\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
+ let mut mismatches = Vec::new();
+ for native in [false, true] {
+ for (heads, invalid) in [
+ (&[200u16][..], None),
+ (&[103u16, 200][..], None),
+ (&[103, 200, 200][..], Some("multiple final statuses")),
+ (&[429][..], Some("unsupported_capture_path")),
+ (&[103, 429][..], Some("unsupported_capture_path")),
+ (&[503][..], Some("unsupported_capture_path")),
+ (&[103, 503][..], Some("unsupported_capture_path")),
+ (&[103][..], Some("missing final status")),
+ (
+ &[200, 103][..],
+ Some("informational head after final status"),
+ ),
+ ] {
+ let root = setup();
+ let mut profile = profile();
+ if native {
+ profile.ingress_protocol = profile.capability.upstream_protocol;
+ }
+ let capture = Capture::open(&root, &profile, None, body.len(), true).unwrap();
+ capture.record(1, body);
+ capture.record(2, &[]);
+ for head in heads {
+ capture.record(3, &head.to_le_bytes());
+ }
+ capture.record(4, &response[..23]);
+ capture.record(4, &response[23..]);
+ capture.record(5, &[]);
+ drop(capture);
+ for size in [0, 1, 4096] {
+ let result = replay_capture(&root.join("attempt-1.capture"), size).await;
+ let expected = invalid.or(native.then_some("unsupported_capture_path"));
+ let matched = match (&result, expected) {
+ (Err(actual), Some(expected)) => *actual == expected,
+ (Ok(actual), None) => actual["decoder"]["result"] == "accepted",
+ _ => false,
+ };
+ if !matched {
+ mismatches.push(format!(
+ "heads={heads:?} native={native} chunk={size} expected={expected:?} actual={result:?}"
+ ));
+ }
+ }
+ fs::remove_dir_all(root).unwrap();
+ }
+ }
+ assert!(mismatches.is_empty(), "{}", mismatches.join("\n"));
+}
diff --git a/scripts/ci-shards.py b/scripts/ci-shards.py
index 61a08410..ffd1d5b4 100755
--- a/scripts/ci-shards.py
+++ b/scripts/ci-shards.py
@@ -27,6 +27,7 @@ INTEGRATION_SHARDS = {
("hiroute-domain", "compute"),
("hiroute-client-core", "transport"),
("hiroute-diagnostics", "files_safety"),
+ ("hiroute-diagnostics", "response_failure_compatibility"),
("hiroute-diagnostics", "worker_probe_progress"),
("hiroute-diagnostics", "writer_bounds"),
("hiroute-cli", "contracts"),
diff --git a/scripts/private-stream-capture.py b/scripts/private-stream-capture.py
new file mode 100644
index 00000000..ade0e51a
--- /dev/null
+++ b/scripts/private-stream-capture.py
@@ -0,0 +1,252 @@
+#!/usr/bin/env python3
+"""Run one isolated Unix Debug gateway/daemon with bounded private capture.
+
+The executable must come from the supplied committed candidate's managed build.
+This tool supplies provenance, NOT a build attestation. Never use a daily daemon.
+No network requests or model calls are issued by this tool.
+"""
+import argparse
+import fcntl
+import hashlib
+import json
+import os
+from pathlib import Path
+import shutil
+import signal
+import stat
+import subprocess
+import time
+
+GROUP_STOP_GRACE_SECONDS = 5
+
+
+def require_supervision_primitives():
+ if not all(hasattr(os, name) for name in ("waitid", "P_PID", "WEXITED", "WNOHANG", "WNOWAIT")):
+ raise ValueError("capture requires non-reaping Unix waitid (macOS: Python 3.13 or newer)")
+
+
+def candidate_exited(child):
+ # Keep the exited leader waitable so its PID/session cannot be recycled
+ # before stop has signalled every member of our process group.
+ return os.waitid(os.P_PID, child.pid, os.WEXITED | os.WNOHANG | os.WNOWAIT) is not None
+
+
+def write_private(path, data):
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+ with os.fdopen(fd, "wb") as stream:
+ stream.write(data)
+
+
+def create_session(root, source_sha, binary, client, seconds):
+ root.mkdir(mode=0o700)
+ stamp = int(time.time())
+ session = dict(source_sha=source_sha, binary_sha256=hashlib.sha256(binary.read_bytes()).hexdigest(),
+ client=client, expires_at=stamp + seconds, delete_after=stamp + 86400)
+ write_private(root / "session.json", json.dumps(session).encode())
+ return session
+
+
+def stop(child):
+ if child.returncode is not None:
+ raise ValueError("candidate leader was reaped before process-group cleanup")
+ if os.getpgid(child.pid) != child.pid or os.getsid(child.pid) != child.pid:
+ raise ValueError("candidate does not own its isolated session/process group")
+ # The waitable leader anchors this group ID even when it has already
+ # exited. A descendant may still own capture files or ignore SIGTERM.
+ try:
+ os.killpg(child.pid, signal.SIGTERM)
+ deadline = time.monotonic() + GROUP_STOP_GRACE_SECONDS
+ while time.monotonic() < deadline:
+ time.sleep(min(0.02, max(0, deadline - time.monotonic())))
+ finally:
+ # Graceful termination of the retention owner can interrupt the grace
+ # wait. It must still stop descendants before attempting raw cleanup.
+ try:
+ os.killpg(child.pid, signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ finally:
+ child.wait(timeout=5)
+
+
+def supervise(args):
+ require_supervision_primitives()
+ if not 1 <= args.seconds <= 3600:
+ raise ValueError("duration must be 1..3600 seconds")
+ if not 1 <= args.attempts <= 3:
+ raise ValueError("attempt budget must be 1..3")
+ if not args.command:
+ raise ValueError("supply the isolated candidate executable and its arguments")
+ command = args.command[1:] if args.command[0] == "--" else args.command
+ binary = Path(command[0]).resolve(strict=True)
+ source = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
+ if source != args.source_sha or subprocess.check_output(["git", "status", "--porcelain"]):
+ raise ValueError("run from the clean, exact committed source checkout")
+ root = args.root.resolve()
+ if root.exists():
+ raise ValueError("capture root must be new")
+ session = create_session(root, source, binary, args.client, args.seconds)
+ retain_session(args, root, binary, command, source, session["delete_after"])
+
+
+def retain_session(args, root, binary, command, source, delete_after):
+ require_supervision_primitives()
+ # This bounded owner survives the CLI returning. It owns the candidate child,
+ # stops/reaps it, then removes the raw session at expiry (or on graceful stop).
+ # No shared daemon, system scheduler, or daily process is modified.
+ read_fd, write_fd = os.pipe()
+ owner = os.fork()
+ if owner == 0:
+ os.close(read_fd)
+ os.setsid()
+ with open(os.devnull, "wb") as null:
+ os.dup2(null.fileno(), 1)
+ os.dup2(null.fileno(), 2)
+ def terminate(_signum, _frame):
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ raise SystemExit(1)
+ signal.signal(signal.SIGTERM, terminate)
+ signal.signal(signal.SIGINT, terminate)
+ reports = os.fdopen(write_fd, "w")
+ def report(value):
+ value["retention_owner_pid"] = os.getpid()
+ print(json.dumps(value), file=reports, flush=True)
+ exit_code = 0
+ # A backwards wall-clock adjustment must not extend retention.
+ deadline = time.monotonic() + max(0, min(86400, delete_after - time.time()))
+ try:
+ supervise_candidate(args, root, binary, command, source, report)
+ reports.close() # The CLI can return; this owner remains until deletion.
+ while time.time() < delete_after and time.monotonic() < deadline:
+ time.sleep(min(60, max(0, deadline - time.monotonic()),
+ max(0, delete_after - time.time())))
+ except BaseException:
+ exit_code = 1
+ finally:
+ # Reporting is best effort once the CLI reader has gone. A failed
+ # flush can fail again on close, but must not skip raw deletion.
+ # Graceful stop has already reached candidate group cleanup; do not
+ # let a second signal interrupt this mandatory cleanup section.
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ try:
+ reports.close()
+ except BaseException:
+ exit_code = 1
+ finally:
+ try:
+ # SIGKILL is asynchronous: after reaping the leader, a killed
+ # descendant may still be releasing its kernel flock. This
+ # bounded wait applies only after our group-stop path.
+ cleanup(root, require_expired=False, lock_wait_seconds=5)
+ except BaseException:
+ exit_code = 1
+ finally:
+ os._exit(exit_code)
+ os.close(write_fd)
+ try:
+ with os.fdopen(read_fd) as reports:
+ for line in reports:
+ print(line.rstrip(), flush=True)
+ except BaseException:
+ os.kill(owner, signal.SIGTERM)
+ os.waitpid(owner, 0)
+ raise
+ return owner
+
+
+def supervise_candidate(args, root, binary, command, source, report):
+ env = dict(os.environ, HIROUTE_PRIVATE_STREAM_CAPTURE=str(root))
+ # Third-party stdout is not evidence; discard it instead of an unbounded log.
+ child = None
+ reason = "process_exit"
+ try:
+ with open(os.devnull, "wb") as log:
+ child = subprocess.Popen([str(binary), *command[1:]], env=env, stdout=log,
+ stderr=subprocess.STDOUT, start_new_session=True)
+ deadline = time.monotonic() + args.seconds
+ report({"capture_root": str(root), "source_sha": source,
+ "pid": child.pid, "model_request_budget": args.attempts})
+ while not candidate_exited(child):
+ files = list(root.glob("*.capture"))
+ if (root / "stopped").exists():
+ reason = "first_gateway_failure"
+ break
+ if len(files) >= args.attempts and not capture_active(root):
+ reason = "attempt_budget"
+ break
+ if time.monotonic() >= deadline:
+ reason = "time_budget"
+ break
+ # Bound all files including private child logs, beyond writer's own budget.
+ if sum(p.stat().st_size for p in root.iterdir() if p.is_file()) >= 32 * 1024 * 1024:
+ reason = "byte_budget"
+ break
+ time.sleep(0.02)
+ finally:
+ if child is not None:
+ stop(child)
+ report({"stop_reason": reason, "process_exit": child.returncode if child else None,
+ "scenario": "unassessed", "retention_seconds": 86400})
+
+
+def capture_lock(root):
+ fd = os.open(root / "active.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
+ info = os.fstat(fd)
+ if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 or info.st_uid != os.geteuid():
+ os.close(fd)
+ raise ValueError("unsafe capture lock")
+ return os.fdopen(fd, "rb+")
+
+
+def capture_active(root):
+ with capture_lock(root) as lock:
+ try:
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except BlockingIOError:
+ return True
+ return False
+
+
+def cleanup(root, require_expired=True, lock_wait_seconds=0):
+ # Only this exact, expired, owner-only session. Never walk a general evidence tree.
+ if root.is_symlink() or root.resolve() != root or root.stat().st_uid != os.geteuid() or stat.S_IMODE(root.stat().st_mode) != 0o700:
+ raise ValueError("unsafe root")
+ with capture_lock(root) as lock:
+ deadline = time.monotonic() + max(0, lock_wait_seconds)
+ while True:
+ try:
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ break
+ except BlockingIOError as error:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise ValueError("capture still owned; stop its process first") from error
+ time.sleep(min(0.02, remaining))
+ session = json.loads((root / "session.json").read_text())
+ if require_expired and int(time.time()) < session["delete_after"]:
+ raise ValueError("session has not reached retention expiry")
+ if any(p.is_symlink() or not p.is_file() for p in root.iterdir()):
+ raise ValueError("unexpected session entry")
+ shutil.rmtree(root)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ sub = parser.add_subparsers(dest="action", required=True)
+ run = sub.add_parser("run")
+ run.add_argument("--root", type=Path, required=True)
+ run.add_argument("--source-sha", required=True)
+ run.add_argument("--client", required=True)
+ run.add_argument("--seconds", type=int, default=300)
+ run.add_argument("--attempts", type=int, default=3)
+ run.add_argument("command", nargs=argparse.REMAINDER)
+ gc = sub.add_parser("cleanup")
+ gc.add_argument("root", type=Path)
+ args = parser.parse_args()
+ supervise(args) if args.action == "run" else cleanup(args.root)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/test-private-stream-capture.py b/scripts/test-private-stream-capture.py
new file mode 100644
index 00000000..e2a82a04
--- /dev/null
+++ b/scripts/test-private-stream-capture.py
@@ -0,0 +1,336 @@
+#!/usr/bin/env python3
+import ctypes
+import fcntl
+import importlib.util
+import json
+import os
+from pathlib import Path
+import select
+import signal
+import stat
+import subprocess
+import sys
+import time
+from types import SimpleNamespace
+import tempfile
+import threading
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("capture", Path(__file__).with_name("private-stream-capture.py"))
+capture = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(capture)
+
+
+class CaptureSessionTest(unittest.TestCase):
+ def test_private_provenance_and_expired_session_cleanup(self):
+ with tempfile.TemporaryDirectory() as parent:
+ binary = Path(parent) / "binary"
+ binary.write_bytes(b"fixture binary")
+ root = Path(parent).resolve() / "private"
+ with patch.object(capture.time, "time", return_value=1000):
+ capture.create_session(root, "a" * 40, binary, "fixture", 60)
+ self.assertEqual(stat.S_IMODE(root.stat().st_mode), 0o700)
+ self.assertEqual(stat.S_IMODE((root / "session.json").stat().st_mode), 0o600)
+ session = json.loads((root / "session.json").read_text())
+ self.assertEqual(session["expires_at"], 1060)
+ self.assertEqual(len(session["binary_sha256"]), 64)
+ with patch.object(capture.time, "time", return_value=2000):
+ with self.assertRaises(ValueError):
+ capture.cleanup(root)
+ with capture.capture_lock(root) as lock:
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ with patch.object(capture.time, "time", return_value=100000):
+ with self.assertRaises(ValueError):
+ capture.cleanup(root)
+ # The stable inode remains; only live kernel ownership blocks cleanup.
+ with patch.object(capture.time, "time", return_value=100000):
+ capture.cleanup(root)
+ self.assertFalse(root.exists())
+
+ def test_abnormal_and_forced_exit_release_lock_for_expired_cleanup(self):
+ for forced in [False, True]:
+ with self.subTest(forced=forced), tempfile.TemporaryDirectory() as parent:
+ root = Path(parent).resolve() / "private"
+ binary = Path(sys.executable).resolve()
+ capture.create_session(root, "a" * 40, binary, "fixture", 1)
+ code = ("import os,fcntl,time,sys; from pathlib import Path; "
+ "p=Path(os.environ['HIROUTE_PRIVATE_STREAM_CAPTURE']); "
+ "f=os.open(p/'active.lock',os.O_CREAT|os.O_RDWR,0o600); "
+ "fcntl.flock(f,fcntl.LOCK_EX); (p/'raw.capture').write_bytes(b'raw'); "
+ "print('ready',flush=True); " +
+ ("time.sleep(60)" if forced else "os._exit(3)"))
+ with subprocess.Popen([str(binary), "-c", code], stdout=subprocess.PIPE,
+ env=dict(os.environ, HIROUTE_PRIVATE_STREAM_CAPTURE=str(root)),
+ text=True) as child:
+ self.assertEqual(child.stdout.readline().strip(), "ready")
+ if forced:
+ child.kill()
+ child.wait(timeout=5)
+ self.assertFalse(capture.capture_active(root))
+ with patch.object(capture.time, "time", return_value=time.time() + 86401):
+ capture.cleanup(root)
+ self.assertFalse(root.exists())
+
+ def test_owner_cleanup_waits_for_delayed_kernel_lock_release(self):
+ with tempfile.TemporaryDirectory() as parent:
+ root = Path(parent).resolve() / "private"
+ capture.create_session(root, "a" * 40, Path(sys.executable).resolve(), "fixture", 1)
+ with capture.capture_lock(root) as lock:
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ timer = threading.Timer(0.1, lambda: fcntl.flock(lock, fcntl.LOCK_UN))
+ timer.start()
+ try:
+ capture.cleanup(root, require_expired=False, lock_wait_seconds=1)
+ finally:
+ timer.join()
+ self.assertFalse(root.exists(), "cleanup must wait for the stopped group's lock release")
+
+ def test_owner_cleanup_wait_is_bounded_and_keeps_active_capture(self):
+ with tempfile.TemporaryDirectory() as parent:
+ root = Path(parent).resolve() / "private"
+ capture.create_session(root, "a" * 40, Path(sys.executable).resolve(), "fixture", 1)
+ with capture.capture_lock(root) as lock:
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ started = time.monotonic()
+ with self.assertRaisesRegex(ValueError, "capture still owned"):
+ capture.cleanup(root, require_expired=False, lock_wait_seconds=0.1)
+ self.assertGreaterEqual(time.monotonic() - started, 0.1)
+ self.assertLess(time.monotonic() - started, 2)
+ self.assertTrue(root.exists())
+ self.assertTrue((root / "session.json").is_file())
+
+ def test_retention_owner_deletes_after_cli_returns_or_graceful_stop(self):
+ for early_stop in [False, True]:
+ with self.subTest(early_stop=early_stop), tempfile.TemporaryDirectory() as parent:
+ root = Path(parent).resolve() / "private"
+ binary = Path(sys.executable).resolve()
+ capture.create_session(root, "a" * 40, binary, "fixture", 1)
+ code = ("import os,fcntl; from pathlib import Path; "
+ "p=Path(os.environ['HIROUTE_PRIVATE_STREAM_CAPTURE']); "
+ "f=os.open(p/'active.lock',os.O_CREAT|os.O_RDWR,0o600); "
+ "fcntl.flock(f,fcntl.LOCK_EX); (p/'raw.capture').write_bytes(b'raw'); "
+ "os._exit(3)")
+ args = SimpleNamespace(seconds=1, attempts=1)
+ with patch.object(capture, "GROUP_STOP_GRACE_SECONDS", 0.1, create=True):
+ owner = capture.retain_session(args, root, binary, [str(binary), "-c", code],
+ "a" * 40, time.time() + (60 if early_stop else 1))
+ try:
+ self.assertTrue((root / "raw.capture").exists())
+ if early_stop:
+ os.kill(owner, signal.SIGTERM)
+ deadline = time.monotonic() + 5
+ while root.exists() and time.monotonic() < deadline:
+ time.sleep(0.02)
+ self.assertFalse(root.exists(), "raw retention must be executed without another CLI call")
+ finally:
+ if root.exists():
+ os.kill(owner, signal.SIGTERM)
+ os.waitpid(owner, 0)
+
+ @unittest.skipUnless(sys.platform.startswith("linux"), "controlled Linux child subreaper")
+ def test_expired_retention_stops_descendant_after_leader_exits(self):
+ self.check_descendant_cleanup(leader_exits=True)
+
+ @unittest.skipUnless(sys.platform.startswith("linux"), "controlled Linux child subreaper")
+ def test_expired_retention_stops_descendant_after_leader_exits_on_term(self):
+ self.check_descendant_cleanup(leader_exits=False)
+
+ @unittest.skipUnless(sys.platform.startswith("linux"), "controlled Linux child subreaper")
+ def test_owner_term_during_group_stop_still_kills_descendant(self):
+ self.check_descendant_cleanup(leader_exits=False, owner_term_during_stop=True)
+
+ @unittest.skipUnless(sys.platform.startswith("linux"), "controlled Linux child subreaper")
+ def test_reader_exit_cannot_skip_owner_cleanup_after_broken_report_and_close(self):
+ # Terminate the real CLI reader after the first report. Its owner later
+ # reports a natural candidate exit to the now-closed pipe, then closes
+ # that same buffered stream. Both failures must still reach deletion.
+ libc = ctypes.CDLL(None, use_errno=True)
+ previous = ctypes.c_int()
+ self.assertEqual(libc.prctl(37, ctypes.byref(previous), 0, 0, 0), 0)
+ self.assertEqual(libc.prctl(36, 1, 0, 0, 0), 0)
+ cli = owner = candidate = None
+ try:
+ with tempfile.TemporaryDirectory() as parent:
+ root = Path(parent).resolve() / "private"
+ binary = Path(sys.executable).resolve()
+ capture.create_session(root, "a" * 40, binary, "fixture", 10)
+ delete_after = int(time.time()) + 2
+ session_path = root / "session.json"
+ session = json.loads(session_path.read_text())
+ session["delete_after"] = delete_after
+ session_path.write_text(json.dumps(session))
+ candidate_code = '''import fcntl, os, time
+from pathlib import Path
+root = Path(os.environ["HIROUTE_PRIVATE_STREAM_CAPTURE"])
+fd = os.open(root / "active.lock", os.O_CREAT | os.O_RDWR, 0o600)
+fcntl.flock(fd, fcntl.LOCK_EX)
+raw = os.open(root / "raw.capture", os.O_CREAT | os.O_WRONLY, 0o600)
+os.write(raw, b"synthetic private body")
+os.close(raw)
+(root.parent / "ready").touch()
+while not (root.parent / "finish-candidate").exists():
+ time.sleep(0.01)
+os._exit(0)
+'''
+ cli_code = '''import importlib.util, sys
+from pathlib import Path
+from types import SimpleNamespace
+spec = importlib.util.spec_from_file_location("capture", sys.argv[1])
+capture = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(capture)
+capture.GROUP_STOP_GRACE_SECONDS = 0.1
+binary = Path(sys.executable).resolve()
+capture.retain_session(SimpleNamespace(seconds=10, attempts=1), Path(sys.argv[2]),
+ binary, [str(binary), "-c", sys.argv[4]], "a" * 40, float(sys.argv[3]))
+'''
+ cli = subprocess.Popen(
+ [str(binary), "-c", cli_code, capture.__file__, str(root),
+ str(delete_after), candidate_code], stdout=subprocess.PIPE, text=True,
+ )
+ self.assertTrue(select.select([cli.stdout], [], [], 5)[0], "CLI did not report its owner")
+ first = json.loads(cli.stdout.readline())
+ owner, candidate = first["retention_owner_pid"], first["pid"]
+ deadline = time.monotonic() + 5
+ while not (root.parent / "ready").exists() and time.monotonic() < deadline:
+ time.sleep(0.01)
+ self.assertTrue((root / "raw.capture").is_file())
+ cli.terminate()
+ self.assertEqual(cli.wait(timeout=5), -signal.SIGTERM)
+ cli.stdout.close()
+ cli = None
+ (root.parent / "finish-candidate").touch()
+ deadline = time.monotonic() + 5
+ while root.exists() and time.monotonic() < deadline:
+ time.sleep(0.02)
+ self.assertFalse(root.exists(), "closed CLI report pipe skipped mandatory raw cleanup")
+ status = self.wait_for_owned_child(owner)
+ owner = None
+ self.assertTrue(os.WIFEXITED(status))
+ self.assertEqual(os.WEXITSTATUS(status), 1, "report failure must remain a failed owner exit")
+ finally:
+ if cli is not None:
+ if cli.poll() is None:
+ cli.kill()
+ cli.wait(timeout=5)
+ cli.stdout.close()
+ # waitpid proves these are our adopted children before signaling;
+ # no process scan or potentially recycled external PID is used.
+ for pid in [owner, candidate]:
+ if pid is not None:
+ try:
+ completed, _ = os.waitpid(pid, os.WNOHANG)
+ if completed == 0:
+ os.kill(pid, signal.SIGKILL)
+ os.waitpid(pid, 0)
+ except ChildProcessError:
+ pass
+ self.assertEqual(libc.prctl(36, previous.value, 0, 0, 0), 0)
+
+ def check_descendant_cleanup(self, leader_exits, owner_term_during_stop=False):
+ # Adopt only this test's orphan, so its SIGKILL status and reaping can be
+ # asserted without leaving a zombie behind or trusting a recycled PID.
+ libc = ctypes.CDLL(None, use_errno=True)
+ previous = ctypes.c_int()
+ self.assertEqual(libc.prctl(37, ctypes.byref(previous), 0, 0, 0), 0)
+ self.assertEqual(libc.prctl(36, 1, 0, 0, 0), 0)
+ owner = descendant = None
+ try:
+ with tempfile.TemporaryDirectory() as parent:
+ root = Path(parent).resolve() / "private"
+ binary = Path(sys.executable).resolve()
+ capture.create_session(root, "a" * 40, binary, "fixture", 1)
+ delete_after = int(time.time()) + 2
+ session_path = root / "session.json"
+ session = json.loads(session_path.read_text())
+ session["delete_after"] = delete_after
+ session_path.write_text(json.dumps(session))
+ code = '''import fcntl, os, signal, sys, time
+from pathlib import Path
+root = Path(os.environ["HIROUTE_PRIVATE_STREAM_CAPTURE"])
+ready = root.parent / "descendant.pid"
+if os.fork() == 0:
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ fd = os.open(root / "active.lock", os.O_CREAT | os.O_RDWR, 0o600)
+ fcntl.flock(fd, fcntl.LOCK_EX)
+ (root / "raw.capture").write_bytes(b"synthetic private body")
+ ready.write_text(str(os.getpid()))
+ while True:
+ time.sleep(1)
+while not ready.exists():
+ time.sleep(0.01)
+if sys.argv[1] == "exit":
+ os._exit(0)
+def terminate(*_):
+ if sys.argv[1] == "owner_term":
+ os.kill(os.getppid(), signal.SIGTERM)
+ os._exit(0)
+signal.signal(signal.SIGTERM, terminate)
+while True:
+ time.sleep(1)
+'''
+ args = SimpleNamespace(seconds=1, attempts=1)
+ mode = "owner_term" if owner_term_during_stop else "exit" if leader_exits else "term"
+ with patch.object(capture, "GROUP_STOP_GRACE_SECONDS", 0.1, create=True):
+ owner = capture.retain_session(
+ args, root, binary,
+ [str(binary), "-c", code, mode],
+ "a" * 40, delete_after,
+ )
+ descendant = int((root.parent / "descendant.pid").read_text())
+ deadline = time.monotonic() + 5
+ while root.exists() and time.monotonic() < deadline:
+ time.sleep(0.02)
+ self.assertFalse(root.exists(), "raw retention survives the candidate's exited leader")
+ status = self.wait_for_owned_child(descendant)
+ descendant = None
+ self.assertTrue(os.WIFSIGNALED(status))
+ self.assertEqual(os.WTERMSIG(status), signal.SIGKILL)
+ status = self.wait_for_owned_child(owner)
+ owner = None
+ self.assertTrue(os.WIFEXITED(status))
+ self.assertEqual(os.WEXITSTATUS(status), 1 if owner_term_during_stop else 0)
+ finally:
+ for pid in [descendant, owner]:
+ if pid is not None:
+ try:
+ completed, _ = os.waitpid(pid, os.WNOHANG)
+ if completed == 0:
+ os.kill(pid, signal.SIGKILL)
+ os.waitpid(pid, 0)
+ except ChildProcessError:
+ pass
+ self.assertEqual(libc.prctl(36, previous.value, 0, 0, 0), 0)
+
+ def wait_for_owned_child(self, pid):
+ deadline = time.monotonic() + 5
+ while time.monotonic() < deadline:
+ completed, status = os.waitpid(pid, os.WNOHANG)
+ if completed:
+ return status
+ time.sleep(0.02)
+ self.fail("owned child did not terminate")
+
+ def test_stop_rejects_a_child_in_the_callers_process_group(self):
+ child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"])
+ try:
+ with self.assertRaisesRegex(ValueError, "isolated session/process group"):
+ capture.stop(child)
+ self.assertIsNone(child.poll())
+ finally:
+ child.kill()
+ child.wait(timeout=5)
+
+ def test_private_writer_does_not_follow_or_overwrite(self):
+ with tempfile.TemporaryDirectory() as parent:
+ target = Path(parent) / "target"
+ target.write_bytes(b"untouched")
+ link = Path(parent) / "link"
+ link.symlink_to(target)
+ with self.assertRaises(FileExistsError):
+ capture.write_private(link, b"overwrite")
+ self.assertEqual(target.read_bytes(), b"untouched")
+
+
+if __name__ == "__main__":
+ unittest.main()