Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.
Rust ai-agentsai-gatewayclaude-codecodexcoding-agentcost-optimizationdeepseekllm-gatewayllm-routerlocal-firstmodel-routingrusttask-routingtauri
fix(gateway): add precise response diagnostics and bounded replay (#62)
澄潭122c9ca4e6ca
diff --git a/contracts/compatibility-support.v1.json b/contracts/compatibility-support.v1.jsonindex a104b89d..4bd1ef2f 100644--- a/contracts/compatibility-support.v1.json+++ b/contracts/compatibility-support.v1.json@@ -1259,6 +1259,17 @@"crates/diagnostics/src/runtime.rs"],"field": "optional model, reasoning, HTTP protocol, provider error and model terminal result, target and commit facts in retained V1 logs"+ },+ {+ "contract": "hiroute.diagnostic-event/v1",+ "field": "ResponseFailure payload before detailed response reason, field and decoder position extension",+ "mode": "frozen_fixture",+ "owner": "diagnostics",+ "reason": "The current diagnostic reader accepts supported v0.2.0 ResponseFailure bytes from released producer d625e653048b2306c88da07826d52027e9891675, defaulting absent optional attempt correlation, field and position. New writers emit the current closed event contract; old readers are not promised forward compatibility.",+ "removal_condition": "Retire only when v0.2.0 diagnostic records are outside the supported source and retention policy.",+ "allowed_paths": [+ "crates/diagnostics/tests/response_failure_compatibility.rs"+ ]}]}diff --git a/crates/diagnostics/src/event/model.rs b/crates/diagnostics/src/event/model.rsindex d7662284..e58e657f 100644--- a/crates/diagnostics/src/event/model.rs+++ b/crates/diagnostics/src/event/model.rs@@ -113,6 +113,42 @@ pub struct ResponseFailure {pub attempt_index: u64,pub stage: ResponseFailureStage,pub reason: ResponseFailureReason,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub attempt_token: Option<CorrelationToken>,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub field: Option<ResponseFailureField>,+ /// One-based SSE event visited, including the event rejected by the decoder.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub frame_index: Option<u64>,+ /// Bytes supplied to the decoder, an upper bound rather than a token offset.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub received_bytes: Option<u64>,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum ResponseFailureField {+ Object,+ Choices,+ Delta,+ Content,+ ToolCalls,+ ToolId,+ ToolName,+ ToolArguments,+ FinishReason,+ Usage,+ SystemFingerprint,+ ServiceTier,+ Logprobs,+ Model,+ Id,+ Type,+ Index,+ Role,+ Output,+ Status,+ Unknown,}#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]@@ -140,6 +176,16 @@ pub enum ResponseFailureReason {NoSemanticOutput,ProviderRejected,OtherProtocol,+ UnsupportedField,+ UnsupportedValue,+ MissingToolIdentity,+ InvalidToolArguments,+ ProviderStateNotPortable,+ ToolIdBindingRequired,+ ToolContinuationConflict,+ Capability,+ ContextProjection,+ Serialization,}#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]diff --git a/crates/diagnostics/tests/response_failure_compatibility.rs b/crates/diagnostics/tests/response_failure_compatibility.rsnew file mode 100644index 00000000..4951c488--- /dev/null+++ b/crates/diagnostics/tests/response_failure_compatibility.rs@@ -0,0 +1,19 @@+use hiroute_diagnostics::event::{ResponseFailure, ResponseFailureReason, ResponseFailureStage};++#[test]+fn supported_release_response_failure_retains_meaning_and_defaults_extensions() {+ // Frozen from the serializer at supported release d625e653048b2306c88da07826d52027e9891675:+ // crates/diagnostics/src/event/model.rs::ResponseFailure. Registered in compatibility support.+ let released =+ r#"{"request_token":null,"attempt_index":3,"stage":"decode","reason":"other_protocol"}"#;+ let current: ResponseFailure = serde_json::from_str(released).unwrap();+ assert_eq!(current.attempt_index, 3);+ assert_eq!(current.stage, ResponseFailureStage::Decode);+ assert_eq!(current.reason, ResponseFailureReason::OtherProtocol);+ assert!(current.request_token.is_none());+ assert!(current.attempt_token.is_none());+ assert!(current.field.is_none());+ assert!(current.frame_index.is_none());+ assert!(current.received_bytes.is_none());+ assert_eq!(serde_json::to_string(¤t).unwrap(), released);+}diff --git a/crates/gateway/README.md b/crates/gateway/README.mdindex 3c282ccc..7b62f883 100644--- a/crates/gateway/README.md+++ b/crates/gateway/README.md@@ -85,6 +85,7 @@ provider references, observation and the future tool-selection boundary.| Decision diagnostics use the production transport/parser without running a business model | `classifier_diagnostic_uses_the_production_transport_and_exact_protocol` in [classification.rs](src/core_runtime/classification.rs) || Accepted execution history survives client rewrite/compaction with observation disabled | [accepted_history.rs](../../tools/e2e-harness/tests/p0_gateway_runtime/accepted_history.rs) || Tool continuation survives restart with authentication | [continuation.rs](../../tools/e2e-harness/tests/p0_gateway_protocol/continuation.rs) |+| Claude direct tool callers preserve native history and portable call/result pairing | `messages_direct_tool_caller_preserves_native_history_and_portable_tool_pairing` in [adapter tests](src/adapters/tests.rs); ingress rejects programmatic execution authority || Live publication cutover pins each request and preserves the last good version | [p0_gateway_request_authority.rs](../../tools/e2e-harness/tests/p0_gateway_request_authority.rs) |Use the repository [test planner](../../scripts/test-plan.py) and@@ -107,6 +108,60 @@ request and Agent turn observations even when its native bytes were delivered.The decision diagnostic uses closed error codes; provider text is not a publicerror message or a substitute for evidence of credential failure.+## Response failure evidence++`runtime/driver/response_diagnostics.rs` owns the closed, payload-free failure+classification; `adapters/response/decoder.rs` reports SSE event ordinal and+received-byte upper bound without per-token logs. Unix Debug-only+`runtime/stream_capture.rs` can capture an explicitly enabled private session;+its `stream_replay` example restores the actual attempt profile and tool mapping.+This is offline decoder evidence, not proof of network timing or task completion.+After successful provider materialization, private capture registers its existing+weak handle once a pending attempt is staged, including explicitly unauthenticated+profiles whose staging occurs after materialization rather than a credential read.+Private capture binds that request-owned weak handle at formal attempt promotion,+after validating the pending binding and credential. A prebody failure closes+body capture and seals the file before `stopped`; later promotion may replace+the sample atomically with an identical content prefix, one typed correlation+record and a new durable seal. A private temporary copy is synced before rename,+so termination always leaves either complete version; temporary bytes count toward+the session quota. If that quota cannot hold both copies, the original survives. It+rechecks the private directory, file identity/permissions/owner, expiry and+original file/session/record budgets. Content capture never reopens, and the+weak handle cannot retain a writer or bind another request or attempt.++The helper requires Unix Python with non-reaping `os.waitid`/`os.WNOWAIT`+(Linux, or Python 3.13+ on macOS). It checks support before creating a session.+From a clean checkout of the exact candidate, wrap the managed Debug daemon and+its isolated settings arguments with:++```sh+python3 scripts/private-stream-capture.py run \+ --root /absolute/new-private-session --source-sha FULL_CANDIDATE_SHA \+ --client isolated-client-version --seconds 300 --attempts 3 \+ -- /absolute/managed-debug-hirouted ISOLATED_DAEMON_ARGUMENTS+```++The helper issues no requests. Disable client retries, use a separate API-key+context, and stop on the first failure. Each private file is limited to 8 MiB,+the session to 32 MiB and four underlying attempts, and capture expires within+one hour. Credentials in transport headers are excluded. Never commit or upload+the files. Run the same candidate's Debug `stream_replay` executable with the+private `attempt-N.capture` path to check original, one-byte and 4096-byte chunks;+it prints only safe results. Incomplete or unsealed samples are rejected.+Replay supports successful cross-protocol decoder paths, skips informational+HTTP heads, and rejects native projector or non-success paths as+`unsupported_capture_path` without a decoder verdict.+A separate, bounded retention owner stops the isolated candidate's entire process+group, including descendants after the leader exits, and reaps the leader. It+deletes the raw session within 24 hours, even after the CLI returns. Its PID is printed;+SIGTERM also stops the candidate and deletes the session immediately. Keep the+owner running until deletion. Host shutdown or SIGKILL of that owner cannot run+cleanup; after recovery, delete the exact expired session with+`python3 scripts/private-stream-capture.py cleanup /absolute/new-private-session`.+Kernel locks release on candidate crashes, so stale lock files do not prevent+recovery cleanup. No shared scheduler or daily daemon is modified.+Known incomplete nonstream JSON responses follow the same prebody relay boundaryas streaming failures, for both native and converted protocols. Reported usagesurvives fallback; partial response content is not delivered. Attempt/deadlinediff --git a/crates/gateway/examples/stream_replay.rs b/crates/gateway/examples/stream_replay.rsnew file mode 100644index 00000000..74d460f7--- /dev/null+++ b/crates/gateway/examples/stream_replay.rs@@ -0,0 +1,29 @@+//! Private offline replay of a Debug capture; never prints bodies or error text.+#[cfg(all(unix, debug_assertions))]+#[tokio::main]+async fn main() {+ let args: Vec<_> = std::env::args_os().collect();+ if args.len() != 2 {+ eprintln!("usage: stream_replay PRIVATE_CAPTURE");+ std::process::exit(2);+ }+ for chunk in [0, 1, 4096] {+ match hiroute_gateway::runtime::stream_capture::replay_capture(+ std::path::Path::new(&args[1]),+ chunk,+ )+ .await+ {+ Ok(summary) => println!("{summary}"),+ Err(reason) => {+ eprintln!("{reason}");+ std::process::exit(1);+ }+ }+ }+}+#[cfg(not(all(unix, debug_assertions)))]+fn main() {+ eprintln!("stream replay requires Unix Debug build");+ std::process::exit(2);+}diff --git a/crates/gateway/src/adapters/response/decoder.rs b/crates/gateway/src/adapters/response/decoder.rsindex 1ff3b7b5..c068e61e 100644--- a/crates/gateway/src/adapters/response/decoder.rs+++ b/crates/gateway/src/adapters/response/decoder.rs@@ -43,6 +43,8 @@ pub struct NativeResponseDecoder {framer: Option<SseFramer>,ended: bool,end_stream_pending: bool,+ received_bytes: u64,+ frame_index: u64,}impl NativeResponseDecoder {@@ -186,6 +188,8 @@ impl NativeResponseDecoder {framer,ended: false,end_stream_pending: false,+ received_bytes: 0,+ frame_index: 0,})}@@ -205,6 +209,7 @@ impl NativeResponseDecoder {if self.events.len() >= MAX_QUEUED_EVENTS {return Ok(ResponseDecodeStatus::NeedDrain);}+ self.received_bytes = self.received_bytes.saturating_add(bytes.len() as u64);let need_drain = if self.streaming {if end_stream {self.end_stream_pending = true;@@ -260,6 +265,7 @@ impl NativeResponseDecoder {&self.budget,&mut self.state,&mut self.events,+ &mut self.frame_index,)?;if matches!(outcome, SseFeedOutcome::Complete) && self.end_stream_pending {self.finish_stream_eof()?;@@ -277,6 +283,11 @@ impl NativeResponseDecoder {self.events.drain(..).collect()}+ /// Safe failure position; received bytes are an upper bound for a multi-event read.+ pub fn diagnostic_position(&self) -> (u64, u64) {+ (self.frame_index, self.received_bytes)+ }+pub fn sse_complexity(&self) -> Option<SseComplexity> {self.framer.as_ref().map(SseFramer::complexity)}@@ -310,13 +321,14 @@ impl NativeResponseDecoder {) -> Result<SseFeedOutcome, ProtocolAdapterError> {let charged =ChargedBytes::copy_from_opaque(&self.budget, MemoryRole::TransportInflight, bytes)- .map_err(|error| ModelIrError::InvalidSse(error.to_string()))?;+ .map_err(|_| ModelIrError::BufferLimit(bytes.len()))?;let outcome = run_sse(self.framer.as_mut().expect("streaming decoder has framer"),Some((charged, end_stream)),&self.budget,&mut self.state,&mut self.events,+ &mut self.frame_index,)?;Ok(outcome)}@@ -337,12 +349,14 @@ fn run_sse(budget: &StreamBudget,state: &mut ProtocolState,output: &mut VecDeque<ModelStreamEventV1>,+ frame_index: &mut u64,) -> Result<SseFeedOutcome, ProtocolAdapterError> {let mut visitor = NativeVisitor {budget,state,output,protocol_error: None,+ frame_index,};let mut sink = DiscardOutput;let framed = match input {@@ -352,7 +366,16 @@ fn run_sse(if let Some(error) = visitor.protocol_error {return Err(error);}- framed.map_err(|error| ModelIrError::InvalidSse(error.to_string()).into())+ framed.map_err(|error| {+ match error {+ SseError::BudgetExceeded+ | SseError::EventLimit+ | SseError::PendingLimit+ | SseError::OutputLimit => ModelIrError::BufferLimit(usize::MAX),+ other => ModelIrError::InvalidSse(other.to_string()),+ }+ .into()+ })}struct NativeVisitor<'a> {@@ -360,6 +383,7 @@ struct NativeVisitor<'a> {state: &'a mut ProtocolState,output: &'a mut VecDeque<ModelStreamEventV1>,protocol_error: Option<ProtocolAdapterError>,+ frame_index: &'a mut u64,}impl SseVisitor for NativeVisitor<'_> {@@ -371,6 +395,7 @@ impl SseVisitor for NativeVisitor<'_> {if self.output.len() >= MAX_QUEUED_EVENTS {return Err(SseError::NeedDrain);}+ *self.frame_index = self.frame_index.saturating_add(1);if event.fields().any(|field| !field.comment && field.name.starts_with(b"data") && field.name != b"data")@@ -415,3 +440,46 @@ impl BoundedOutputSink for DiscardOutput {Ok(())}}++#[cfg(test)]+mod diagnostic_tests {+ use super::*;+ use crate::server::core_runtime::profiles::fixed_reasoning;++ #[test]+ fn actual_sse_budget_rejection_is_distinct_from_malformed_sse() {+ let profile = CandidateProtocolProfile::exact_portable_path(+ IngressProtocol::Messages,+ IngressProtocol::ChatCompletions,+ "physical",+ fixed_reasoning("fixed"),+ );+ for fragmented in [false, true] {+ let budget = BudgetTree::new(1024, 1024).unwrap().stream(1024).unwrap();+ let mut decoder =+ NativeResponseDecoder::new_for_attempt(&profile, 200, true, None, budget.clone())+ .unwrap();+ let error = if fragmented {+ (0..16)+ .find_map(|_| decoder.feed(&[b'x'; 256], false).err())+ .expect("retained frame must exhaust the owner")+ } else {+ decoder.feed(&[b'x'; 2048], false).unwrap_err()+ };+ assert!(+ matches!(+ error,+ ProtocolAdapterError::ModelIr(ModelIrError::BufferLimit(_))+ ),+ "{error:?}"+ );+ drop(decoder);+ assert_eq!(budget.snapshot().unwrap().live, 0);+ }+ let mut decoder = NativeResponseDecoder::new(&profile, 200, true).unwrap();+ assert!(matches!(+ decoder.feed(b"data-invalid: {}\n\n", false),+ Err(ProtocolAdapterError::ModelIr(ModelIrError::InvalidSse(_)))+ ));+ }+}diff --git a/crates/gateway/src/adapters/tool_projection.rs b/crates/gateway/src/adapters/tool_projection.rsindex 7ba1c2be..71d3e37d 100644--- a/crates/gateway/src/adapters/tool_projection.rs+++ b/crates/gateway/src/adapters/tool_projection.rs@@ -11,7 +11,7 @@ use super::ProtocolAdapterError;const MAX_CHAT_TOOL_NAME_BYTES: usize = 64;-#[derive(Clone, Debug, Eq, PartialEq)]+#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)]pub(crate) struct ChatToolIdentity {pub(crate) emitted_name: String,pub(crate) kind: ToolKindV1,@@ -19,7 +19,7 @@ pub(crate) struct ChatToolIdentity {pub(crate) local_name: String,}-#[derive(Clone, Debug, PartialEq)]+#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]pub(crate) struct ChatToolProjection {entries: Vec<ChatToolIdentity>,}diff --git a/crates/gateway/src/core_runtime.rs b/crates/gateway/src/core_runtime.rsindex db468918..e515e581 100644--- a/crates/gateway/src/core_runtime.rs+++ b/crates/gateway/src/core_runtime.rs@@ -1733,3 +1733,7 @@ mod inbound_auth_tests;#[cfg(test)]#[path = "core_runtime/acceptance_tests.rs"]mod acceptance_tests;++#[cfg(all(test, unix, debug_assertions, feature = "e2e-test-control"))]+#[path = "core_runtime/response_diagnostics_tests.rs"]+mod response_diagnostics_tests;diff --git a/crates/gateway/src/core_runtime/response_diagnostics_tests.rs b/crates/gateway/src/core_runtime/response_diagnostics_tests.rsnew file mode 100644index 00000000..68161f05--- /dev/null+++ b/crates/gateway/src/core_runtime/response_diagnostics_tests.rs@@ -0,0 +1,1034 @@+//! Deterministic upstream injections through the production HTTPS provider and listener.+//! Each child owns its environment, diagnostics, Replay and optional private capture.+use std::fs;+use std::io::{Read, Write};+use std::net::{TcpListener, TcpStream};+use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt};+use std::path::{Path, PathBuf};+use std::process::Command;+use std::sync::{Arc, Mutex};+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};++use hiroute_diagnostics::event::ProcessRole;+use hiroute_diagnostics::level::DiagnosticLevel;+use hiroute_diagnostics::record::Component;+use hiroute_diagnostics::runtime::{DiagnosticRuntime, RuntimeConfig};+use serde_json::{Value, json};++use super::*;+use crate::server::core_runtime::observation::{+ GatewayObservationSinks, ObservationAck, ObservationNack, ObservationRecord,+ ObservationRecordSink, accounted_acknowledgement,+};+use crate::server::publication::{+ AliasPlanV1, GatewayPublicationInstaller, GatewayPublicationSnapshotV3, GrantV1, ModelRouteV2,+ token_sha256,+};+use crate::server::test_control::{+ E2E_DIAL_CONFIG_ENV, E2E_DIAL_CONFIG_FILE, TestTlsListener, sealed_native_candidate,+ write_dial_config,+};++const CHILD_CASE: &str = "HIROUTE_STREAM_DIAGNOSTIC_CASE";+const CHILD_ROOT: &str = "HIROUTE_STREAM_DIAGNOSTIC_ROOT";+const TOKEN: &str = "SENTINEL_CLIENT_CREDENTIAL";+const TOOL: &str = "SENTINEL_DYNAMIC_TOOL";+const TEST: &str = "server::core_runtime::response_diagnostics_tests::production_listener_stream_diagnostics_and_private_capture";++struct PrivateRoot(PathBuf);+impl Drop for PrivateRoot {+ fn drop(&mut self) {+ let _ = fs::remove_dir_all(&self.0);+ }+}++#[test]+fn production_listener_stream_diagnostics_and_private_capture() {+ if let Ok(case) = std::env::var(CHILD_CASE) {+ run_case(&case, &PathBuf::from(std::env::var_os(CHILD_ROOT).unwrap()));+ return;+ }+ let mut failures = Vec::new();+ for case in [+ "unsupported_known",+ "unknown_metadata",+ "unsupported_value",+ "invalid_type",+ "invalid_json",+ "prebody_invalid_json",+ "prebody_capture_invalid_json",+ "prebody_capture_relay",+ "prebody_capture_no_credential",+ "prebody_capture_no_credential_relay",+ "invalid_sse",+ "missing_tool",+ "invalid_arguments",+ "invalid_lifecycle",+ "missing_terminal",+ "duplicate_terminal",+ "resource_limit",+ "drain_failure",+ "capture_known",+ "capture_unknown",+ "capture_disabled",+ ] {+ if let Err(failure) = run_child_case(case) {+ failures.push(failure);+ }+ }+ assert!(failures.is_empty(), "{}", failures.join("\n"));+}++#[test]+fn production_completed_prefix_capture_keeps_formal_attempt_correlation() {+ run_child_case("capture_known").unwrap();+}++#[test]+fn production_prebody_failed_capture_keeps_promoted_attempt_correlation() {+ let failures: Vec<_> = ["prebody_capture_invalid_json", "prebody_capture_relay"]+ .into_iter()+ .filter_map(|case| run_child_case(case).err())+ .collect();+ assert!(failures.is_empty(), "{}", failures.join("\n"));+}++#[test]+fn production_no_credential_prebody_capture_keeps_promoted_attempt_correlation() {+ let failures: Vec<_> = [+ "prebody_capture_no_credential",+ "prebody_capture_no_credential_relay",+ ]+ .into_iter()+ .filter_map(|case| run_child_case(case).err())+ .collect();+ assert!(failures.is_empty(), "{}", failures.join("\n"));+}++fn run_child_case(case: &str) -> Result<(), String> {+ let mut random = [0u8; 16];+ getrandom::fill(&mut random).unwrap();+ let root = PrivateRoot(+ fs::canonicalize(std::env::temp_dir())+ .unwrap()+ .join(format!(+ "hiroute-stream-acceptance-{:x}",+ u128::from_le_bytes(random)+ )),+ );+ fs::DirBuilder::new().mode(0o700).create(&root.0).unwrap();+ let mut child = Command::new(std::env::current_exe().unwrap());+ child+ .args(["--exact", TEST, "--nocapture"])+ .env(CHILD_CASE, case)+ .env(CHILD_ROOT, &root.0)+ .env(E2E_DIAL_CONFIG_ENV, root.0.join(E2E_DIAL_CONFIG_FILE))+ .env("HIROUTE_REPLAY_ROOT", root.0.join("replay"))+ .env_remove("HIROUTE_PRIVATE_STREAM_CAPTURE");+ if matches!(+ case,+ "capture_known"+ | "capture_unknown"+ | "prebody_capture_invalid_json"+ | "prebody_capture_relay"+ | "prebody_capture_no_credential"+ | "prebody_capture_no_credential_relay"+ ) {+ child.env("HIROUTE_PRIVATE_STREAM_CAPTURE", root.0.join("capture"));+ }+ let output = child.output().unwrap();+ if output.status.success() && String::from_utf8_lossy(&output.stdout).contains("1 passed") {+ println!("stream injection {case}: green");+ Ok(())+ } else {+ Err(format!(+ "{case}: {}\n{}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ ))+ }+}++#[derive(Clone, Default)]+struct Facts(Arc<Mutex<Vec<Value>>>);+impl ObservationRecordSink for Facts {+ fn deliver(&self, record: &ObservationRecord) -> Result<ObservationAck, ObservationNack> {+ self.0+ .lock()+ .unwrap()+ .push(serde_json::from_slice(record.payload()).unwrap());+ Ok(accounted_acknowledgement(record))+ }+}++fn chat(delta: Value, finish: Value) -> Value {+ json!({"id":"SENTINEL_RESPONSE_BODY","model":"physical",+ "choices":[{"index":0,"delta":delta,"finish_reason":finish}]})+}+fn frame(value: Value) -> Vec<u8> {+ format!("data: {value}\n\n").into_bytes()+}+fn tool(name: Option<&str>, arguments: &str) -> Value {+ let mut call = json!({"index":0,"type":"function","function":{"arguments":arguments}});+ if let Some(name) = name {+ call["id"] = "SENTINEL_NATIVE_CALL".into();+ call["function"]["name"] = name.into();+ }+ json!({"tool_calls":[call]})+}++fn sample(case: &str) -> (Vec<u8>, Option<&'static str>, Option<&'static str>, u64) {+ let (mut wire, reason, field, index) = match case {+ "unsupported_known" => {+ let mut value = chat(json!({}), Value::Null);+ value["system_fingerprint"] = "SENTINEL_PROVIDER_TEXT".into();+ (+ frame(value),+ Some("unsupported_field"),+ Some("system_fingerprint"),+ 1,+ )+ }+ "unknown_metadata" => {+ // Unknown sibling metadata is already tolerated by the production+ // decoder. Keep that semantic policy and verify it never enters logs.+ let mut value = chat(json!({"content":"SENTINEL_PROVIDER_TEXT"}), json!("stop"));+ value["SENTINEL_UNKNOWN_FIELD"] = "SENTINEL_PROVIDER_TEXT".into();+ value["usage"] = json!({"prompt_tokens":11,"completion_tokens":7,"total_tokens":18});+ (frame(value), None, None, 2)+ }+ "invalid_type" => (+ frame(chat(+ json!({"content":{"SENTINEL_PROVIDER_TEXT":true}}),+ Value::Null,+ )),+ Some("invalid_field"),+ Some("content"),+ 1,+ ),+ "unsupported_value" => (+ frame(chat(json!({"role":"SENTINEL_PROVIDER_ROLE"}), Value::Null)),+ Some("unsupported_value"),+ None,+ 1,+ ),+ "invalid_json"+ | "prebody_invalid_json"+ | "prebody_capture_invalid_json"+ | "prebody_capture_relay"+ | "prebody_capture_no_credential"+ | "prebody_capture_no_credential_relay" => (+ b"data: {\"SENTINEL_PROVIDER_TEXT\":\n\n".to_vec(),+ Some("invalid_json"),+ None,+ 1,+ ),+ "invalid_sse" => (+ b"data-invalid: SENTINEL_PROVIDER_TEXT\n\n".to_vec(),+ Some("invalid_sse"),+ None,+ 1,+ ),+ "missing_tool" => (+ frame(chat(tool(None, "{}"), Value::Null)),+ Some("missing_tool_identity"),+ None,+ 1,+ ),+ "invalid_arguments" => (+ frame(chat(+ tool(Some(TOOL), "{\"SENTINEL_ARGUMENT\":"),+ json!("tool_calls"),+ )),+ Some("invalid_tool_arguments"),+ None,+ 1,+ ),+ "invalid_lifecycle" => {+ let mut wire = frame(chat(tool(Some(TOOL), "{"), Value::Null));+ let mut changed = tool(Some(TOOL), "}");+ changed["tool_calls"][0]["id"] = "SENTINEL_CHANGED_CALL".into();+ wire.extend(frame(chat(changed, Value::Null)));+ (wire, Some("invalid_lifecycle"), None, 2)+ }+ "missing_terminal" => (+ frame(chat(+ json!({"content":"SENTINEL_PROVIDER_TEXT"}),+ Value::Null,+ )),+ Some("missing_terminal"),+ None,+ 1,+ ),+ "duplicate_terminal" => {+ let mut wire = frame(chat(+ json!({"content":"SENTINEL_PROVIDER_TEXT"}),+ json!("stop"),+ ));+ wire.extend_from_slice(b"data: [DONE]\n\ndata: [DONE]\n\n");+ (wire, Some("duplicate_terminal"), None, 3)+ }+ "resource_limit" => (+ frame(chat(+ json!({"content":"x".repeat(6 * 1024 * 1024)}),+ Value::Null,+ )),+ Some("resource_limit"),+ None,+ 0,+ ),+ "drain_failure" => {+ let mut wire = Vec::new();+ for _ in 0..40 {+ wire.extend(frame(chat(+ json!({"content":"SENTINEL_PROVIDER_TEXT"}),+ Value::Null,+ )));+ }+ let mut value = chat(json!({}), Value::Null);+ value["system_fingerprint"] = "SENTINEL_PROVIDER_TEXT".into();+ wire.extend(frame(value));+ (+ wire,+ Some("unsupported_field"),+ Some("system_fingerprint"),+ 41,+ )+ }+ "capture_unknown" => (+ frame(chat(+ tool(Some("SENTINEL_UNKNOWN_TOOL"), "{}"),+ json!("tool_calls"),+ )),+ Some("missing_tool_identity"),+ None,+ 1,+ ),+ "capture_known" => {+ let mut value = chat(tool(Some(TOOL), "{}"), json!("tool_calls"));+ value["usage"] = json!({"prompt_tokens":11,"completion_tokens":7,"total_tokens":18});+ (frame(value), None, None, 2)+ }+ "capture_disabled" => {+ let mut value = chat(json!({"content":"SENTINEL_PROVIDER_TEXT"}), json!("stop"));+ value["usage"] = json!({"prompt_tokens":11,"completion_tokens":7,"total_tokens":18});+ (frame(value), None, None, 2)+ }+ _ => panic!("unknown test case"),+ };+ if matches!(+ case,+ "invalid_arguments"+ | "unknown_metadata"+ | "capture_known"+ | "capture_unknown"+ | "capture_disabled"+ ) {+ wire.extend_from_slice(b"data: [DONE]\n\n");+ }+ (wire, reason, field, index)+}++fn run_case(case: &str, root: &Path) {+ let capture_enabled = std::env::var_os("HIROUTE_PRIVATE_STREAM_CAPTURE").is_some();+ if capture_enabled {+ create_capture_session(&root.join("capture"));+ }+ let (tail, reason, field, frame_index) = sample(case);+ let prebody = case.starts_with("prebody_");+ let relay = case.ends_with("_relay");+ let no_credential = case.contains("no_credential");+ let after_body = reason.is_some() && !prebody;+ let mut wire = if after_body {+ frame(chat(+ json!({"content":"SENTINEL_INITIAL_BODY"}),+ Value::Null,+ ))+ } else {+ Vec::new()+ };+ let prefix_len = wire.len();+ wire.extend(tail);+ let frame_index = if after_body && frame_index != 0 {+ frame_index + 1+ } else {+ frame_index+ };+ let (body_delivered, wait_for_body) = std::sync::mpsc::sync_channel(0);+ let listener = TestTlsListener::bind("stream-diagnostic.invalid").unwrap();+ write_dial_config(root, &[&listener]).unwrap();+ let peer = listener.try_clone().unwrap();+ let sent = wire.clone();+ let upstream = std::thread::spawn(move || {+ let (mut stream, _) = peer.accept().unwrap();+ stream+ .set_read_timeout(Some(Duration::from_secs(10)))+ .unwrap();+ let request = read_request(&mut stream);+ write!(stream, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", sent.len()).unwrap();+ if after_body {+ stream.write_all(&sent[..prefix_len]).unwrap();+ stream.flush().unwrap();+ // No error bytes exist on the wire until the client has received+ // the first real body unit. This proves the afterbody boundary.+ wait_for_body.recv_timeout(Duration::from_secs(3)).unwrap();+ let _ = stream.write_all(&sent[prefix_len..]);+ } else {+ let _ = stream.write_all(&sent);+ }+ let _ = stream.finish();+ if relay {+ let (mut stream, _) = peer.accept().unwrap();+ let _second_request = read_request(&mut stream);+ let mut value = chat(json!({"content":"SENTINEL_RECOVERY_BODY"}), json!("stop"));+ value["usage"] = json!({"prompt_tokens":11,"completion_tokens":7,"total_tokens":18});+ let mut recovery = frame(value);+ recovery.extend_from_slice(b"data: [DONE]\n\n");+ write!(stream, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", recovery.len()).unwrap();+ stream.write_all(&recovery).unwrap();+ let _ = stream.finish();+ }+ request+ });+ let publications = Arc::new(GatewayPublicationInstaller::open(root.join("lkg.json")).unwrap());+ let publication = publication(&listener, prebody && !relay, no_credential);+ if let crate::server::publication::GatewayPrepareOutcome::Prepared(prepared) =+ publications.prepare(publication).unwrap()+ {+ publications.publish(prepared).unwrap();+ }+ let credential = json!({"schema_version":"hiroute.gateway.credential-leases/v1",+ "credential_ref":"stream-credential","keys":[{"key_id":"stream-key","generation":1,+ "authorization":"Bearer SENTINEL_UPSTREAM_CREDENTIAL"}]});+ fs::write(+ root.join("lease.json"),+ serde_json::to_vec(&credential).unwrap(),+ )+ .unwrap();+ fs::write(root.join("credentials.json"), serde_json::to_vec(&json!({+ "schema_version":"hiroute.gateway.credentials/v1","credentials":{"stream-credential":"lease.json"}+ })).unwrap()).unwrap();+ let mut ports = ProductionPorts::fail_closed(publications);+ ports.credentials = Arc::new(+ crate::server::composition::FileCredentialResolver::open(&root.join("credentials.json"))+ .unwrap(),+ );+ ports.runtime_state = Arc::new(crate::ports::InMemoryRuntimeStateStore::default());+ let diagnostics = DiagnosticRuntime::start(RuntimeConfig {+ root: root.join("diagnostics"),+ role: ProcessRole::Daemon,+ component: Component::Gateway,+ parent_session_id: None,+ level_override: Some(DiagnosticLevel::Debug),+ });+ let facts = Facts::default();+ let mut sinks = GatewayObservationSinks::discard();+ sinks.execution_fact = Arc::new(facts.clone());+ let observation =+ GatewayObservation::with_sinks(512 * 1024, sinks).with_diagnostics(diagnostics.port());+ let runtime = ProductionGatewayRuntime::compose_with_planner_and_observation(+ ports,+ Arc::new(PublicationPlannerInputAuthority),+ Arc::new(observation),+ );+ let address = TcpListener::bind("127.0.0.1:0")+ .unwrap()+ .local_addr()+ .unwrap();+ let mut gateway = crate::server::GatewayLauncher::from_runtime(address, Arc::new(runtime))+ .unwrap()+ .start_managed()+ .unwrap();+ let body = serde_json::to_vec(&json!({"model":"stream-alias","max_tokens":64,+ "messages":[{"role":"user","content":"SENTINEL_REQUEST_BODY"}],"stream":true,+ "tools":[{"name":TOOL,"description":"SENTINEL_TOOL_DESCRIPTION","input_schema":{"type":"object"}}]})).unwrap();+ let mut client = TcpStream::connect(address).unwrap();+ client+ .set_read_timeout(Some(Duration::from_secs(15)))+ .unwrap();+ write!(client, "POST /v1/messages HTTP/1.1\r\nHost: {address}\r\nX-HiRoute-Token: {TOKEN}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).unwrap();+ client.write_all(&body).unwrap();+ let mut response = Vec::new();+ if after_body {+ client+ .set_read_timeout(Some(Duration::from_secs(2)))+ .unwrap();+ while !response+ .windows(b"SENTINEL_INITIAL_BODY".len())+ .any(|w| w == b"SENTINEL_INITIAL_BODY")+ {+ let mut chunk = [0u8; 4096];+ let n = client+ .read(&mut chunk)+ .expect("initial body was not delivered before the error");+ assert!(n != 0, "response ended before initial body delivery");+ response.extend_from_slice(&chunk[..n]);+ }+ body_delivered.send(()).unwrap();+ client+ .set_read_timeout(Some(Duration::from_secs(15)))+ .unwrap();+ }+ // A failed committed stream may end with a reset. The formal execution fact is the oracle.+ let _ = client.read_to_end(&mut response);+ assert!(+ response.starts_with(if prebody && !relay {+ b"HTTP/1.1 502"+ } else {+ b"HTTP/1.1 200"+ }),+ "{}",+ String::from_utf8_lossy(&response)+ );+ if case == "capture_known" {+ let response = String::from_utf8_lossy(&response);+ assert!(response.contains("\"type\":\"tool_use\""));+ assert!(response.contains(&format!("\"name\":\"{TOOL}\"")));+ assert!(response.contains("\"partial_json\":\"{}\""));+ assert!(response.contains("event: message_stop"));+ }+ if relay {+ assert!(+ response+ .windows(b"SENTINEL_RECOVERY_BODY".len())+ .any(|w| w == b"SENTINEL_RECOVERY_BODY")+ );+ }+ let upstream_request = upstream.join().unwrap();+ assert!(!upstream_request.is_empty());+ if no_credential {+ assert!(+ !String::from_utf8_lossy(&upstream_request)+ .to_ascii_lowercase()+ .contains("authorization:")+ );+ }+ wait_finished(&facts, reason.is_some() && !relay);+ gateway.shutdown();+ gateway.join(Duration::from_secs(5)).unwrap();+ let status = diagnostics.status();+ assert!(status.armed);+ assert!(status.unavailable.is_none());+ assert!(status.settings_error.is_none());+ diagnostics.shutdown();+ // Keep the collector live through Gateway shutdown; request_finished is+ // a terminal business fact, not a barrier for every observation producer.+ let facts = facts.0.lock().unwrap().clone();+ if no_credential {+ assert!(+ !facts.iter().any(+ |r| r.pointer("/fact/kind").and_then(Value::as_str) == Some("credential_lease")+ )+ );+ }+ let log = fs::read_to_string(root.join("diagnostics/daemon/current.jsonl")).unwrap();+ assert!(+ !log.contains("SENTINEL"),+ "untrusted body/tool/header leaked into product diagnostics"+ );+ let records: Vec<Value> = log+ .lines()+ .map(|line| serde_json::from_str(line).unwrap())+ .collect();+ assert!(+ records+ .iter()+ .any(|r| payload(r, "level_applied").is_some_and(|v| v["level"] == "debug")),+ "actual Debug level missing: {log}"+ );+ if relay {+ assert_prebody_relay_business(&facts);+ } else {+ assert_business(&facts, reason.is_some(), after_body);+ }+ if let Some(reason) = reason {+ let failures: Vec<_> = records+ .iter()+ .filter_map(|r| payload(r, "response_failure"))+ .collect();+ assert_eq!(failures.len(), 1, "{case}: {log}");+ let failure = failures[0];+ assert_eq!(failure["reason"], reason, "{case}");+ assert_eq!(+ failure.get("field").and_then(Value::as_str),+ field,+ "{case}"+ );+ assert_eq!(failure["stage"], "decode", "{case}");+ if frame_index != 0 {+ assert_eq!(failure["frame_index"], frame_index, "{case}");+ }+ assert!(+ failure["received_bytes"]+ .as_u64()+ .is_some_and(|n| n > 0 && n <= wire.len() as u64)+ );+ assert_eq!(failure["attempt_index"], 1);+ let begin = records+ .iter()+ .find_map(|r| payload(r, "attempt_begin"))+ .unwrap();+ let end = records+ .iter()+ .find_map(|r| payload(r, "request_end"))+ .unwrap();+ if after_body {+ assert!(!failure["attempt_token"].is_null());+ assert_eq!(failure["attempt_token"], begin["attempt_token"]);+ } else {+ // Prebody classification precedes formal attempt promotion. The+ // diagnostic must retain the request and ordinal, without inventing+ // an unavailable attempt token.+ assert!(failure["attempt_token"].is_null());+ }+ assert!(!failure["request_token"].is_null());+ assert_eq!(failure["request_token"], end["request_token"]);+ } else {+ assert!(+ records+ .iter()+ .all(|r| payload(r, "response_failure").is_none())+ );+ }+ wait_replay_clean(&root.join("replay"));+ if capture_enabled {+ assert_capture(+ &root.join("capture"),+ &upstream_request,+ &wire,+ reason,+ &records,+ &facts,+ );+ } else {+ assert!(+ !root.join("capture").exists(),+ "default-disabled path created capture state"+ );+ }+}++fn publication(+ provider: &TestTlsListener,+ limit_prebody_attempts: bool,+ no_credential: bool,+) -> GatewayPublicationSnapshotV3 {+ let protocol = IngressProtocol::Messages;+ GatewayPublicationSnapshotV3::seal(+ "personal/default",+ "stream-authority",+ 1,+ 1,+ "stream-renderer/v1",+ vec![AliasPlanV1 {+ served_model_id: "stream-alias".into(),+ purpose: "stream diagnostics acceptance".into(),+ agent_plan_revision: 1,+ protocols: vec![protocol],+ overall_timeout_ms: 30_000,+ // The initial malformed entity is tested with an explicit one-attempt+ // budget. Other cases retain both candidates and prove no replay once+ // the client has actually received the first body unit.+ max_attempts: if limit_prebody_attempts { 1 } else { 2 },+ routing: None,+ candidates: (1..=2)+ .map(|i| {+ let mut candidate = sealed_native_candidate(+ i,+ &format!("stream-target-{i}"),+ &["stream-credential".into()],+ provider.authority(),+ "physical",+ &[(protocol, IngressProtocol::ChatCompletions)],+ );+ if no_credential {+ candidate.credential_refs = vec![format!("credential/none/stream-{i}")];+ for profile in &mut candidate.protocol_profiles {+ profile.connector.authentication =+ hiroute_domain::GatewayCriticalFactV1::Exact(+ hiroute_domain::GatewayAuthenticationSemanticsV1::None,+ );+ }+ candidate.protocol_profile_digest =+ hiroute_domain::CanonicalDigest::of(&candidate.protocol_profiles)+ .unwrap();+ }+ candidate+ })+ .collect(),+ }],+ vec![GrantV1 {+ grant_id: "stream-grant".into(),+ generation: 1,+ bearer_token_sha256: token_sha256(TOKEN),+ protocol,+ route_protocols: Default::default(),+ routes: [(+ "stream-alias".into(),+ ModelRouteV2::Plan {+ plan_id: "legacy/stream-alias".into(),+ alias: "stream-alias".into(),+ revision: 1,+ semantic_digest: hiroute_domain::CanonicalDigest::of_bytes(b"stream-alias"),+ },+ )]+ .into_iter()+ .collect(),+ }],+ )+ .unwrap()+}++fn read_request(stream: &mut impl Read) -> Vec<u8> {+ let mut bytes = Vec::new();+ let mut expected = None;+ loop {+ let mut chunk = [0u8; 4096];+ let len = stream.read(&mut chunk).unwrap();+ assert!(len > 0, "upstream request ended before its body");+ bytes.extend_from_slice(&chunk[..len]);+ if expected.is_none()+ && let Some(end) = bytes.windows(4).position(|w| w == b"\r\n\r\n")+ {+ let head = String::from_utf8_lossy(&bytes[..end]);+ let length = head+ .lines()+ .find_map(|line| {+ line.to_ascii_lowercase()+ .strip_prefix("content-length: ")+ .map(str::to_owned)+ })+ .unwrap()+ .parse::<usize>()+ .unwrap();+ expected = Some(end + 4 + length);+ }+ if expected.is_some_and(|expected| bytes.len() == expected) {+ return bytes;+ }+ assert!(bytes.len() < 1024 * 1024);+ }+}++fn payload<'a>(record: &'a Value, name: &str) -> Option<&'a Value> {+ record+ .get(name)+ .or_else(|| record.as_object()?.values().find_map(|v| payload(v, name)))+}+fn wait_finished(facts: &Facts, failed: bool) {+ let deadline = Instant::now() + Duration::from_secs(5);+ loop {+ let records = facts.0.lock().unwrap().clone();+ let finished = records+ .iter()+ .any(|r| r.pointer("/fact/kind").and_then(Value::as_str) == Some("request_finished"));+ // The native completion producer can deliver its usage after the+ // request's terminal fact. Successful fixtures require both known+ // usage sources; assert_business still checks the exact count/content.+ let usage_complete = failed+ || records+ .iter()+ .filter(|r| {+ r.pointer("/fact/kind").and_then(Value::as_str) == Some("usage_and_cache")+ })+ .count()+ >= 2;+ if finished && usage_complete {+ return;+ }+ assert!(+ Instant::now() < deadline,+ "incomplete terminal/usage facts: {records:?}"+ );+ std::thread::sleep(Duration::from_millis(5));+ }+}+fn assert_prebody_relay_business(facts: &[Value]) {+ let named = |name| {+ facts+ .iter()+ .filter(|r| r.pointer("/fact/kind").and_then(Value::as_str) == Some(name))+ .collect::<Vec<_>>()+ };+ let starts = named("attempt_started");+ assert_eq!(starts.len(), 2);+ let finished = named("attempt_finished");+ assert_eq!(finished.len(), 2);+ let first = finished.iter().find(|r| r["fact"]["ordinal"] == 1).unwrap();+ let second = finished.iter().find(|r| r["fact"]["ordinal"] == 2).unwrap();+ assert_eq!(first["fact"]["outcome"], "rejected");+ assert_eq!(first["fact"]["disposition"], "continue");+ assert_eq!(first["fact"]["retryable"], true);+ assert_eq!(first["fact"]["commits"]["downstream_headers"], "clear");+ assert_eq!(first["fact"]["commits"]["downstream_semantic"], "clear");+ assert_eq!(second["fact"]["outcome"], "accepted");+ assert_eq!(+ second["fact"]["commits"]["downstream_headers"],+ "write_confirmed"+ );+ let requests = named("request_finished");+ assert_eq!(requests.len(), 1);+ assert_eq!(requests[0]["fact"]["outcome"], "accepted");+ assert_eq!(requests[0]["fact"]["attempts_started"], 2);+ assert_eq!(requests[0]["fact"]["attempts_finished"], 2);+ assert_eq!(named("semantic_commit").len(), 1);+ let usage = named("usage_and_cache");+ assert_eq!(usage.len(), 2);+ for record in usage {+ assert_eq!(record["fact"]["ordinal"], 2);+ assert_eq!(record["fact"]["input_tokens"], 11);+ assert_eq!(record["fact"]["output_tokens"], 7);+ }+}++fn assert_business(facts: &[Value], failed: bool, after_body: bool) {+ let named = |name| {+ facts+ .iter()+ .filter(|r| r.pointer("/fact/kind").and_then(Value::as_str) == Some(name))+ .collect::<Vec<_>>()+ };+ assert_eq!(+ named("attempt_started").len(),+ 1,+ "only one actual attempt may run: bounded prebody or committed afterbody"+ );+ let finished = named("attempt_finished");+ assert_eq!(finished.len(), 1);+ assert_eq!(finished[0]["fact"]["ordinal"], 1);+ assert_eq!(+ finished[0]["fact"]["outcome"],+ if after_body {+ "postcommit_transport_failed"+ } else if failed {+ "rejected"+ } else {+ "accepted"+ }+ );+ if failed && !after_body {+ // Relay eligibility remains Continue; the separately frozen one-attempt+ // budget ends the request without promoting another candidate.+ assert_eq!(finished[0]["fact"]["disposition"], "continue");+ assert_eq!(finished[0]["fact"]["retryable"], true);+ assert_eq!(+ finished[0]["fact"]["commits"]["downstream_semantic"],+ "clear"+ );+ }+ if after_body {+ let semantic_started = finished[0]["fact"]["commits"]["downstream_semantic"] != "clear";+ assert_eq!(+ finished[0]["fact"]["termination_reason"],+ if semantic_started {+ "stream_started_no_retry"+ } else {+ "provider_encoder_failure"+ }+ );+ assert_eq!(+ finished[0]["fact"]["stream_outcome"],+ if semantic_started {+ "stream_started_no_retry"+ } else {+ "aborted_before_semantic_commit"+ }+ );+ }+ assert_eq!(+ finished[0]["fact"]["commits"]["upstream_request"],+ "write_confirmed"+ );+ assert_eq!(+ finished[0]["fact"]["commits"]["downstream_headers"],+ if failed && !after_body {+ "clear"+ } else {+ "write_confirmed"+ }+ );+ let requests = named("request_finished");+ assert_eq!(finished[0]["fact"]["cleanup_outcome"], "completed");+ assert_eq!(requests.len(), 1);+ assert_eq!(+ requests[0]["fact"]["outcome"],+ if failed {+ if finished[0]["fact"]["commits"]["downstream_semantic"] != "clear" {+ "postcommit_transport_failed"+ } else {+ "failed"+ }+ } else {+ "accepted"+ }+ );+ assert_eq!(requests[0]["fact"]["attempts_started"], 1);+ assert_eq!(requests[0]["fact"]["attempts_finished"], 1);+ let semantic_commits = named("semantic_commit").len();+ assert_eq!(semantic_commits, if failed && !after_body { 0 } else { 1 });+ let usage = named("usage_and_cache");+ // These failed entities report no usage: preserve absence rather than+ // synthesizing zero tokens from an incomplete response.+ assert_eq!(usage.len(), if failed { 0 } else { 2 });+ if !failed {+ assert_eq!(+ usage+ .iter()+ .filter(|r| r["fact"]["source"] == "provider_completion")+ .count(),+ 1+ );+ }+ for record in usage {+ assert_eq!(record["fact"]["ordinal"], 1);+ assert!(matches!(+ record["fact"]["source"].as_str(),+ Some("provider_completion" | "accepted_canonical_model_event")+ ));+ assert_eq!(record["fact"]["input_tokens"], 11);+ assert_eq!(record["fact"]["output_tokens"], 7);+ }+}+fn wait_replay_clean(root: &Path) {+ let deadline = Instant::now() + Duration::from_secs(5);+ loop {+ if !root.exists() || fs::read_dir(root).unwrap().next().is_none() {+ return;+ }+ assert!(+ Instant::now() < deadline,+ "Replay resources survived request cleanup"+ );+ std::thread::sleep(Duration::from_millis(5));+ }+}+fn create_capture_session(root: &Path) {+ fs::DirBuilder::new().mode(0o700).create(root).unwrap();+ let now = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .unwrap()+ .as_secs();+ let mut file = fs::OpenOptions::new()+ .create_new(true)+ .write(true)+ .mode(0o600)+ .open(root.join("session.json"))+ .unwrap();+ file.write_all(serde_json::to_string(&json!({"source_sha":"a".repeat(40),"binary_sha256":"b".repeat(64),+ "client":"deterministic-production-listener-fixture","expires_at":now+60,"delete_after":now+120})).unwrap().as_bytes()).unwrap();+}++fn assert_capture(+ root: &Path,+ upstream: &[u8],+ entity: &[u8],+ reason: Option<&str>,+ diagnostics: &[Value],+ facts: &[Value],+) {+ let lock = fs::OpenOptions::new()+ .read(true)+ .write(true)+ .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)+ .open(root.join("active.lock"))+ .unwrap();+ // The stable inode survives both clean exit and crashes; kernel ownership+ // must be released before cleanup can use this same lock.+ rustix::fs::flock(&lock, rustix::fs::FlockOperation::NonBlockingLockExclusive)+ .expect("writer ownership was not released");+ let files: Vec<_> = fs::read_dir(root)+ .unwrap()+ .map(|e| e.unwrap().path())+ .filter(|p| p.extension().is_some_and(|e| e == "capture"))+ .collect();+ assert_eq!(files.len(), 1);+ let path = &files[0];+ assert_eq!(+ fs::metadata(path).unwrap().permissions().mode() & 0o777,+ 0o600+ );+ let bytes = fs::read(path).unwrap();+ assert!(+ !bytes+ .windows(b"SENTINEL_UPSTREAM_CREDENTIAL".len())+ .any(|w| w == b"SENTINEL_UPSTREAM_CREDENTIAL")+ );+ let mut remaining = bytes.as_slice();+ let mut records = Vec::new();+ while !remaining.is_empty() {+ assert!(remaining.len() >= 9);+ let size = u64::from_le_bytes(remaining[1..9].try_into().unwrap()) as usize;+ assert!(size <= remaining.len() - 9);+ records.push((remaining[0], &remaining[9..9 + size]));+ remaining = &remaining[9 + size..];+ }+ assert_eq!(records.last().unwrap().0, 8);+ assert!(records.iter().any(|r| r.0 == 2));+ let captured = |kind| {+ records+ .iter()+ .filter(|r| r.0 == kind)+ .flat_map(|r| r.1.iter().copied())+ .collect::<Vec<_>>()+ };+ let body_offset = upstream.windows(4).position(|w| w == b"\r\n\r\n").unwrap() + 4;+ assert_eq!(+ captured(1),+ upstream[body_offset..],+ "capture must use actual prepared request reader"+ );+ assert_eq!(+ captured(4),+ entity,+ "ordered read blocks must reproduce the exact entity"+ );+ let context: Value = serde_json::from_slice(records[0].1).unwrap();+ assert_eq!(+ context["request_bytes"],+ (upstream.len() - body_offset) as u64+ );+ assert_eq!(context["profile"]["ingress_protocol"], "messages");+ assert_eq!(+ context["profile"]["capability"]["upstream_protocol"],+ "chat_completions"+ );+ assert!(!context["chat_tools"].is_null());+ assert_eq!(records.iter().filter(|r| r.0 == 7).count(), 1);+ let correlation: Value =+ serde_json::from_slice(records.iter().find(|r| r.0 == 7).unwrap().1).unwrap();+ let attempt = diagnostics+ .iter()+ .find_map(|r| payload(r, "attempt_begin"))+ .unwrap();+ assert_eq!(correlation["request_token"], attempt["request_token"]);+ assert_eq!(correlation["attempt_token"], attempt["attempt_token"]);+ assert_eq!(correlation["attempt_index"], attempt["attempt_index"]);+ assert_eq!(+ correlation["request_id"],+ facts+ .iter()+ .find_map(|r| r.pointer("/correlation/request_id"))+ .unwrap()+ .clone()+ );+ let rt = tokio::runtime::Runtime::new().unwrap();+ for chunks in [0, 1, 4096] {+ let replay = rt+ .block_on(crate::runtime::stream_capture::replay_capture(path, chunks))+ .unwrap();+ assert_eq!(+ replay["decoder"]["result"],+ if reason.is_some() {+ "rejected"+ } else {+ "accepted"+ }+ );+ if let Some(reason) = reason {+ assert_eq!(replay["decoder"]["reason"], reason);+ } else {+ assert_eq!(replay["decoder"]["tool_calls"], 1);+ }+ assert!(!replay.to_string().contains("SENTINEL"));+ }+}diff --git a/crates/gateway/src/observation/provider.rs b/crates/gateway/src/observation/provider.rsindex ced83905..9fe97c3a 100644--- a/crates/gateway/src/observation/provider.rs+++ b/crates/gateway/src/observation/provider.rs@@ -111,6 +111,12 @@ impl ProviderRuntimePort for ObservedProductionProvider {if let (Some(observation), Some(credential_ref)) = (&observation, no_credential_ref) {observation.no_credential_materialized(&stable_binding_id, &credential_ref);}+ // The unauthenticated profile has no credential-read staging hook.+ // Bind only after authoritative materialization and its None staging.+ #[cfg(all(unix, debug_assertions))]+ if let Some(observation) = &observation {+ inner.register_private_capture(observation);+ }let tracker = observation.as_ref().and_then(|observation| {observation.begin_response_capture(&stable_binding_id,diff --git a/crates/gateway/src/observation/request.rs b/crates/gateway/src/observation/request.rsindex 8fb0a3b7..9b4921bd 100644--- a/crates/gateway/src/observation/request.rs+++ b/crates/gateway/src/observation/request.rs@@ -82,6 +82,12 @@ pub(super) struct RequestObservationInner {#[derive(Default)]pub(super) struct RequestObservationState {+ #[cfg(all(unix, debug_assertions))]+ pub(super) private_capture: Option<(+ String,+ String,+ crate::runtime::stream_capture::PendingCapture,+ )>,pub(super) agent_plan_id: Option<String>,pub(super) candidates: BTreeMap<String, CandidateObservation>,pub(super) pending_attempt: Option<AttemptObservation>,@@ -243,32 +249,6 @@ impl RequestObservation {.token(CorrelationDomain::Attempt, attempt_id)}- pub(crate) fn response_failure(- &self,- stage: hiroute_diagnostics::event::ResponseFailureStage,- reason: hiroute_diagnostics::event::ResponseFailureReason,- ) {- let state = self.inner.state.lock().unwrap_or_else(|e| e.into_inner());- let attempt = state- .accepted_attempt- .as_ref()- .or(state.current_attempt.as_ref())- .or(state.pending_attempt.as_ref());- let ordinal = attempt- .map(|a| a.ordinal)- .filter(|n| *n != 0)- .unwrap_or(state.next_attempt_ordinal);- drop(state);- self.emit_diagnostic(DiagnosticEvent::ResponseFailure(- hiroute_diagnostics::event::ResponseFailure {- request_token: self.inner.request_token,- attempt_index: u64::from(ordinal),- stage,- reason,- },- ));- }-pub(crate) fn reasoning_cleanup(&self,reason: hiroute_diagnostics::event::ReasoningCleanupReason,diff --git a/crates/gateway/src/observation/request/completion.rs b/crates/gateway/src/observation/request/completion.rsindex eab33381..6d4c3609 100644--- a/crates/gateway/src/observation/request/completion.rs+++ b/crates/gateway/src/observation/request/completion.rs@@ -98,6 +98,10 @@ impl RequestObservation {.unwrap_or_else(|| failure.termination_reason.as_str());let mut state = self.lock_state();let pending = state.pending_attempt.take();+ #[cfg(all(unix, debug_assertions))]+ {+ state.private_capture = None;+ }if disposition == Disposition::Continue {state.previous_attempt_id = pending.and_then(|attempt| attempt.previous_attempt_id);state.next_attempt_reason = Some(format!("precommit_fallback_after_{error_class}"));@@ -302,6 +306,23 @@ impl RequestObservation {],);state.current_attempt = Some(attempt.clone());+ #[cfg(all(unix, debug_assertions))]+ let capture = state+ .private_capture+ .take()+ .filter(|(binding, credential, _)| {+ *binding == attempt.stable_binding_id && *credential == attempt.credential_ref+ });+ drop(state);+ #[cfg(all(unix, debug_assertions))]+ if let Some((_, _, capture)) = capture {+ capture.promoted(crate::runtime::stream_capture::CaptureCorrelation {+ request_token: self.inner.request_token,+ request_id: self.inner.metadata.request_id.clone(),+ attempt_index: attempt.ordinal,+ attempt_token: self.attempt_token(&attempt.attempt_id),+ });+ }Some(attempt)}diff --git a/crates/gateway/src/observation/request/diagnostic.rs b/crates/gateway/src/observation/request/diagnostic.rsindex e2f3e634..631422fc 100644--- a/crates/gateway/src/observation/request/diagnostic.rs+++ b/crates/gateway/src/observation/request/diagnostic.rs@@ -5,6 +5,88 @@ use crate::server::core_runtime::observation::wire_diagnostic;use hiroute_diagnostics::{correlation::CorrelationDomain, event::DiagnosticEvent};impl RequestObservation {+ #[cfg(test)]+ pub(crate) fn response_failure(+ &self,+ stage: hiroute_diagnostics::event::ResponseFailureStage,+ reason: hiroute_diagnostics::event::ResponseFailureReason,+ ) {+ self.response_failure_detail(stage, reason, None, None);+ }++ #[cfg(all(unix, debug_assertions))]+ pub(crate) fn register_private_capture(+ &self,+ capture: crate::runtime::stream_capture::PendingCapture,+ ) {+ let mut state = self.lock_state();+ if let Some(pending) = state.pending_attempt.as_ref() {+ state.private_capture = Some((+ pending.stable_binding_id.clone(),+ pending.credential_ref.clone(),+ capture,+ ));+ }+ }++ #[cfg(all(unix, debug_assertions))]+ pub(crate) fn private_capture_correlation(+ &self,+ ) -> crate::runtime::stream_capture::CaptureCorrelation {+ let state = self.inner.state.lock().unwrap_or_else(|e| e.into_inner());+ let attempt = state+ .accepted_attempt+ .as_ref()+ .or(state.current_attempt.as_ref())+ .or(state.pending_attempt.as_ref());+ crate::runtime::stream_capture::CaptureCorrelation {+ request_token: self.inner.request_token,+ request_id: self.inner.metadata.request_id.clone(),+ attempt_index: attempt+ .map(|a| a.ordinal)+ .filter(|n| *n != 0)+ .unwrap_or(state.next_attempt_ordinal),+ attempt_token: attempt+ .filter(|a| !a.attempt_id.is_empty())+ .and_then(|a| self.attempt_token(&a.attempt_id)),+ }+ }++ pub(crate) fn response_failure_detail(+ &self,+ stage: hiroute_diagnostics::event::ResponseFailureStage,+ reason: hiroute_diagnostics::event::ResponseFailureReason,+ field: Option<hiroute_diagnostics::event::ResponseFailureField>,+ position: Option<(u64, u64)>,+ ) {+ let state = self.inner.state.lock().unwrap_or_else(|e| e.into_inner());+ let attempt = state+ .accepted_attempt+ .as_ref()+ .or(state.current_attempt.as_ref())+ .or(state.pending_attempt.as_ref());+ let ordinal = attempt+ .map(|a| a.ordinal)+ .filter(|n| *n != 0)+ .unwrap_or(state.next_attempt_ordinal);+ let attempt_token = attempt+ .filter(|a| !a.attempt_id.is_empty())+ .and_then(|a| self.attempt_token(&a.attempt_id));+ drop(state);+ self.emit_diagnostic(DiagnosticEvent::ResponseFailure(+ hiroute_diagnostics::event::ResponseFailure {+ request_token: self.inner.request_token,+ attempt_index: u64::from(ordinal),+ stage,+ reason,+ attempt_token,+ field,+ frame_index: position.map(|p| p.0).filter(|n| *n != 0),+ received_bytes: position.map(|p| p.1),+ },+ ));+ }+pub(in crate::server::core_runtime::observation) fn wire_diagnostic(&self,mut event: hiroute_diagnostics::event::UpstreamWire,diff --git a/crates/gateway/src/runtime.rs b/crates/gateway/src/runtime.rsindex 26a7de71..a2ed7071 100644--- a/crates/gateway/src/runtime.rs+++ b/crates/gateway/src/runtime.rs@@ -6,6 +6,8 @@mod driver;mod state;+#[cfg(all(unix, debug_assertions))]+pub mod stream_capture;pub use driver::*;pub use state::StateAccessError;diff --git a/crates/gateway/src/runtime/driver.rs b/crates/gateway/src/runtime/driver.rsindex 240c30cc..2a55b11b 100644--- a/crates/gateway/src/runtime/driver.rs+++ b/crates/gateway/src/runtime/driver.rs@@ -40,7 +40,7 @@ use super::state::{mod materialization;mod response;-mod response_diagnostics;+pub(crate) mod response_diagnostics;mod selection;pub(crate) use materialization::TargetResolver;@@ -165,6 +165,8 @@ impl ProductionReplaySeedEnvelope {}pub struct ProductionAttemptState {+ #[cfg(all(unix, debug_assertions))]+ capture: Option<super::stream_capture::Capture>,response_status: Option<http::StatusCode>,resolved_target: TransportTarget,permits: OwnedAttemptStatePermits,@@ -190,6 +192,16 @@ pub struct ProductionAttemptState {}impl ProductionAttemptState {+ #[cfg(all(unix, debug_assertions))]+ pub(crate) fn register_private_capture(+ &self,+ request: &crate::server::core_runtime::observation::RequestObservation,+ ) {+ if let Some(capture) = &self.capture {+ capture.register(request);+ }+ }+pub(crate) fn chat_tool_projection_for_observation(&self,) -> Option<&adapters::ChatToolProjection> {@@ -198,6 +210,8 @@ impl ProductionAttemptState {}pub struct ProductionReadiness {+ #[cfg(all(unix, debug_assertions))]+ capture: Option<super::stream_capture::Capture>,response_status: http::StatusCode,content_type: &'static str,prefix: ChargedBodyQueue,diff --git a/crates/gateway/src/runtime/driver/materialization.rs b/crates/gateway/src/runtime/driver/materialization.rsindex baa91b2d..66d10fd9 100644--- a/crates/gateway/src/runtime/driver/materialization.rs+++ b/crates/gateway/src/runtime/driver/materialization.rs@@ -459,6 +459,13 @@ pub(super) async fn materialize_attempt(),Some(_) | None => None,};+ #[cfg(all(unix, debug_assertions))]+ let capture = crate::runtime::stream_capture::Capture::begin(+ profile,+ prepared.chat_tool_projection.as_ref(),+ prepared.wire_len,+ streaming,+ );let chat_tool_projection = prepared.chat_tool_projection.take();let body = adapters::sequential_attempt_body(prepared.clone(),@@ -467,6 +474,11 @@ pub(super) async fn materialize_attempt(quantum,).map_err(safe_error)?;+ #[cfg(all(unix, debug_assertions))]+ let body = match capture.as_ref() {+ Some(capture) => capture.wrap(body),+ None => body,+ };let lease = context.leases.acquire().map_err(safe_error)?;let mut headers = HeaderMap::new();let authority = resolved_target.authority.as_ref();@@ -548,6 +560,8 @@ pub(super) async fn materialize_attempt(body,},ProductionAttemptState {+ #[cfg(all(unix, debug_assertions))]+ capture,response_status: None,resolved_target,permits,diff --git a/crates/gateway/src/runtime/driver/response.rs b/crates/gateway/src/runtime/driver/response.rsindex 3bdfa70f..381f6636 100644--- a/crates/gateway/src/runtime/driver/response.rs+++ b/crates/gateway/src/runtime/driver/response.rs@@ -57,6 +57,10 @@ pub(super) fn classify_precommit(state: &mut ProductionAttemptState,event: PrecommitEvent,) -> Result<PrecommitClassification<ProductionReadiness, ProductionDecodedSse>, Arc<str>> {+ #[cfg(all(unix, debug_assertions))]+ if let Some(capture) = &state.capture {+ capture.response(&event);+ }match event {PrecommitEvent::ResponseHead(head) if head.status().is_informational() => {Ok(PrecommitClassification::pending())@@ -126,7 +130,8 @@ pub(super) fn classify_precommit(.ok_or_else(|| Arc::from("native response projector is unavailable"))?.feed(bytes.bytes(), false);drop(bytes);- if projected.is_err() {+ if let Err(error) = projected {+ response_diagnostics::adapter(FailureStage::NativeProjection, error);let raw = response_decode_failure(status, state.retry_after);return classify_state_failure(state,@@ -142,7 +147,8 @@ pub(super) fn classify_precommit(.ok_or_else(|| Arc::from("native response decoder is unavailable"))?.feed(bytes.bytes(), false);drop(bytes);- if decoded.is_err() {+ if let Err(error) = decoded {+ response_diagnostics::adapter(FailureStage::Decode, error);let raw = response_decode_failure(status, state.retry_after);return classify_state_failure(state,@@ -380,6 +386,10 @@ pub(super) fn accepted_response_head(readiness: &ProductionReadiness,published: &PublishedDisposition,) -> Result<GatewayResponseHead, Arc<str>> {+ #[cfg(all(unix, debug_assertions))]+ if let Some(capture) = &readiness.capture {+ capture.record_attempt_correlation();+ }match published.disposition {Disposition::Accept | Disposition::Terminate => {let mut headers = HeaderMap::new();@@ -399,6 +409,24 @@ pub(super) fn accepted_response_head(pub(super) fn encode_accepted_event(readiness: &mut ProductionReadiness,event: ProviderAcceptedEvent<ProductionDecodedSse>,+) -> Result<Option<AcceptedBodyFrame>, Arc<str>> {+ #[cfg(all(unix, debug_assertions))]+ if let (Some(capture), ProviderAcceptedEvent::Raw(raw)) = (&readiness.capture, &event) {+ capture.response(raw);+ }+ let result = encode_accepted_event_inner(readiness, event);+ #[cfg(all(unix, debug_assertions))]+ if result.is_err()+ && let Some(capture) = &readiness.capture+ {+ capture.failed();+ }+ result+}++fn encode_accepted_event_inner(+ readiness: &mut ProductionReadiness,+ event: ProviderAcceptedEvent<ProductionDecodedSse>,) -> Result<Option<AcceptedBodyFrame>, Arc<str>> {let event = match event {ProviderAcceptedEvent::Terminal {@@ -470,7 +498,9 @@ pub(super) fn encode_accepted_event(.take().ok_or_else(|| Arc::from("accepted native decoder is unavailable"))?.finish()- .map_err(|_| Arc::from("accepted native stream ended without terminal"))?;+ .map_err(|error| {+ response_diagnostics::adapter(FailureStage::EndOfStream, error)+ })?;(rendered, true)};queue_accepted_stream_output(readiness, rendered, projected_terminal)@@ -743,6 +773,10 @@ fn classify_state_failure(raw: RawAttemptFailure,local_status: StatusCode,) -> Result<PrecommitClassification<ProductionReadiness, ProductionDecodedSse>, Arc<str>> {+ #[cfg(all(unix, debug_assertions))]+ if let Some(capture) = &state.capture {+ capture.failed();+ }let failure = classify_failure(&raw, connector_error_profile(&state.profile)?);let facts = failure_facts(&failure);state.classified_failure = Some(failure);@@ -784,6 +818,8 @@ fn take_readiness(prefix_eos_pending: bool,) -> Result<ProductionReadiness, Arc<str>> {Ok(ProductionReadiness {+ #[cfg(all(unix, debug_assertions))]+ capture: state.capture.take(),response_status,content_type,prefix: state@@ -897,9 +933,13 @@ fn decode_stream_chunk_readiness(.renderer.as_mut().ok_or_else(|| Arc::from("accepted client renderer is unavailable"))?;- let mut status = decoder- .feed(bytes, end_stream)- .map_err(|error| response_diagnostics::adapter(FailureStage::Decode, error))?;+ let mut status = decoder.feed(bytes, end_stream).map_err(|error| {+ response_diagnostics::adapter_at(+ FailureStage::Decode,+ error,+ Some(decoder.diagnostic_position()),+ )+ })?;let mut output = Vec::new();loop {for event in decoder.take_events() {@@ -926,9 +966,13 @@ fn decode_stream_chunk_readiness(if status != adapters::ResponseDecodeStatus::NeedDrain {break;}- status = decoder- .resume()- .map_err(|_| Arc::from("accepted native stream drain failed"))?;+ status = decoder.resume().map_err(|error| {+ response_diagnostics::adapter_at(+ FailureStage::Decode,+ error,+ Some(decoder.diagnostic_position()),+ )+ })?;}let terminal = status == adapters::ResponseDecodeStatus::Terminal;if terminal && !end_stream {@@ -943,10 +987,13 @@ fn finish_native_stream_on_terminal(let active = decoder.as_mut().ok_or_else(|| Arc::from("native stream decoder is unavailable"))?;- if active- .feed(&[], true)- .map_err(|_| Arc::from("native stream has a malformed terminal tail"))?- != adapters::ResponseDecodeStatus::Terminal+ if active.feed(&[], true).map_err(|error| {+ response_diagnostics::adapter_at(+ FailureStage::EndOfStream,+ error,+ Some(active.diagnostic_position()),+ )+ })? != adapters::ResponseDecodeStatus::Terminal|| !active.take_events().is_empty(){return Err(Arc::from("native stream has events after terminal"));@@ -955,7 +1002,7 @@ fn finish_native_stream_on_terminal(.take().expect("terminal decoder is present").finish()- .map_err(|_| Arc::from("native stream terminal is incomplete"))?;+ .map_err(|error| response_diagnostics::adapter(FailureStage::EndOfStream, error))?;Ok(())}diff --git a/crates/gateway/src/runtime/driver/response_diagnostics.rs b/crates/gateway/src/runtime/driver/response_diagnostics.rsindex 583223dc..61950487 100644--- a/crates/gateway/src/runtime/driver/response_diagnostics.rs+++ b/crates/gateway/src/runtime/driver/response_diagnostics.rs@@ -3,13 +3,18 @@ use crate::server::core_runtime::{model_ir::{ModelError, ModelIrError},};use hiroute_diagnostics::event::{- ResponseFailureReason as Reason, ResponseFailureStage as Stage, WireProviderError,+ ResponseFailureField as Field, ResponseFailureReason as Reason, ResponseFailureStage as Stage,+ WireProviderError,};use std::sync::Arc;pub(super) fn note(stage: Stage, reason: Reason) {+ report(stage, reason, None, None);+}++fn report(stage: Stage, reason: Reason, field: Option<Field>, position: Option<(u64, u64)>) {if let Some(observation) = crate::server::core_runtime::observation::active_request() {- observation.response_failure(stage, reason);+ observation.response_failure_detail(stage, reason, field, position);}}@@ -64,31 +69,87 @@ fn model_error_code(error: &ModelError, response_status: Option<u16>) -> WirePro}pub(super) fn adapter(stage: Stage, error: ProtocolAdapterError) -> Arc<str> {- note(stage, category(&error));+ adapter_at(stage, error, None)+}++pub(super) fn adapter_at(+ stage: Stage,+ error: ProtocolAdapterError,+ position: Option<(u64, u64)>,+) -> Arc<str> {+ let (reason, field) = category(&error);+ report(stage, reason, field, position);Arc::from("provider response processing failed")}-fn category(error: &ProtocolAdapterError) -> Reason {- match error {+pub(crate) fn category(error: &ProtocolAdapterError) -> (Reason, Option<Field>) {+ let mut field = None;+ let reason = match error {ProtocolAdapterError::ModelIr(error) => match error {ModelIrError::BufferLimit(_) => Reason::ResourceLimit,ModelIrError::InvalidJson(_) => Reason::InvalidJson,ModelIrError::InvalidSse(_) => Reason::InvalidSse,- ModelIrError::ExpectedObject | ModelIrError::InvalidField(_) => Reason::InvalidField,+ ModelIrError::ExpectedObject => {+ field = Some(Field::Object);+ Reason::InvalidField+ }+ ModelIrError::InvalidField(name) => {+ field = Some(known_field(name));+ Reason::InvalidField+ }+ ModelIrError::UnsupportedField(name) => {+ field = Some(known_field(name));+ Reason::UnsupportedField+ }+ ModelIrError::UnsupportedValue(_) => Reason::UnsupportedValue,+ ModelIrError::InvalidToolArguments(_) => Reason::InvalidToolArguments,+ ModelIrError::MissingToolIdentity(_) => Reason::MissingToolIdentity,+ ModelIrError::ProviderStateNotPortable => Reason::ProviderStateNotPortable,+ ModelIrError::ToolIdBindingRequired(_) => Reason::ToolIdBindingRequired,+ ModelIrError::ToolContinuationConflict => Reason::ToolContinuationConflict,ModelIrError::InvalidResponseLifecycle(_) => Reason::InvalidLifecycle,ModelIrError::MissingTerminalEvent => Reason::MissingTerminal,ModelIrError::DuplicateTerminalEvent => Reason::DuplicateTerminal,- _ => Reason::OtherProtocol,+ ModelIrError::ResponsesPreviousResponseIdUnsupported+ | ModelIrError::ResponsesConversationUnsupported => Reason::UnsupportedValue,},ProtocolAdapterError::ClientUnrepresentable(_) => Reason::Unrepresentable,- _ => Reason::OtherProtocol,+ ProtocolAdapterError::Capability(_) => Reason::Capability,+ ProtocolAdapterError::Context(_) => Reason::ContextProjection,+ ProtocolAdapterError::Serialization(_) => Reason::Serialization,+ };+ (reason, field)+}++// Exact allowlist only: never parse an error message or echo an unknown field path.+fn known_field(name: &str) -> Field {+ match name {+ "choices" => Field::Choices,+ "delta" => Field::Delta,+ "content" => Field::Content,+ "tool_calls" => Field::ToolCalls,+ "tool id" | "tool_call.id" => Field::ToolId,+ "tool name" | "tool_call.function.name" => Field::ToolName,+ "arguments" | "tool_call.function.arguments" => Field::ToolArguments,+ "finish_reason" => Field::FinishReason,+ "usage" => Field::Usage,+ "system_fingerprint" => Field::SystemFingerprint,+ "service_tier" => Field::ServiceTier,+ "logprobs" => Field::Logprobs,+ "model" => Field::Model,+ "id" => Field::Id,+ "type" => Field::Type,+ "index" => Field::Index,+ "role" => Field::Role,+ "output" => Field::Output,+ "status" => Field::Status,+ _ => Field::Unknown,}}#[cfg(test)]mod tests {use super::*;-#[test]fn model_errors_keep_safe_attribution_without_provider_text() {let known = ModelError { status: Some(400), code: Some("InvalidParameter".into()), message: Some("<400> InternalError.Algo.InvalidParameter: The thinking_budget parameter must be a positive integer and not greater than 81920".into()), retryable: None };@@ -115,26 +176,52 @@ mod tests {}#[test]- fn diagnostic_categories_distinguish_local_limits_and_protocol_failures_without_payloads() {+ fn diagnostic_categories_are_closed_and_never_serialize_dynamic_messages() {for (error, expected) in [(ModelIrError::BufferLimit(100), Reason::ResourceLimit),(- ModelIrError::InvalidJson("secret provider body".into()),+ ModelIrError::InvalidJson("SECRET_BODY".into()),Reason::InvalidJson,),(- ModelIrError::InvalidSse("secret event".into()),+ ModelIrError::InvalidSse("SECRET_BODY".into()),Reason::InvalidSse,),+ (+ ModelIrError::UnsupportedField("SECRET_BODY".into()),+ Reason::UnsupportedField,+ ),+ (+ ModelIrError::UnsupportedValue("SECRET_BODY".into()),+ Reason::UnsupportedValue,+ ),+ (+ ModelIrError::MissingToolIdentity("SECRET_TOOL".into()),+ Reason::MissingToolIdentity,+ ),+ (+ ModelIrError::InvalidToolArguments("SECRET_KEY".into()),+ Reason::InvalidToolArguments,+ ),+ (+ ModelIrError::ProviderStateNotPortable,+ Reason::ProviderStateNotPortable,+ ),+ (+ ModelIrError::InvalidResponseLifecycle("SECRET_BODY".into()),+ Reason::InvalidLifecycle,+ ),(ModelIrError::MissingTerminalEvent, Reason::MissingTerminal),(ModelIrError::DuplicateTerminalEvent,Reason::DuplicateTerminal,),] {- let category = category(&error.into());- assert_eq!(category, expected);- assert!(!serde_json::to_string(&category).unwrap().contains("secret"));+ let result = category(&error.into());+ assert_eq!(result.0, expected);+ assert!(!serde_json::to_string(&result).unwrap().contains("SECRET"));}+ assert_eq!(known_field("system_fingerprint"), Field::SystemFingerprint);+ assert_eq!(known_field("tool_calls.SECRET_TOOL"), Field::Unknown);}}diff --git a/crates/gateway/src/runtime/driver/response_prefix.rs b/crates/gateway/src/runtime/driver/response_prefix.rsindex 3d059642..594e7973 100644--- a/crates/gateway/src/runtime/driver/response_prefix.rs+++ b/crates/gateway/src/runtime/driver/response_prefix.rs@@ -92,9 +92,13 @@ fn decode_prefix(.renderer.as_mut().ok_or_else(|| Arc::from("client stream renderer is unavailable"))?;- let mut status = decoder- .feed(bytes, end_stream)- .map_err(|error| response_diagnostics::adapter(FailureStage::Decode, error))?;+ let mut status = decoder.feed(bytes, end_stream).map_err(|error| {+ response_diagnostics::adapter_at(+ FailureStage::Decode,+ error,+ Some(decoder.diagnostic_position()),+ )+ })?;let mut output = Vec::new();loop {for event in decoder.take_events() {@@ -120,9 +124,13 @@ fn decode_prefix(if status != adapters::ResponseDecodeStatus::NeedDrain {break;}- status = decoder- .resume()- .map_err(|_| Arc::from("stream prefix drain failed"))?;+ status = decoder.resume().map_err(|error| {+ response_diagnostics::adapter_at(+ FailureStage::Decode,+ error,+ Some(decoder.diagnostic_position()),+ )+ })?;}let terminal = status == adapters::ResponseDecodeStatus::Terminal;if terminal && !end_stream {@@ -133,7 +141,7 @@ fn decode_prefix(.take().expect("prefix decoder is present").finish()- .map_err(|_| Arc::from("stream prefix ended without terminal"))?;+ .map_err(|error| response_diagnostics::adapter(FailureStage::EndOfStream, error))?;}push_native_prefix_unit(state, output, terminal || end_stream)?;Ok(None)diff --git a/crates/gateway/src/runtime/driver/response_terminal_tests.rs b/crates/gateway/src/runtime/driver/response_terminal_tests.rsindex fd37b7c9..9d804e27 100644--- a/crates/gateway/src/runtime/driver/response_terminal_tests.rs+++ b/crates/gateway/src/runtime/driver/response_terminal_tests.rs@@ -29,6 +29,8 @@ fn accepted_codex_terminal_event_crosses_transport_frames_and_keeps_eos() {).unwrap();let mut readiness = ProductionReadiness {+ #[cfg(all(unix, debug_assertions))]+ capture: None,response_status: StatusCode::OK,content_type: "text/event-stream",prefix,@@ -133,6 +135,8 @@ fn accepted_chat_terminal_closes_responses_stream_without_waiting_for_transport_);let client = ClientProtocolProfile::for_candidate(&profile).unwrap();let mut readiness = ProductionReadiness {+ #[cfg(all(unix, debug_assertions))]+ capture: None,response_status: StatusCode::OK,content_type: "text/event-stream",prefix,diff --git a/crates/gateway/src/runtime/stream_capture.rs b/crates/gateway/src/runtime/stream_capture.rsnew file mode 100644index 00000000..f6b35968--- /dev/null+++ b/crates/gateway/src/runtime/stream_capture.rs@@ -0,0 +1,531 @@+//! Opt-in Unix Debug-only forensic capture. Never feeds the product diagnostic log.+//! A session owns a private directory and an exclusive capture lock. Failed/partial+//! capture is not replay evidence; it never changes the business result.+use std::fs::{self, File, OpenOptions};+use std::io::{self, Read, Write};+use std::os::unix::fs::{MetadataExt, OpenOptionsExt};+use std::path::{Path, PathBuf};+use std::sync::{Arc, Mutex, Weak};+use std::time::{SystemTime, UNIX_EPOCH};++use hiroute_gateway_core::runtime::attempt::{AttemptError, AttemptRequestBodyReader};+use hiroute_gateway_core::runtime::body::ChargedBytes;+use serde::{Deserialize, Serialize};++use crate::server::core_runtime::{adapters, profiles::CandidateProtocolProfile};++mod replay;+pub use replay::replay_capture;+#[cfg(test)]+mod tests;++const MAX_FILE: u64 = 8 * 1024 * 1024;+const MAX_TOTAL: u64 = 32 * 1024 * 1024;+const MAX_ATTEMPTS: usize = 4;+const MAX_RECORDS: u64 = 65536;++#[derive(Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+struct Session {+ source_sha: String,+ binary_sha256: String,+ client: String,+ expires_at: u64,+ delete_after: u64,+}++#[derive(Serialize, Deserialize)]+struct Context {+ session: Session,+ profile: CandidateProtocolProfile,+ chat_tools: Option<adapters::ChatToolProjection>,+ request_bytes: u64,+ streaming: bool,+ correlation: serde_json::Value,+ // Values of transport headers have no role in the response decoder.+ transport_header_values_redacted: bool,+}++#[derive(Clone)]+pub(crate) struct Capture(Arc<Mutex<Writer>>);++pub(crate) struct PendingCapture(Weak<Mutex<Writer>>);++#[derive(Serialize)]+pub(crate) struct CaptureCorrelation {+ pub(crate) request_token: Option<hiroute_diagnostics::identity::CorrelationToken>,+ pub(crate) request_id: String,+ pub(crate) attempt_index: u32,+ pub(crate) attempt_token: Option<hiroute_diagnostics::identity::CorrelationToken>,+}++impl PendingCapture {+ pub(crate) fn promoted(self, correlation: CaptureCorrelation) {+ if let Some(writer) = self.0.upgrade() {+ Capture(writer).correlate(&correlation, true);+ }+ }+}++struct Writer {+ file: File,+ path: PathBuf,+ _lock: File,+ root: PathBuf,+ written: u64,+ records: u64,+ limit: u64,+ expires_at: u64,+ active: bool,+ attempt_correlated: bool,+ failed: bool,+ failed_seal_offset: Option<u64>,+}++fn now() -> u64 {+ SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .unwrap_or_default()+ .as_secs()+}++fn private_file(path: &Path) -> io::Result<File> {+ OpenOptions::new()+ .write(true)+ .create_new(true)+ .mode(0o600)+ .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)+ .open(path)+}++fn read_private(path: &Path, limit: u64) -> io::Result<Vec<u8>> {+ let file = OpenOptions::new()+ .read(true)+ .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)+ .open(path)?;+ let metadata = file.metadata()?;+ if !metadata.is_file()+ || metadata.mode() & 0o777 != 0o600+ || metadata.uid() != rustix::process::geteuid().as_raw()+ || metadata.len() > limit+ {+ return Err(io::Error::other("unsafe or oversized capture file"));+ }+ let mut bytes = Vec::new();+ file.take(limit + 1).read_to_end(&mut bytes)?;+ if bytes.len() as u64 > limit {+ return Err(io::Error::other("capture file grew"));+ }+ Ok(bytes)+}++impl Capture {+ pub(crate) fn begin(+ profile: &CandidateProtocolProfile,+ chat_tools: Option<&adapters::ChatToolProjection>,+ request_bytes: usize,+ streaming: bool,+ ) -> Option<Self> {+ let root = PathBuf::from(std::env::var_os("HIROUTE_PRIVATE_STREAM_CAPTURE")?);+ Self::open(&root, profile, chat_tools, request_bytes, streaming).ok()+ }++ pub(crate) fn register(+ &self,+ request: &crate::server::core_runtime::observation::RequestObservation,+ ) {+ request.register_private_capture(PendingCapture(Arc::downgrade(&self.0)));+ }++ fn open(+ root: &Path,+ profile: &CandidateProtocolProfile,+ chat_tools: Option<&adapters::ChatToolProjection>,+ request_bytes: usize,+ streaming: bool,+ ) -> io::Result<Self> {+ // Canonical spelling rejects symlinks in every component, not just the leaf.+ let metadata = fs::symlink_metadata(root)?;+ if !root.is_absolute()+ || fs::canonicalize(root)? != root+ || !metadata.is_dir()+ || metadata.mode() & 0o777 != 0o700+ || metadata.uid() != rustix::process::geteuid().as_raw()+ {+ return Err(io::Error::other(+ "capture requires a private canonical directory",+ ));+ }+ let session: Session =+ serde_json::from_slice(&read_private(&root.join("session.json"), 4096)?)?;+ let time = now();+ if session.expires_at <= time+ || session.expires_at > time + 3600+ || session.delete_after <= session.expires_at+ || session.delete_after > time + 86400+ || !hex(&session.source_sha, 40)+ || !hex(&session.binary_sha256, 64)+ || session.client.is_empty()+ || session.client.len() > 128+ || root.join("stopped").exists()+ {+ return Err(io::Error::other(+ "invalid, expired or stopped capture session",+ ));+ }+ let lock = OpenOptions::new()+ .read(true)+ .write(true)+ .create(true)+ .truncate(false)+ .mode(0o600)+ .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)+ .open(root.join("active.lock"))?;+ let metadata = lock.metadata()?;+ if !metadata.is_file()+ || metadata.mode() & 0o777 != 0o600+ || metadata.uid() != rustix::process::geteuid().as_raw()+ {+ return Err(io::Error::other("unsafe capture lock"));+ }+ // Keep the inode stable. Kernel ownership disappears on crash/kill and+ // cleanup acquires this same lock before removing an expired session.+ rustix::fs::flock(&lock, rustix::fs::FlockOperation::NonBlockingLockExclusive)?;+ if root.join("stopped").exists() || now() >= session.expires_at {+ return Err(io::Error::other("capture session stopped or expired"));+ }+ Self::open_locked(+ root,+ lock,+ session,+ profile,+ chat_tools,+ request_bytes,+ streaming,+ )+ }++ fn open_locked(+ root: &Path,+ lock: File,+ session: Session,+ profile: &CandidateProtocolProfile,+ chat_tools: Option<&adapters::ChatToolProjection>,+ request_bytes: usize,+ streaming: bool,+ ) -> io::Result<Self> {+ let mut total = 0u64;+ let mut attempts = 0;+ for entry in fs::read_dir(root)? {+ let entry = entry?;+ let metadata = fs::symlink_metadata(entry.path())?;+ if !metadata.is_file() {+ return Err(io::Error::other("unexpected capture directory entry"));+ }+ total = total.saturating_add(metadata.len());+ if entry.path().extension().is_some_and(|e| e == "capture") {+ attempts += 1;+ }+ }+ if attempts >= MAX_ATTEMPTS || total >= MAX_TOTAL || request_bytes as u64 >= MAX_FILE {+ return Err(io::Error::other("capture budget exhausted"));+ }+ let mut profile = profile.clone();+ if let crate::server::core_runtime::profiles::CriticalFact::Exact(headers) =+ &mut profile.connector.headers+ {+ for (_, value) in &mut headers.required_headers {+ *value = "redacted".into();+ }+ }+ let context = Context {+ session,+ profile,+ chat_tools: chat_tools.cloned(),+ request_bytes: request_bytes as u64,+ streaming,+ correlation: correlation(),+ transport_header_values_redacted: true,+ };+ let path = root.join(format!("attempt-{}.capture", attempts + 1));+ let file = private_file(&path)?;+ let mut writer = Writer {+ file,+ path,+ _lock: lock,+ root: root.to_owned(),+ written: 0,+ records: 0,+ limit: MAX_FILE.min(MAX_TOTAL - total),+ expires_at: context.session.expires_at,+ active: true,+ attempt_correlated: false,+ failed: false,+ failed_seal_offset: None,+ };+ writer.write(0, &serde_json::to_vec(&context)?)?;+ Ok(Self(Arc::new(Mutex::new(writer))))+ }++ fn record(&self, kind: u8, bytes: &[u8]) {+ let mut writer = self.0.lock().unwrap_or_else(|p| p.into_inner());+ if writer.active && writer.write(kind, bytes).is_err() {+ writer.active = false;+ }+ }++ pub(crate) fn response(&self, event: &hiroute_gateway_core::runtime::attempt::PrecommitEvent) {+ use hiroute_gateway_core::runtime::attempt::PrecommitEvent;+ self.record_attempt_correlation();+ match event {+ PrecommitEvent::ResponseHead(head) => {+ self.record(3, &head.status().as_u16().to_le_bytes());+ }+ PrecommitEvent::Body(bytes) => self.record(4, bytes.bytes()),+ PrecommitEvent::EndStream => self.record(5, &[]),+ PrecommitEvent::SseEvent { .. } => {}+ }+ }++ pub(crate) fn record_attempt_correlation(&self) {+ // Snapshot observation before taking the writer lock. Formal promotion+ // supplies its own validated snapshot and never calls back into state.+ if let Some(request) = crate::server::core_runtime::observation::active_request() {+ self.correlate(&request.private_capture_correlation(), false);+ }+ }++ fn correlate(&self, correlation: &CaptureCorrelation, allow_sealed: bool) {+ let mut writer = self.0.lock().unwrap_or_else(|p| p.into_inner());+ if writer.attempt_correlated || correlation.attempt_token.is_none() {+ return;+ }+ let Ok(bytes) = serde_json::to_vec(correlation) else {+ return;+ };+ if writer.active {+ writer.attempt_correlated = true;+ if writer.write(7, &bytes).is_err() {+ writer.active = false;+ }+ } else if allow_sealed && let Some(offset) = writer.failed_seal_offset.take() {+ // Never overwrite the sample after publishing `stopped`: its owner+ // may terminate us at any instruction. Publish a complete replacement.+ writer.attempt_correlated = true;+ let _ = writer.correlate_sealed(offset, &bytes);+ }+ }++ pub(crate) fn failed(&self) {+ let mut writer = self.0.lock().unwrap_or_else(|p| p.into_inner());+ if writer.failed {+ return;+ }+ writer.failed = true;+ if writer.active && writer.write(6, &[]).is_err() {+ writer.active = false;+ }+ // The supervisor can terminate the process immediately after `stopped`.+ // Seal and flush this sample before publishing that signal, even while+ // the request reader or response driver still owns another Capture.+ writer.failed_seal_offset = writer.seal();+ let _ = private_file(&writer.root.join("stopped"));+ }++ pub(crate) fn wrap(+ &self,+ body: Box<dyn AttemptRequestBodyReader>,+ ) -> Box<dyn AttemptRequestBodyReader> {+ Box::new(CapturedBody {+ body,+ capture: self.clone(),+ ended: false,+ })+ }+}++fn correlation() -> serde_json::Value {+ crate::server::core_runtime::observation::active_request()+ .and_then(|r| serde_json::to_value(r.private_capture_correlation()).ok())+ .unwrap_or(serde_json::Value::Null)+}++fn hex(value: &str, len: usize) -> bool {+ value.len() == len && value.bytes().all(|c| c.is_ascii_hexdigit())+}++impl Writer {+ fn seal(&mut self) -> Option<u64> {+ // A missing seal is never replayable. Partial requests remain rejected+ // by the independent request-completeness check during replay.+ let seal_offset = self.written;+ let sealed = self.active && self.write(8, &[]).is_ok();+ self.active = false;+ let flushed = self.file.sync_all().is_ok();+ if !flushed && sealed {+ self.invalidate_seal(seal_offset);+ }+ if !flushed || !sealed {+ return None;+ }+ Some(seal_offset)+ }++ fn invalidate_seal(&self, offset: u64) {+ // A partial write or flush failure must not advertise sealed evidence.+ if self.file.set_len(offset).is_err() {+ let _ = fs::remove_file(&self.path);+ }+ }++ fn correlate_sealed(&mut self, offset: u64, bytes: &[u8]) -> io::Result<()> {+ self.check_correlation_tail(offset, bytes.len(), true)?;+ let replacement = CorrelationReplacement {+ file: private_file(&self.path.with_extension("correlation"))?,+ path: self.path.with_extension("correlation"),+ };+ let mut file = &replacement.file;+ let source = OpenOptions::new()+ .read(true)+ .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)+ .open(&self.path)?;+ if io::copy(&mut source.take(offset), &mut file)? != offset {+ return Err(io::Error::other("incomplete capture replacement"));+ }+ file.write_all(&[7])?;+ #[cfg(test)]+ tests::correlation_checkpoint(&self.root, "partial");+ file.write_all(&(bytes.len() as u64).to_le_bytes())?;+ file.write_all(bytes)?;+ file.write_all(&[8])?;+ file.write_all(&0u64.to_le_bytes())?;+ file.sync_all()?;+ // Recheck original identity and the now-present temporary copy's total.+ self.check_correlation_tail(offset, bytes.len(), false)?;+ let next_file = file.try_clone()?;+ #[cfg(test)]+ tests::correlation_checkpoint(&self.root, "ready");+ fs::rename(&replacement.path, &self.path)?;+ self.file = next_file;+ self.written = offset + 18 + bytes.len() as u64;+ self.records += 1;+ #[cfg(test)]+ tests::correlation_checkpoint(&self.root, "published");+ File::open(&self.root)?.sync_all()?;+ Ok(())+ }++ fn check_correlation_tail(+ &self,+ offset: u64,+ bytes: usize,+ reserve_copy: bool,+ ) -> io::Result<()> {+ let root = fs::symlink_metadata(&self.root)?;+ let file = self.file.metadata()?;+ let path = fs::symlink_metadata(&self.path)?;+ let uid = rustix::process::geteuid().as_raw();+ let new_size = offset.saturating_add(18).saturating_add(bytes as u64);+ let mut total = 0u64;+ for entry in fs::read_dir(&self.root)? {+ let metadata = fs::symlink_metadata(entry?.path())?;+ if !metadata.is_file() || metadata.uid() != uid {+ return Err(io::Error::other("unsafe capture directory entry"));+ }+ total = total.saturating_add(metadata.len());+ }+ if !root.is_dir()+ || root.mode() & 0o777 != 0o700+ || root.uid() != uid+ || fs::canonicalize(&self.root)? != self.root+ || !file.is_file()+ || file.mode() & 0o777 != 0o600+ || file.uid() != uid+ || !path.is_file()+ || path.ino() != file.ino()+ || path.dev() != file.dev()+ || file.len() != self.written+ || self.written != offset.saturating_add(9)+ || now() >= self.expires_at+ || new_size > self.limit+ || total.saturating_add(if reserve_copy { new_size } else { 0 }) > MAX_TOTAL+ || self.records.saturating_add(1) > MAX_RECORDS+ {+ return Err(io::Error::other(+ "unsafe or over-limit capture correlation tail",+ ));+ }+ Ok(())+ }++ // Binary records: kind:u8, length:u64 LE, exact bytes. File order is read order;+ // offsets are cumulative per direction. These are NOT TCP/TLS packet boundaries.+ fn write(&mut self, kind: u8, bytes: &[u8]) -> io::Result<()> {+ let size = 9u64.saturating_add(bytes.len() as u64);+ if now() >= self.expires_at+ || self.written.saturating_add(size) > self.limit+ || self.records >= MAX_RECORDS+ {+ self.active = false;+ return Err(io::Error::other("capture bound reached"));+ }+ self.file.write_all(&[kind])?;+ self.file.write_all(&(bytes.len() as u64).to_le_bytes())?;+ self.file.write_all(bytes)?;+ self.written += size;+ self.records += 1;+ Ok(())+ }+}+// A killed process can leave a bounded private temporary file. The existing+// session retention owner removes it together with the rest of the session.+struct CorrelationReplacement {+ file: File,+ path: PathBuf,+}+impl Drop for CorrelationReplacement {+ fn drop(&mut self) {+ let _ = fs::remove_file(&self.path);+ }+}++impl Drop for Writer {+ fn drop(&mut self) {+ let _ = self.seal();+ }+}++struct CapturedBody {+ body: Box<dyn AttemptRequestBodyReader>,+ capture: Capture,+ ended: bool,+}+impl std::fmt::Debug for CapturedBody {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.write_str("PrivateCapturedBody")+ }+}+impl AttemptRequestBodyReader for CapturedBody {+ fn visible_bytes(&self) -> usize {+ self.body.visible_bytes()+ }+ fn max_chunk_bytes(&self) -> usize {+ self.body.max_chunk_bytes()+ }+ fn high_water_bytes(&self) -> usize {+ self.body.high_water_bytes()+ }+ fn next_chunk(&mut self) -> Result<Option<ChargedBytes>, AttemptError> {+ let chunk = self.body.next_chunk()?;+ if let Some(bytes) = &chunk {+ self.capture.record(1, bytes.bytes());+ } else if !self.ended {+ self.capture.record(2, &[]);+ self.ended = true;+ }+ Ok(chunk)+ }+ fn release(&mut self) {+ self.body.release();+ }+}diff --git a/crates/gateway/src/runtime/stream_capture/replay.rs b/crates/gateway/src/runtime/stream_capture/replay.rsnew file mode 100644index 00000000..70325de2--- /dev/null+++ b/crates/gateway/src/runtime/stream_capture/replay.rs@@ -0,0 +1,142 @@+use super::*;+use crate::server::core_runtime::adapters::{+ NativeResponseDecoder, ProtocolAdapterError, ResponseDecodeStatus,+};+use hiroute_gateway_core::runtime::body::BudgetTree;++/// Offline only. `chunk_bytes == 0` preserves captured reader blocks; other values+/// rechunk the same response entity. Never claims to reproduce socket timing.+pub async fn replay_capture(+ path: &Path,+ chunk_bytes: usize,+) -> Result<serde_json::Value, &'static str> {+ let bytes = read_private(path, MAX_FILE).map_err(|_| "capture unavailable or unsafe")?;+ let records = records(&bytes)?;+ let context: Context = serde_json::from_slice(+ records+ .first()+ .filter(|r| r.0 == 0)+ .ok_or("missing context")?+ .1,+ )+ .map_err(|_| "invalid context")?;+ if now() >= context.session.delete_after {+ return Err("capture retention expired");+ }+ if records.last().map(|r| r.0) != Some(8) {+ return Err("capture incomplete or over limit");+ }+ let request_bytes: usize = records.iter().filter(|r| r.0 == 1).map(|r| r.1.len()).sum();+ if request_bytes as u64 != context.request_bytes || !records.iter().any(|r| r.0 == 2) {+ return Err("request body capture incomplete");+ }+ let mut status = None;+ for (_, bytes) in records.iter().filter(|r| r.0 == 3) {+ let head = u16::from_le_bytes((*bytes).try_into().map_err(|_| "invalid status")?);+ if !(100..600).contains(&head) {+ return Err("invalid status");+ }+ if (100..200).contains(&head) {+ if status.is_some() {+ return Err("informational head after final status");+ }+ continue;+ }+ if status.replace(head).is_some() {+ return Err("multiple final statuses");+ }+ }+ let status = status.ok_or("missing final status")?;+ // Native successful delivery uses NativeResponseProjector, while retryable+ // HTTP failures can bypass decoding entirely. Do not substitute a decoder+ // verdict for a production path this offline reader cannot restore.+ if !(200..300).contains(&status)+ || context.profile.ingress_protocol == context.profile.capability.upstream_protocol+ {+ return Err("unsupported_capture_path");+ }+ let entity: Vec<u8> = records+ .iter()+ .filter(|r| r.0 == 4)+ .flat_map(|r| r.1.iter().copied())+ .collect();+ let chunks: Box<dyn Iterator<Item = &[u8]>> = if chunk_bytes == 0 {+ Box::new(records.iter().filter(|r| r.0 == 4).map(|r| r.1))+ } else {+ Box::new(entity.chunks(chunk_bytes))+ };+ let tree =+ BudgetTree::new(8 * 1024 * 1024, 8 * 1024 * 1024).map_err(|_| "budget unavailable")?;+ let budget = tree+ .stream(8 * 1024 * 1024)+ .map_err(|_| "budget unavailable")?;+ let delivery =+ adapters::ActiveResponseDelivery::new(context.profile.ingress_protocol, budget.clone());+ let result = adapters::with_active_response_delivery(delivery, async {+ let mut decoder = NativeResponseDecoder::new_for_attempt(+ &context.profile,+ status,+ context.streaming && (200..300).contains(&status),+ context.chat_tools,+ budget,+ )?;+ let mut event_count = 0u64;+ for chunk in chunks {+ drain(&mut decoder, chunk, false, &mut event_count)?;+ }+ drain(&mut decoder, &[], true, &mut event_count)?;+ let position = decoder.diagnostic_position();+ let decoded = decoder.finish()?;+ Ok::<_, ProtocolAdapterError>((event_count, position, decoded.response.tool_id_map.len()))+ })+ .await;+ let result = match result {+ Ok((events, (frame, received), tools)) => {+ serde_json::json!({"result":"accepted", "events":events, "frame_index":frame, "received_bytes":received, "tool_calls":tools})+ }+ Err(error) => {+ let (reason, field) = crate::runtime::driver::response_diagnostics::category(&error);+ serde_json::json!({"result":"rejected", "reason":reason, "field":field})+ }+ };+ Ok(+ serde_json::json!({"source_sha": context.session.source_sha, "request_bytes": request_bytes,+ "response_bytes": entity.len(), "chunk_bytes": chunk_bytes, "decoder": result,+ "captured_gateway_failure": records.iter().any(|r| r.0 == 6)}),+ )+}++fn drain(+ decoder: &mut NativeResponseDecoder,+ bytes: &[u8],+ eof: bool,+ events: &mut u64,+) -> Result<(), ProtocolAdapterError> {+ let mut status = decoder.feed(bytes, eof)?;+ loop {+ *events += decoder.take_events().len() as u64;+ if status != ResponseDecodeStatus::NeedDrain {+ break;+ }+ status = decoder.resume()?;+ }+ Ok(())+}++pub(super) fn records(mut bytes: &[u8]) -> Result<Vec<(u8, &[u8])>, &'static str> {+ let mut records = Vec::new();+ while !bytes.is_empty() {+ if bytes.len() < 9 || records.len() as u64 >= MAX_RECORDS {+ return Err("invalid record boundary");+ }+ let kind = bytes[0];+ let len = u64::from_le_bytes(bytes[1..9].try_into().map_err(|_| "invalid length")?);+ bytes = &bytes[9..];+ if kind > 8 || len > bytes.len() as u64 {+ return Err("invalid record");+ }+ records.push((kind, &bytes[..len as usize]));+ bytes = &bytes[len as usize..];+ }+ Ok(records)+}diff --git a/crates/gateway/src/runtime/stream_capture/tests.rs b/crates/gateway/src/runtime/stream_capture/tests.rsnew file mode 100644index 00000000..c640c6b9--- /dev/null+++ b/crates/gateway/src/runtime/stream_capture/tests.rs@@ -0,0 +1,482 @@+use super::*;+use crate::server::core_runtime::profiles::fixed_reasoning;+use crate::server::request_plan::IngressProtocol;+use std::os::unix::fs::PermissionsExt;+use std::time::{Duration, Instant};++fn setup() -> PathBuf {+ let mut random = [0u8; 16];+ getrandom::fill(&mut random).unwrap();+ let root = fs::canonicalize(std::env::temp_dir())+ .unwrap()+ .join(format!("hiroute-capture-{:x}", u128::from_le_bytes(random)));+ fs::create_dir(&root).unwrap();+ fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap();+ let session = Session {+ source_sha: "a".repeat(40),+ binary_sha256: "b".repeat(64),+ client: "fixture".into(),+ expires_at: now() + 60,+ delete_after: now() + 120,+ };+ private_file(&root.join("session.json"))+ .unwrap()+ .write_all(&serde_json::to_vec(&session).unwrap())+ .unwrap();+ root+}+fn profile() -> CandidateProtocolProfile {+ CandidateProtocolProfile::exact_portable_path(+ IngressProtocol::Messages,+ IngressProtocol::ChatCompletions,+ "physical",+ fixed_reasoning("fixed"),+ )+}++fn promoted_correlation(index: u32) -> CaptureCorrelation {+ CaptureCorrelation {+ request_token: Some(hiroute_diagnostics::identity::CorrelationToken::from_bytes(+ [1; 32],+ )),+ request_id: "private-request".into(),+ attempt_index: index,+ attempt_token: Some(hiroute_diagnostics::identity::CorrelationToken::from_bytes(+ [index as u8; 32],+ )),+ }+}++pub(super) fn correlation_checkpoint(root: &Path, stage: &str) {+ if std::env::var_os("HIROUTE_CAPTURE_INTERRUPT_ROOT").as_deref() == Some(root.as_os_str())+ && std::env::var("HIROUTE_CAPTURE_INTERRUPT_STAGE").as_deref() == Ok(stage)+ {+ private_file(&root.join("checkpoint"))+ .unwrap()+ .write_all(stage.as_bytes())+ .unwrap();+ loop {+ std::thread::park();+ }+ }+}++#[tokio::test]+async fn sealed_correlation_survives_process_termination() {+ const TEST: &str =+ "runtime::stream_capture::tests::sealed_correlation_survives_process_termination";+ if let Some(root) = std::env::var_os("HIROUTE_CAPTURE_INTERRUPT_ROOT") {+ let root = PathBuf::from(root);+ let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();+ capture.record(1, b"{}");+ capture.record(2, &[]);+ capture.record(3, &200u16.to_le_bytes());+ capture.record(4, b"data: {invalid\n\n");+ capture.failed();+ PendingCapture(Arc::downgrade(&capture.0)).promoted(promoted_correlation(1));+ panic!("missing interruption checkpoint");+ }+ for stage in ["partial", "ready", "published"] {+ let root = setup();+ let mut child = std::process::Command::new(std::env::current_exe().unwrap())+ .args(["--exact", TEST, "--nocapture"])+ .env("HIROUTE_CAPTURE_INTERRUPT_ROOT", &root)+ .env("HIROUTE_CAPTURE_INTERRUPT_STAGE", stage)+ .stdout(std::process::Stdio::null())+ .spawn()+ .unwrap();+ let deadline = Instant::now() + Duration::from_secs(10);+ while !root.join("checkpoint").exists() && Instant::now() < deadline {+ if child.try_wait().unwrap().is_some() {+ break;+ }+ std::thread::sleep(Duration::from_millis(5));+ }+ let reached = root.join("checkpoint").exists();+ let stopped = root.join("stopped").exists();+ // SIGKILL cannot execute a destructor or finish an in-progress write.+ let _ = child.kill();+ let status = child.wait().unwrap();+ assert!(+ reached && stopped,+ "{stage}: checkpoint not reached: {status}"+ );+ let path = root.join("attempt-1.capture");+ let bytes = read_private(&path, MAX_FILE).unwrap();+ let records = replay::records(&bytes).unwrap();+ assert_eq!(records.last().unwrap().0, 8, "{stage}");+ assert_eq!(records.iter().filter(|r| r.0 == 8).count(), 1, "{stage}");+ assert_eq!(+ records.iter().filter(|r| r.0 == 7).count(),+ usize::from(stage == "published"),+ "{stage}"+ );+ for chunk in [0, 1, 4096] {+ let replay = replay_capture(&path, chunk).await.unwrap();+ assert_eq!(replay["decoder"]["reason"], "invalid_json", "{stage}");+ assert_eq!(replay["captured_gateway_failure"], true, "{stage}");+ }+ fs::remove_dir_all(root).unwrap();+ }+}++#[test]+fn promoted_failure_appends_only_one_bounded_correlation_without_reopening_content() {+ let root = setup();+ let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();+ capture.record(1, b"{}");+ capture.record(2, &[]);+ capture.record(3, &200u16.to_le_bytes());+ capture.record(4, b"original response");+ capture.failed();+ let path = root.join("attempt-1.capture");+ let initial = fs::read(&path).unwrap();+ assert!(root.join("stopped").exists());+ assert_eq!(replay::records(&initial).unwrap().last().unwrap().0, 8);+ PendingCapture(Arc::downgrade(&capture.0)).promoted(promoted_correlation(1));+ let bound = fs::read(&path).unwrap();+ assert_eq!(&initial[..initial.len() - 9], &bound[..initial.len() - 9]);+ let records = replay::records(&bound).unwrap();+ assert_eq!(records.iter().filter(|r| r.0 == 7).count(), 1);+ assert_eq!(records.iter().filter(|r| r.0 == 8).count(), 1);+ assert_eq!(records.last().unwrap().0, 8);+ let correlation: serde_json::Value =+ serde_json::from_slice(records.iter().find(|r| r.0 == 7).unwrap().1).unwrap();+ assert_eq!(correlation["attempt_index"], 1);+ assert!(!capture.0.lock().unwrap().active);+ PendingCapture(Arc::downgrade(&capture.0)).promoted(promoted_correlation(2));+ capture.record(4, b"must not capture a later response");+ capture.failed();+ assert_eq!(fs::read(&path).unwrap(), bound);+ let weak = PendingCapture(Arc::downgrade(&capture.0));+ drop(capture);+ weak.promoted(promoted_correlation(3));+ assert_eq!(fs::read(&path).unwrap(), bound);+ fs::remove_dir_all(root).unwrap();+}++#[test]+fn sealed_correlation_respects_original_bounds_and_file_identity() {+ for guard in [+ "file_limit",+ "session_limit",+ "temporary_copy_limit",+ "records",+ "expiry",+ "permissions",+ "identity",+ ] {+ let root = setup();+ let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();+ capture.record(1, b"{}");+ capture.record(2, &[]);+ capture.failed();+ let path = root.join("attempt-1.capture");+ let initial = fs::read(&path).unwrap();+ match guard {+ "file_limit" => {+ let mut w = capture.0.lock().unwrap();+ w.limit = w.written;+ }+ "session_limit" => {+ private_file(&root.join("other.capture"))+ .unwrap()+ .set_len(MAX_TOTAL - initial.len() as u64)+ .unwrap();+ }+ "temporary_copy_limit" => {+ // The final delta fits, but both complete copies cannot coexist.+ let correlation_bytes = serde_json::to_vec(&promoted_correlation(1)).unwrap().len();+ let existing_bytes: u64 = fs::read_dir(&root)+ .unwrap()+ .map(|entry| entry.unwrap().metadata().unwrap().len())+ .sum();+ private_file(&root.join("other.capture"))+ .unwrap()+ .set_len(MAX_TOTAL - existing_bytes - correlation_bytes as u64 - 9)+ .unwrap();+ }+ "records" => capture.0.lock().unwrap().records = MAX_RECORDS,+ "expiry" => capture.0.lock().unwrap().expires_at = now() - 1,+ "permissions" => fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(),+ "identity" => {+ fs::rename(&path, root.join("original.capture")).unwrap();+ private_file(&path).unwrap().write_all(&initial).unwrap();+ }+ _ => unreachable!(),+ }+ PendingCapture(Arc::downgrade(&capture.0)).promoted(promoted_correlation(1));+ assert_eq!(fs::read(&path).unwrap(), initial, "{guard}");+ assert!(!capture.0.lock().unwrap().active, "{guard}");+ drop(capture);+ fs::remove_dir_all(root).unwrap();+ }+}++// Only after the test has dropped its previous writer. A concurrent process+// spawn can inherit the open-file description until CLOEXEC closes it. Wait+// for that actual kernel ownership to end; all other errors remain failures.+fn open_after_capture_release(root: &Path, mut on_blocked: impl FnMut()) -> Capture {+ let deadline = Instant::now() + Duration::from_secs(2);+ loop {+ match Capture::open(root, &profile(), None, 2, true) {+ Ok(capture) => return capture,+ Err(error)+ if error.kind() == io::ErrorKind::WouldBlock && Instant::now() < deadline =>+ {+ on_blocked();+ std::thread::sleep(Duration::from_millis(1));+ }+ Err(error) => panic!("capture ownership did not release: {error}"),+ }+ }+}++#[test]+fn completed_capture_waits_for_inherited_lock_descriptor_release() {+ let root = setup();+ let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();+ // A descriptor duplicate models the same open-file description inherited+ // across fork, even when the originating writer has already been dropped.+ let inherited = capture.0.lock().unwrap()._lock.try_clone().unwrap();+ drop(capture);+ assert_eq!(+ Capture::open(&root, &profile(), None, 2, true)+ .err()+ .unwrap()+ .kind(),+ io::ErrorKind::WouldBlock,+ "live descriptor ownership must still reject another writer"+ );+ let (release, observed) = std::sync::mpsc::channel();+ let holder = std::thread::spawn(move || {+ observed.recv_timeout(Duration::from_secs(2)).unwrap();+ std::thread::sleep(Duration::from_millis(50));+ drop(inherited);+ });+ let mut blocked = 0;+ let next = open_after_capture_release(&root, || {+ blocked += 1;+ if blocked == 1 {+ release.send(()).unwrap();+ }+ });+ holder.join().unwrap();+ assert!(+ blocked > 0,+ "re-admission must observe the retained kernel lock"+ );+ drop(next);+ fs::remove_dir_all(root).unwrap();+}++#[tokio::test]+async fn capture_preserves_bytes_and_replays_request_bound_tools_across_chunks() {+ let root = setup();+ let request = serde_json::json!({"model":"alias", "max_tokens":64,"messages":[{"role":"user","content":"hello"}], "tools":[{"name":"known","description":"tool", "input_schema":{"type":"object"}}]});+ let ir = adapters::decode_ingress_request(IngressProtocol::Messages, &request).unwrap();+ let tools = adapters::ChatToolProjection::for_request(&ir).unwrap();+ let body = serde_json::to_vec(&request).unwrap();+ let capture = Capture::open(&root, &profile(), Some(&tools), body.len(), true).unwrap();+ capture.record(1, &body[..9]);+ capture.record(1, &body[9..]);+ capture.record(2, &[]);+ capture.record(3, &103u16.to_le_bytes());+ capture.record(3, &200u16.to_le_bytes());+ let sse = b"data: {\"id\":\"resp\",\"model\":\"physical\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call\",\"type\":\"function\",\"function\":{\"name\":\"unknown_secret_tool\",\"arguments\":\"{}\"}}]},\"finish_reason\":null}]}\n\ndata: [DONE]\n\n";+ capture.record(4, &sse[..17]);+ capture.record(4, &sse[17..]);+ capture.failed();+ // The supervisor may stop the process as soon as this marker appears.+ // A failed sample must already be durable and sealed while other owners still exist.+ assert!(root.join("stopped").exists());+ assert!(root.join("active.lock").exists());+ let at_stop = read_private(&root.join("attempt-1.capture"), MAX_FILE).unwrap();+ assert_eq!(replay::records(&at_stop).unwrap().last().unwrap().0, 8);+ for size in [0, 1, 4096] {+ let replay = replay_capture(&root.join("attempt-1.capture"), size)+ .await+ .unwrap();+ assert_eq!(replay["decoder"]["result"], "rejected");+ assert_eq!(replay["decoder"]["reason"], "missing_tool_identity");+ }+ capture.record(4, b"cannot append after seal");+ drop(capture);+ let path = root.join("attempt-1.capture");+ assert_eq!(+ fs::metadata(&path).unwrap().permissions().mode() & 0o777,+ 0o600+ );+ let bytes = read_private(&path, MAX_FILE).unwrap();+ let records = replay::records(&bytes).unwrap();+ let actual: Vec<u8> = records+ .iter()+ .filter(|r| r.0 == 1)+ .flat_map(|r| r.1.iter().copied())+ .collect();+ assert_eq!(actual, body);+ let chunks: Vec<_> = records.iter().filter(|r| r.0 == 4).map(|r| r.1).collect();+ assert_eq!(chunks, vec![&sse[..17], &sse[17..]]);+ for size in [0, 1, 4096] {+ let result = replay_capture(&path, size).await.unwrap();+ assert_eq!(result["decoder"]["result"], "rejected");+ assert!(!result.to_string().contains("unknown_secret_tool"));+ }+ assert!(+ Capture::open(&root, &profile(), None, 2, true).is_err(),+ "first failure stops further captures"+ );+ fs::remove_dir_all(root).unwrap();+}++#[tokio::test]+async fn bounded_capture_is_fail_closed_without_mutating_business_input() {+ let root = setup();+ let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();+ assert!(+ Capture::open(&root, &profile(), None, 2, true).is_err(),+ "one writer per session"+ );+ {+ let mut writer = capture.0.lock().unwrap();+ writer.limit = writer.written + 10;+ }+ capture.record(1, b"{}");+ drop(capture);+ assert!(+ replay_capture(&root.join("attempt-1.capture"), 0)+ .await+ .is_err()+ );+ fs::set_permissions(root.join("session.json"), fs::Permissions::from_mode(0o644)).unwrap();+ assert!(Capture::open(&root, &profile(), None, 2, true).is_err());+ fs::remove_dir_all(root).unwrap();+}++#[test]+fn capture_redacts_transport_headers_and_enforces_expiry() {+ let root = setup();+ let mut profile = profile();+ if let crate::server::core_runtime::profiles::CriticalFact::Exact(headers) =+ &mut profile.connector.headers+ {+ headers+ .required_headers+ .push(("x-api-key".into(), "SECRET_CREDENTIAL".into()));+ }+ let capture = Capture::open(&root, &profile, None, 2, true).unwrap();+ {+ let mut writer = capture.0.lock().unwrap();+ writer.expires_at = now() - 1;+ }+ capture.record(1, b"{}");+ drop(capture);+ let bytes = read_private(&root.join("attempt-1.capture"), MAX_FILE).unwrap();+ assert!(!String::from_utf8_lossy(&bytes).contains("SECRET_CREDENTIAL"));+ assert_ne!(replay::records(&bytes).unwrap().last().unwrap().0, 8);+ fs::remove_dir_all(root).unwrap();+}++#[tokio::test]+async fn capture_limits_and_partial_request_cannot_produce_replay_evidence() {+ let root = setup();+ assert!(Capture::open(&root, &profile(), None, MAX_FILE as usize, true).is_err());+ assert!(root.join("active.lock").exists());+ let capture = open_after_capture_release(&root, || {});+ capture.record(1, b"{");+ capture.record(3, &200u16.to_le_bytes());+ capture.failed();+ drop(capture);+ assert_eq!(+ replay_capture(&root.join("attempt-1.capture"), 0)+ .await+ .unwrap_err(),+ "request body capture incomplete"+ );+ fs::remove_dir_all(root).unwrap();++ let root = setup();+ for _ in 0..MAX_ATTEMPTS {+ drop(open_after_capture_release(&root, || {}));+ }+ assert!(Capture::open(&root, &profile(), None, 2, true).is_err());+ assert!(root.join("active.lock").exists());+ fs::remove_dir_all(root).unwrap();++ let root = setup();+ private_file(&root.join("existing.capture"))+ .unwrap()+ .set_len(MAX_TOTAL)+ .unwrap();+ assert!(Capture::open(&root, &profile(), None, 2, true).is_err());+ assert!(root.join("active.lock").exists());+ fs::remove_dir_all(root).unwrap();++ let root = setup();+ let capture = Capture::open(&root, &profile(), None, 2, true).unwrap();+ capture.0.lock().unwrap().records = MAX_RECORDS;+ capture.record(1, b"{}");+ drop(capture);+ assert!(+ replay_capture(&root.join("attempt-1.capture"), 0)+ .await+ .is_err()+ );+ assert!(replay::records(&[4, 2, 0, 0, 0, 0, 0, 0, 0, 1]).is_err());+ fs::remove_dir_all(root).unwrap();+}++#[tokio::test]+async fn replay_uses_final_head_and_rejects_paths_without_decoder_equivalence() {+ let body = b"{}";+ let response = b"data: {\"id\":\"resp\",\"model\":\"physical\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"resp\",\"model\":\"physical\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";+ let mut mismatches = Vec::new();+ for native in [false, true] {+ for (heads, invalid) in [+ (&[200u16][..], None),+ (&[103u16, 200][..], None),+ (&[103, 200, 200][..], Some("multiple final statuses")),+ (&[429][..], Some("unsupported_capture_path")),+ (&[103, 429][..], Some("unsupported_capture_path")),+ (&[503][..], Some("unsupported_capture_path")),+ (&[103, 503][..], Some("unsupported_capture_path")),+ (&[103][..], Some("missing final status")),+ (+ &[200, 103][..],+ Some("informational head after final status"),+ ),+ ] {+ let root = setup();+ let mut profile = profile();+ if native {+ profile.ingress_protocol = profile.capability.upstream_protocol;+ }+ let capture = Capture::open(&root, &profile, None, body.len(), true).unwrap();+ capture.record(1, body);+ capture.record(2, &[]);+ for head in heads {+ capture.record(3, &head.to_le_bytes());+ }+ capture.record(4, &response[..23]);+ capture.record(4, &response[23..]);+ capture.record(5, &[]);+ drop(capture);+ for size in [0, 1, 4096] {+ let result = replay_capture(&root.join("attempt-1.capture"), size).await;+ let expected = invalid.or(native.then_some("unsupported_capture_path"));+ let matched = match (&result, expected) {+ (Err(actual), Some(expected)) => *actual == expected,+ (Ok(actual), None) => actual["decoder"]["result"] == "accepted",+ _ => false,+ };+ if !matched {+ mismatches.push(format!(+ "heads={heads:?} native={native} chunk={size} expected={expected:?} actual={result:?}"+ ));+ }+ }+ fs::remove_dir_all(root).unwrap();+ }+ }+ assert!(mismatches.is_empty(), "{}", mismatches.join("\n"));+}diff --git a/scripts/ci-shards.py b/scripts/ci-shards.pyindex 61a08410..ffd1d5b4 100755--- a/scripts/ci-shards.py+++ b/scripts/ci-shards.py@@ -27,6 +27,7 @@ INTEGRATION_SHARDS = {("hiroute-domain", "compute"),("hiroute-client-core", "transport"),("hiroute-diagnostics", "files_safety"),+ ("hiroute-diagnostics", "response_failure_compatibility"),("hiroute-diagnostics", "worker_probe_progress"),("hiroute-diagnostics", "writer_bounds"),("hiroute-cli", "contracts"),diff --git a/scripts/private-stream-capture.py b/scripts/private-stream-capture.pynew file mode 100644index 00000000..ade0e51a--- /dev/null+++ b/scripts/private-stream-capture.py@@ -0,0 +1,252 @@+#!/usr/bin/env python3+"""Run one isolated Unix Debug gateway/daemon with bounded private capture.++The executable must come from the supplied committed candidate's managed build.+This tool supplies provenance, NOT a build attestation. Never use a daily daemon.+No network requests or model calls are issued by this tool.+"""+import argparse+import fcntl+import hashlib+import json+import os+from pathlib import Path+import shutil+import signal+import stat+import subprocess+import time++GROUP_STOP_GRACE_SECONDS = 5+++def require_supervision_primitives():+ if not all(hasattr(os, name) for name in ("waitid", "P_PID", "WEXITED", "WNOHANG", "WNOWAIT")):+ raise ValueError("capture requires non-reaping Unix waitid (macOS: Python 3.13 or newer)")+++def candidate_exited(child):+ # Keep the exited leader waitable so its PID/session cannot be recycled+ # before stop has signalled every member of our process group.+ return os.waitid(os.P_PID, child.pid, os.WEXITED | os.WNOHANG | os.WNOWAIT) is not None+++def write_private(path, data):+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)+ with os.fdopen(fd, "wb") as stream:+ stream.write(data)+++def create_session(root, source_sha, binary, client, seconds):+ root.mkdir(mode=0o700)+ stamp = int(time.time())+ session = dict(source_sha=source_sha, binary_sha256=hashlib.sha256(binary.read_bytes()).hexdigest(),+ client=client, expires_at=stamp + seconds, delete_after=stamp + 86400)+ write_private(root / "session.json", json.dumps(session).encode())+ return session+++def stop(child):+ if child.returncode is not None:+ raise ValueError("candidate leader was reaped before process-group cleanup")+ if os.getpgid(child.pid) != child.pid or os.getsid(child.pid) != child.pid:+ raise ValueError("candidate does not own its isolated session/process group")+ # The waitable leader anchors this group ID even when it has already+ # exited. A descendant may still own capture files or ignore SIGTERM.+ try:+ os.killpg(child.pid, signal.SIGTERM)+ deadline = time.monotonic() + GROUP_STOP_GRACE_SECONDS+ while time.monotonic() < deadline:+ time.sleep(min(0.02, max(0, deadline - time.monotonic())))+ finally:+ # Graceful termination of the retention owner can interrupt the grace+ # wait. It must still stop descendants before attempting raw cleanup.+ try:+ os.killpg(child.pid, signal.SIGKILL)+ except ProcessLookupError:+ pass+ finally:+ child.wait(timeout=5)+++def supervise(args):+ require_supervision_primitives()+ if not 1 <= args.seconds <= 3600:+ raise ValueError("duration must be 1..3600 seconds")+ if not 1 <= args.attempts <= 3:+ raise ValueError("attempt budget must be 1..3")+ if not args.command:+ raise ValueError("supply the isolated candidate executable and its arguments")+ command = args.command[1:] if args.command[0] == "--" else args.command+ binary = Path(command[0]).resolve(strict=True)+ source = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()+ if source != args.source_sha or subprocess.check_output(["git", "status", "--porcelain"]):+ raise ValueError("run from the clean, exact committed source checkout")+ root = args.root.resolve()+ if root.exists():+ raise ValueError("capture root must be new")+ session = create_session(root, source, binary, args.client, args.seconds)+ retain_session(args, root, binary, command, source, session["delete_after"])+++def retain_session(args, root, binary, command, source, delete_after):+ require_supervision_primitives()+ # This bounded owner survives the CLI returning. It owns the candidate child,+ # stops/reaps it, then removes the raw session at expiry (or on graceful stop).+ # No shared daemon, system scheduler, or daily process is modified.+ read_fd, write_fd = os.pipe()+ owner = os.fork()+ if owner == 0:+ os.close(read_fd)+ os.setsid()+ with open(os.devnull, "wb") as null:+ os.dup2(null.fileno(), 1)+ os.dup2(null.fileno(), 2)+ def terminate(_signum, _frame):+ signal.signal(signal.SIGTERM, signal.SIG_IGN)+ signal.signal(signal.SIGINT, signal.SIG_IGN)+ raise SystemExit(1)+ signal.signal(signal.SIGTERM, terminate)+ signal.signal(signal.SIGINT, terminate)+ reports = os.fdopen(write_fd, "w")+ def report(value):+ value["retention_owner_pid"] = os.getpid()+ print(json.dumps(value), file=reports, flush=True)+ exit_code = 0+ # A backwards wall-clock adjustment must not extend retention.+ deadline = time.monotonic() + max(0, min(86400, delete_after - time.time()))+ try:+ supervise_candidate(args, root, binary, command, source, report)+ reports.close() # The CLI can return; this owner remains until deletion.+ while time.time() < delete_after and time.monotonic() < deadline:+ time.sleep(min(60, max(0, deadline - time.monotonic()),+ max(0, delete_after - time.time())))+ except BaseException:+ exit_code = 1+ finally:+ # Reporting is best effort once the CLI reader has gone. A failed+ # flush can fail again on close, but must not skip raw deletion.+ # Graceful stop has already reached candidate group cleanup; do not+ # let a second signal interrupt this mandatory cleanup section.+ signal.signal(signal.SIGTERM, signal.SIG_IGN)+ signal.signal(signal.SIGINT, signal.SIG_IGN)+ try:+ reports.close()+ except BaseException:+ exit_code = 1+ finally:+ try:+ # SIGKILL is asynchronous: after reaping the leader, a killed+ # descendant may still be releasing its kernel flock. This+ # bounded wait applies only after our group-stop path.+ cleanup(root, require_expired=False, lock_wait_seconds=5)+ except BaseException:+ exit_code = 1+ finally:+ os._exit(exit_code)+ os.close(write_fd)+ try:+ with os.fdopen(read_fd) as reports:+ for line in reports:+ print(line.rstrip(), flush=True)+ except BaseException:+ os.kill(owner, signal.SIGTERM)+ os.waitpid(owner, 0)+ raise+ return owner+++def supervise_candidate(args, root, binary, command, source, report):+ env = dict(os.environ, HIROUTE_PRIVATE_STREAM_CAPTURE=str(root))+ # Third-party stdout is not evidence; discard it instead of an unbounded log.+ child = None+ reason = "process_exit"+ try:+ with open(os.devnull, "wb") as log:+ child = subprocess.Popen([str(binary), *command[1:]], env=env, stdout=log,+ stderr=subprocess.STDOUT, start_new_session=True)+ deadline = time.monotonic() + args.seconds+ report({"capture_root": str(root), "source_sha": source,+ "pid": child.pid, "model_request_budget": args.attempts})+ while not candidate_exited(child):+ files = list(root.glob("*.capture"))+ if (root / "stopped").exists():+ reason = "first_gateway_failure"+ break+ if len(files) >= args.attempts and not capture_active(root):+ reason = "attempt_budget"+ break+ if time.monotonic() >= deadline:+ reason = "time_budget"+ break+ # Bound all files including private child logs, beyond writer's own budget.+ if sum(p.stat().st_size for p in root.iterdir() if p.is_file()) >= 32 * 1024 * 1024:+ reason = "byte_budget"+ break+ time.sleep(0.02)+ finally:+ if child is not None:+ stop(child)+ report({"stop_reason": reason, "process_exit": child.returncode if child else None,+ "scenario": "unassessed", "retention_seconds": 86400})+++def capture_lock(root):+ fd = os.open(root / "active.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)+ info = os.fstat(fd)+ if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 or info.st_uid != os.geteuid():+ os.close(fd)+ raise ValueError("unsafe capture lock")+ return os.fdopen(fd, "rb+")+++def capture_active(root):+ with capture_lock(root) as lock:+ try:+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)+ except BlockingIOError:+ return True+ return False+++def cleanup(root, require_expired=True, lock_wait_seconds=0):+ # Only this exact, expired, owner-only session. Never walk a general evidence tree.+ if root.is_symlink() or root.resolve() != root or root.stat().st_uid != os.geteuid() or stat.S_IMODE(root.stat().st_mode) != 0o700:+ raise ValueError("unsafe root")+ with capture_lock(root) as lock:+ deadline = time.monotonic() + max(0, lock_wait_seconds)+ while True:+ try:+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)+ break+ except BlockingIOError as error:+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ raise ValueError("capture still owned; stop its process first") from error+ time.sleep(min(0.02, remaining))+ session = json.loads((root / "session.json").read_text())+ if require_expired and int(time.time()) < session["delete_after"]:+ raise ValueError("session has not reached retention expiry")+ if any(p.is_symlink() or not p.is_file() for p in root.iterdir()):+ raise ValueError("unexpected session entry")+ shutil.rmtree(root)+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ sub = parser.add_subparsers(dest="action", required=True)+ run = sub.add_parser("run")+ run.add_argument("--root", type=Path, required=True)+ run.add_argument("--source-sha", required=True)+ run.add_argument("--client", required=True)+ run.add_argument("--seconds", type=int, default=300)+ run.add_argument("--attempts", type=int, default=3)+ run.add_argument("command", nargs=argparse.REMAINDER)+ gc = sub.add_parser("cleanup")+ gc.add_argument("root", type=Path)+ args = parser.parse_args()+ supervise(args) if args.action == "run" else cleanup(args.root)+++if __name__ == "__main__":+ main()diff --git a/scripts/test-private-stream-capture.py b/scripts/test-private-stream-capture.pynew file mode 100644index 00000000..e2a82a04--- /dev/null+++ b/scripts/test-private-stream-capture.py@@ -0,0 +1,336 @@+#!/usr/bin/env python3+import ctypes+import fcntl+import importlib.util+import json+import os+from pathlib import Path+import select+import signal+import stat+import subprocess+import sys+import time+from types import SimpleNamespace+import tempfile+import threading+import unittest+from unittest.mock import patch++spec = importlib.util.spec_from_file_location("capture", Path(__file__).with_name("private-stream-capture.py"))+capture = importlib.util.module_from_spec(spec)+spec.loader.exec_module(capture)+++class CaptureSessionTest(unittest.TestCase):+ def test_private_provenance_and_expired_session_cleanup(self):+ with tempfile.TemporaryDirectory() as parent:+ binary = Path(parent) / "binary"+ binary.write_bytes(b"fixture binary")+ root = Path(parent).resolve() / "private"+ with patch.object(capture.time, "time", return_value=1000):+ capture.create_session(root, "a" * 40, binary, "fixture", 60)+ self.assertEqual(stat.S_IMODE(root.stat().st_mode), 0o700)+ self.assertEqual(stat.S_IMODE((root / "session.json").stat().st_mode), 0o600)+ session = json.loads((root / "session.json").read_text())+ self.assertEqual(session["expires_at"], 1060)+ self.assertEqual(len(session["binary_sha256"]), 64)+ with patch.object(capture.time, "time", return_value=2000):+ with self.assertRaises(ValueError):+ capture.cleanup(root)+ with capture.capture_lock(root) as lock:+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)+ with patch.object(capture.time, "time", return_value=100000):+ with self.assertRaises(ValueError):+ capture.cleanup(root)+ # The stable inode remains; only live kernel ownership blocks cleanup.+ with patch.object(capture.time, "time", return_value=100000):+ capture.cleanup(root)+ self.assertFalse(root.exists())++ def test_abnormal_and_forced_exit_release_lock_for_expired_cleanup(self):+ for forced in [False, True]:+ with self.subTest(forced=forced), tempfile.TemporaryDirectory() as parent:+ root = Path(parent).resolve() / "private"+ binary = Path(sys.executable).resolve()+ capture.create_session(root, "a" * 40, binary, "fixture", 1)+ code = ("import os,fcntl,time,sys; from pathlib import Path; "+ "p=Path(os.environ['HIROUTE_PRIVATE_STREAM_CAPTURE']); "+ "f=os.open(p/'active.lock',os.O_CREAT|os.O_RDWR,0o600); "+ "fcntl.flock(f,fcntl.LOCK_EX); (p/'raw.capture').write_bytes(b'raw'); "+ "print('ready',flush=True); " ++ ("time.sleep(60)" if forced else "os._exit(3)"))+ with subprocess.Popen([str(binary), "-c", code], stdout=subprocess.PIPE,+ env=dict(os.environ, HIROUTE_PRIVATE_STREAM_CAPTURE=str(root)),+ text=True) as child:+ self.assertEqual(child.stdout.readline().strip(), "ready")+ if forced:+ child.kill()+ child.wait(timeout=5)+ self.assertFalse(capture.capture_active(root))+ with patch.object(capture.time, "time", return_value=time.time() + 86401):+ capture.cleanup(root)+ self.assertFalse(root.exists())++ def test_owner_cleanup_waits_for_delayed_kernel_lock_release(self):+ with tempfile.TemporaryDirectory() as parent:+ root = Path(parent).resolve() / "private"+ capture.create_session(root, "a" * 40, Path(sys.executable).resolve(), "fixture", 1)+ with capture.capture_lock(root) as lock:+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)+ timer = threading.Timer(0.1, lambda: fcntl.flock(lock, fcntl.LOCK_UN))+ timer.start()+ try:+ capture.cleanup(root, require_expired=False, lock_wait_seconds=1)+ finally:+ timer.join()+ self.assertFalse(root.exists(), "cleanup must wait for the stopped group's lock release")++ def test_owner_cleanup_wait_is_bounded_and_keeps_active_capture(self):+ with tempfile.TemporaryDirectory() as parent:+ root = Path(parent).resolve() / "private"+ capture.create_session(root, "a" * 40, Path(sys.executable).resolve(), "fixture", 1)+ with capture.capture_lock(root) as lock:+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)+ started = time.monotonic()+ with self.assertRaisesRegex(ValueError, "capture still owned"):+ capture.cleanup(root, require_expired=False, lock_wait_seconds=0.1)+ self.assertGreaterEqual(time.monotonic() - started, 0.1)+ self.assertLess(time.monotonic() - started, 2)+ self.assertTrue(root.exists())+ self.assertTrue((root / "session.json").is_file())++ def test_retention_owner_deletes_after_cli_returns_or_graceful_stop(self):+ for early_stop in [False, True]:+ with self.subTest(early_stop=early_stop), tempfile.TemporaryDirectory() as parent:+ root = Path(parent).resolve() / "private"+ binary = Path(sys.executable).resolve()+ capture.create_session(root, "a" * 40, binary, "fixture", 1)+ code = ("import os,fcntl; from pathlib import Path; "+ "p=Path(os.environ['HIROUTE_PRIVATE_STREAM_CAPTURE']); "+ "f=os.open(p/'active.lock',os.O_CREAT|os.O_RDWR,0o600); "+ "fcntl.flock(f,fcntl.LOCK_EX); (p/'raw.capture').write_bytes(b'raw'); "+ "os._exit(3)")+ args = SimpleNamespace(seconds=1, attempts=1)+ with patch.object(capture, "GROUP_STOP_GRACE_SECONDS", 0.1, create=True):+ owner = capture.retain_session(args, root, binary, [str(binary), "-c", code],+ "a" * 40, time.time() + (60 if early_stop else 1))+ try:+ self.assertTrue((root / "raw.capture").exists())+ if early_stop:+ os.kill(owner, signal.SIGTERM)+ deadline = time.monotonic() + 5+ while root.exists() and time.monotonic() < deadline:+ time.sleep(0.02)+ self.assertFalse(root.exists(), "raw retention must be executed without another CLI call")+ finally:+ if root.exists():+ os.kill(owner, signal.SIGTERM)+ os.waitpid(owner, 0)++ @unittest.skipUnless(sys.platform.startswith("linux"), "controlled Linux child subreaper")+ def test_expired_retention_stops_descendant_after_leader_exits(self):+ self.check_descendant_cleanup(leader_exits=True)++ @unittest.skipUnless(sys.platform.startswith("linux"), "controlled Linux child subreaper")+ def test_expired_retention_stops_descendant_after_leader_exits_on_term(self):+ self.check_descendant_cleanup(leader_exits=False)++ @unittest.skipUnless(sys.platform.startswith("linux"), "controlled Linux child subreaper")+ def test_owner_term_during_group_stop_still_kills_descendant(self):+ self.check_descendant_cleanup(leader_exits=False, owner_term_during_stop=True)++ @unittest.skipUnless(sys.platform.startswith("linux"), "controlled Linux child subreaper")+ def test_reader_exit_cannot_skip_owner_cleanup_after_broken_report_and_close(self):+ # Terminate the real CLI reader after the first report. Its owner later+ # reports a natural candidate exit to the now-closed pipe, then closes+ # that same buffered stream. Both failures must still reach deletion.+ libc = ctypes.CDLL(None, use_errno=True)+ previous = ctypes.c_int()+ self.assertEqual(libc.prctl(37, ctypes.byref(previous), 0, 0, 0), 0)+ self.assertEqual(libc.prctl(36, 1, 0, 0, 0), 0)+ cli = owner = candidate = None+ try:+ with tempfile.TemporaryDirectory() as parent:+ root = Path(parent).resolve() / "private"+ binary = Path(sys.executable).resolve()+ capture.create_session(root, "a" * 40, binary, "fixture", 10)+ delete_after = int(time.time()) + 2+ session_path = root / "session.json"+ session = json.loads(session_path.read_text())+ session["delete_after"] = delete_after+ session_path.write_text(json.dumps(session))+ candidate_code = '''import fcntl, os, time+from pathlib import Path+root = Path(os.environ["HIROUTE_PRIVATE_STREAM_CAPTURE"])+fd = os.open(root / "active.lock", os.O_CREAT | os.O_RDWR, 0o600)+fcntl.flock(fd, fcntl.LOCK_EX)+raw = os.open(root / "raw.capture", os.O_CREAT | os.O_WRONLY, 0o600)+os.write(raw, b"synthetic private body")+os.close(raw)+(root.parent / "ready").touch()+while not (root.parent / "finish-candidate").exists():+ time.sleep(0.01)+os._exit(0)+'''+ cli_code = '''import importlib.util, sys+from pathlib import Path+from types import SimpleNamespace+spec = importlib.util.spec_from_file_location("capture", sys.argv[1])+capture = importlib.util.module_from_spec(spec)+spec.loader.exec_module(capture)+capture.GROUP_STOP_GRACE_SECONDS = 0.1+binary = Path(sys.executable).resolve()+capture.retain_session(SimpleNamespace(seconds=10, attempts=1), Path(sys.argv[2]),+ binary, [str(binary), "-c", sys.argv[4]], "a" * 40, float(sys.argv[3]))+'''+ cli = subprocess.Popen(+ [str(binary), "-c", cli_code, capture.__file__, str(root),+ str(delete_after), candidate_code], stdout=subprocess.PIPE, text=True,+ )+ self.assertTrue(select.select([cli.stdout], [], [], 5)[0], "CLI did not report its owner")+ first = json.loads(cli.stdout.readline())+ owner, candidate = first["retention_owner_pid"], first["pid"]+ deadline = time.monotonic() + 5+ while not (root.parent / "ready").exists() and time.monotonic() < deadline:+ time.sleep(0.01)+ self.assertTrue((root / "raw.capture").is_file())+ cli.terminate()+ self.assertEqual(cli.wait(timeout=5), -signal.SIGTERM)+ cli.stdout.close()+ cli = None+ (root.parent / "finish-candidate").touch()+ deadline = time.monotonic() + 5+ while root.exists() and time.monotonic() < deadline:+ time.sleep(0.02)+ self.assertFalse(root.exists(), "closed CLI report pipe skipped mandatory raw cleanup")+ status = self.wait_for_owned_child(owner)+ owner = None+ self.assertTrue(os.WIFEXITED(status))+ self.assertEqual(os.WEXITSTATUS(status), 1, "report failure must remain a failed owner exit")+ finally:+ if cli is not None:+ if cli.poll() is None:+ cli.kill()+ cli.wait(timeout=5)+ cli.stdout.close()+ # waitpid proves these are our adopted children before signaling;+ # no process scan or potentially recycled external PID is used.+ for pid in [owner, candidate]:+ if pid is not None:+ try:+ completed, _ = os.waitpid(pid, os.WNOHANG)+ if completed == 0:+ os.kill(pid, signal.SIGKILL)+ os.waitpid(pid, 0)+ except ChildProcessError:+ pass+ self.assertEqual(libc.prctl(36, previous.value, 0, 0, 0), 0)++ def check_descendant_cleanup(self, leader_exits, owner_term_during_stop=False):+ # Adopt only this test's orphan, so its SIGKILL status and reaping can be+ # asserted without leaving a zombie behind or trusting a recycled PID.+ libc = ctypes.CDLL(None, use_errno=True)+ previous = ctypes.c_int()+ self.assertEqual(libc.prctl(37, ctypes.byref(previous), 0, 0, 0), 0)+ self.assertEqual(libc.prctl(36, 1, 0, 0, 0), 0)+ owner = descendant = None+ try:+ with tempfile.TemporaryDirectory() as parent:+ root = Path(parent).resolve() / "private"+ binary = Path(sys.executable).resolve()+ capture.create_session(root, "a" * 40, binary, "fixture", 1)+ delete_after = int(time.time()) + 2+ session_path = root / "session.json"+ session = json.loads(session_path.read_text())+ session["delete_after"] = delete_after+ session_path.write_text(json.dumps(session))+ code = '''import fcntl, os, signal, sys, time+from pathlib import Path+root = Path(os.environ["HIROUTE_PRIVATE_STREAM_CAPTURE"])+ready = root.parent / "descendant.pid"+if os.fork() == 0:+ signal.signal(signal.SIGTERM, signal.SIG_IGN)+ fd = os.open(root / "active.lock", os.O_CREAT | os.O_RDWR, 0o600)+ fcntl.flock(fd, fcntl.LOCK_EX)+ (root / "raw.capture").write_bytes(b"synthetic private body")+ ready.write_text(str(os.getpid()))+ while True:+ time.sleep(1)+while not ready.exists():+ time.sleep(0.01)+if sys.argv[1] == "exit":+ os._exit(0)+def terminate(*_):+ if sys.argv[1] == "owner_term":+ os.kill(os.getppid(), signal.SIGTERM)+ os._exit(0)+signal.signal(signal.SIGTERM, terminate)+while True:+ time.sleep(1)+'''+ args = SimpleNamespace(seconds=1, attempts=1)+ mode = "owner_term" if owner_term_during_stop else "exit" if leader_exits else "term"+ with patch.object(capture, "GROUP_STOP_GRACE_SECONDS", 0.1, create=True):+ owner = capture.retain_session(+ args, root, binary,+ [str(binary), "-c", code, mode],+ "a" * 40, delete_after,+ )+ descendant = int((root.parent / "descendant.pid").read_text())+ deadline = time.monotonic() + 5+ while root.exists() and time.monotonic() < deadline:+ time.sleep(0.02)+ self.assertFalse(root.exists(), "raw retention survives the candidate's exited leader")+ status = self.wait_for_owned_child(descendant)+ descendant = None+ self.assertTrue(os.WIFSIGNALED(status))+ self.assertEqual(os.WTERMSIG(status), signal.SIGKILL)+ status = self.wait_for_owned_child(owner)+ owner = None+ self.assertTrue(os.WIFEXITED(status))+ self.assertEqual(os.WEXITSTATUS(status), 1 if owner_term_during_stop else 0)+ finally:+ for pid in [descendant, owner]:+ if pid is not None:+ try:+ completed, _ = os.waitpid(pid, os.WNOHANG)+ if completed == 0:+ os.kill(pid, signal.SIGKILL)+ os.waitpid(pid, 0)+ except ChildProcessError:+ pass+ self.assertEqual(libc.prctl(36, previous.value, 0, 0, 0), 0)++ def wait_for_owned_child(self, pid):+ deadline = time.monotonic() + 5+ while time.monotonic() < deadline:+ completed, status = os.waitpid(pid, os.WNOHANG)+ if completed:+ return status+ time.sleep(0.02)+ self.fail("owned child did not terminate")++ def test_stop_rejects_a_child_in_the_callers_process_group(self):+ child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"])+ try:+ with self.assertRaisesRegex(ValueError, "isolated session/process group"):+ capture.stop(child)+ self.assertIsNone(child.poll())+ finally:+ child.kill()+ child.wait(timeout=5)++ def test_private_writer_does_not_follow_or_overwrite(self):+ with tempfile.TemporaryDirectory() as parent:+ target = Path(parent) / "target"+ target.write_bytes(b"untouched")+ link = Path(parent) / "link"+ link.symlink_to(target)+ with self.assertRaises(FileExistsError):+ capture.write_private(link, b"overwrite")+ self.assertEqual(target.read_bytes(), b"untouched")+++if __name__ == "__main__":+ unittest.main()