Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

fix: remove proxy protocol size limits (#12)

澄潭09f490c6879e
diff --git a/contracts/compatibility-support.v1.json b/contracts/compatibility-support.v1.json
index e0e168ad..cf490a3e 100644
--- a/contracts/compatibility-support.v1.json
+++ b/contracts/compatibility-support.v1.json
@@ -337,6 +337,29 @@
"hiroute.worker-wait-request/v1"
],
"subjects": [
+ {
+ "subject": "terminal_classification_retired_quotas",
+ "audited_prefixes": ["hiroute.candidate-capability"],
+ "current_contracts": ["hiroute.candidate-capability/v1"],
+ "legacy_support": [
+ {
+ "contract": "hiroute.candidate-capability/v1",
+ "mode": "recovery_read_only",
+ "owner": "gateway-protocol",
+ "reason": "Recover the pre-20260922 V1 terminal_classified shape in persisted publications and operation journals. Preserve max_buffered_bytes and max_buffered_blocks only for historical canonical digest validation; execution ignores both. Current producers construct the quota-free variant. The same V1 identifier also occurs in the listed current producer and consumer paths.",
+ "removal_condition": "Remove when active publications, operation journals, LKG snapshots and supported backups contain no terminal_classified descriptors with either retired quota field.",
+ "allowed_paths": [
+ "crates/application/src/compiler/test_fixtures.rs",
+ "crates/daemon/src/control/runtime/candidate_protocol_profiles.rs",
+ "crates/domain/src/routing/gateway_execution.rs",
+ "crates/gateway/src/profiles/capability.rs",
+ "crates/gateway/src/planning/eligibility.rs",
+ "crates/gateway/src/runtime/driver/response.rs",
+ "e2e/product/golden/routing/compiled-publication.v2.json"
+ ]
+ }
+ ]
+ },
{
"subject": "gateway_e2e_build_attestation",
"audited_prefixes": [
diff --git a/crates/cli/tests/work_plans.rs b/crates/cli/tests/work_plans.rs
index 3bc53959..3b2901f7 100644
--- a/crates/cli/tests/work_plans.rs
+++ b/crates/cli/tests/work_plans.rs
@@ -165,6 +165,8 @@ fn work_plans_production_cli_current_authority_and_metadata() {
.env(CHILD, "1")
.env("HOME", home.path())
.env("PATH", "/usr/bin:/bin")
+ .env_remove("CODEX_HOME")
+ .env_remove("CLAUDE_CONFIG_DIR")
.env_remove("ANTHROPIC_AUTH_TOKEN")
.env_remove("ANTHROPIC_BASE_URL")
.status()
@@ -172,7 +174,11 @@ fn work_plans_production_cli_current_authority_and_metadata() {
assert!(status.success());
return;
}
- let root = tempfile::tempdir().unwrap();
+ use std::os::unix::fs::PermissionsExt;
+ let root = tempfile::Builder::new()
+ .permissions(std::fs::Permissions::from_mode(0o700))
+ .tempdir()
+ .unwrap();
let runtime =
ProductionControlRuntime::open_with_release_catalog(root.path().join("storage"), catalog())
.unwrap();
diff --git a/crates/cli/tests/worker_dependencies.rs b/crates/cli/tests/worker_dependencies.rs
index 1330de2d..596403dc 100644
--- a/crates/cli/tests/worker_dependencies.rs
+++ b/crates/cli/tests/worker_dependencies.rs
@@ -83,6 +83,8 @@ fn public_cli_discover_select_and_same_request_replay_are_self_contained() {
.env(CHILD, "1")
.env("HOME", home.path())
.env("PATH", &bin)
+ .env_remove("CODEX_HOME")
+ .env_remove("CLAUDE_CONFIG_DIR")
.env_remove("NPM_CONFIG_PREFIX")
.env_remove("npm_config_prefix")
.env_remove("NPM_CONFIG_CACHE")
diff --git a/crates/daemon/src/control/runtime/candidate_protocol_profiles.rs b/crates/daemon/src/control/runtime/candidate_protocol_profiles.rs
index e5550310..610de5b0 100644
--- a/crates/daemon/src/control/runtime/candidate_protocol_profiles.rs
+++ b/crates/daemon/src/control/runtime/candidate_protocol_profiles.rs
@@ -14,8 +14,6 @@ use hiroute_domain::{
use super::{ProtocolConnectorFacts, ProtocolFace, protocol_label};
const MAX_MATERIALIZED_BUDGET_PROFILES: u64 = 64;
-const MAX_TERMINAL_REFUSAL_BYTES: u64 = 16 * 1024 * 1024;
-const MAX_TERMINAL_REFUSAL_BLOCKS: u32 = 8;
pub(super) fn protocol_profiles(
connector: &ProtocolConnectorFacts,
@@ -184,10 +182,7 @@ pub(super) fn protocol_profiles(
GatewayStateAffinityV1::Unsupported
},
stream_refusal: if upstream_protocol == UpstreamProtocol::Messages {
- GatewayStreamingRefusalSemanticsV1::TerminalClassified {
- max_buffered_bytes: MAX_TERMINAL_REFUSAL_BYTES,
- max_buffered_blocks: MAX_TERMINAL_REFUSAL_BLOCKS,
- }
+ GatewayStreamingRefusalSemanticsV1::TerminalClassified
} else {
GatewayStreamingRefusalSemanticsV1::ExactDelta
},
diff --git a/crates/daemon/src/delegation/profile/mod.rs b/crates/daemon/src/delegation/profile/mod.rs
index 158c0749..17b7ae7c 100644
--- a/crates/daemon/src/delegation/profile/mod.rs
+++ b/crates/daemon/src/delegation/profile/mod.rs
@@ -130,7 +130,14 @@ impl CandidateWorkerProfile {
// This is a new child environment, never merged with std::env::vars().
// Tool discovery is not authentication. Keep an explicit command search
// path while leaving ambient model credentials and configuration behind.
- env.insert("PATH".into(), Zeroizing::new(worker_path(&input)?));
+ env.insert(
+ "PATH".into(),
+ Zeroizing::new(worker_path(
+ input.harness_binary,
+ input.node_binary,
+ input.adapter,
+ )?),
+ );
env.insert(
"HOME".into(),
Zeroizing::new(path_string(&private_root.join("home"))?),
@@ -347,16 +354,13 @@ fn path_string(path: &Path) -> Result<String, DelegationErrorV1> {
.ok_or(DelegationErrorV1::InvalidArguments)
}
-fn worker_path(input: &ProfileInput<'_>) -> Result<String, DelegationErrorV1> {
+fn worker_path(
+ harness: &Path,
+ node: Option<&Path>,
+ adapter: &Path,
+) -> Result<String, DelegationErrorV1> {
let mut paths = Vec::new();
- for executable in [
- Some(input.harness_binary),
- input.node_binary,
- Some(input.adapter),
- ]
- .into_iter()
- .flatten()
- {
+ for executable in [node, Some(harness), Some(adapter)].into_iter().flatten() {
if let Some(parent) = executable.parent() {
paths.push(parent.to_owned());
}
diff --git a/crates/daemon/src/delegation/profile/tests.rs b/crates/daemon/src/delegation/profile/tests.rs
index 3b58c39e..3191497b 100644
--- a/crates/daemon/src/delegation/profile/tests.rs
+++ b/crates/daemon/src/delegation/profile/tests.rs
@@ -1,5 +1,27 @@
use super::*;
+#[cfg(unix)]
+#[test]
+fn nested_cli_shebang_resolves_the_selected_node_runtime() {
+ use std::os::unix::fs::PermissionsExt;
+ let fixture = private_fixture();
+ let node = fixture.path().join("node");
+ std::fs::write(&node, b"#!/bin/sh\necho selected-worker-node\n").unwrap();
+ std::fs::set_permissions(&node, std::fs::Permissions::from_mode(0o700)).unwrap();
+ let harness = fixture.path().join("native-cli");
+ std::fs::write(&harness, b"#!/usr/bin/env node\n").unwrap();
+ std::fs::set_permissions(&harness, std::fs::Permissions::from_mode(0o700)).unwrap();
+ let path = worker_path(&harness, Some(&node), &harness).unwrap();
+ assert_eq!(std::env::split_paths(&path).next().unwrap(), fixture.path());
+ let result = std::process::Command::new(harness)
+ .env_clear()
+ .env("PATH", path)
+ .output()
+ .unwrap();
+ assert!(result.status.success());
+ assert_eq!(result.stdout, b"selected-worker-node\n");
+}
+
#[cfg(unix)]
#[test]
fn isolated_worker_profiles_can_resolve_standard_tools() {
diff --git a/crates/daemon/src/role_all.rs b/crates/daemon/src/role_all.rs
index 93eaeb9a..75baa659 100644
--- a/crates/daemon/src/role_all.rs
+++ b/crates/daemon/src/role_all.rs
@@ -399,9 +399,19 @@ fn start_role_all_with_runtime(
return Err(error);
}
};
- let trusted_hiroute = std::env::current_exe()
+ let bundled_hiroute = std::env::current_exe()
.map_err(|error| RoleAllError::Component("Local Control", error.to_string()))?
- .with_file_name("hiroute")
+ .with_file_name("hiroute");
+ let standalone_home = if config.released_commands_only {
+ Some(
+ hiroute_host_runtime::StandaloneLayout::from_environment()
+ .map_err(|_| RoleAllError::InvalidConfiguration)?
+ .home,
+ )
+ } else {
+ None
+ };
+ let trusted_hiroute = managed_cli_entry(&bundled_hiroute, standalone_home.as_deref())?
.into_os_string()
.into_string()
.map_err(|_| RoleAllError::InvalidConfiguration)?;
@@ -530,6 +540,25 @@ fn selected_codex_auth() -> Result<PathBuf, RoleAllError> {
Ok(root.join("auth.json"))
}
+fn managed_cli_entry(
+ bundled: &std::path::Path,
+ standalone_home: Option<&std::path::Path>,
+) -> Result<PathBuf, RoleAllError> {
+ let Some(home) = standalone_home else {
+ return Ok(bundled.to_owned());
+ };
+ // Persist the installer-owned entry, whose target changes on upgrade, only
+ // after proving it identifies the CLI shipped with this running daemon.
+ let stable = home.join(".local/bin/hiroute");
+ let target = std::fs::canonicalize(&stable).map_err(|_| RoleAllError::InvalidConfiguration)?;
+ let expected =
+ std::fs::canonicalize(bundled).map_err(|_| RoleAllError::InvalidConfiguration)?;
+ if target != expected {
+ return Err(RoleAllError::InvalidConfiguration);
+ }
+ Ok(stable)
+}
+
fn remaining(deadline: Instant, component: &'static str) -> Result<Duration, RoleAllError> {
deadline
.checked_duration_since(Instant::now())
@@ -578,6 +607,31 @@ mod tests {
use super::*;
+ #[test]
+ fn standalone_agent_helper_uses_verified_stable_entry_across_upgrade() {
+ let directory = tempfile::tempdir().unwrap();
+ let home = directory.path();
+ let stable = home.join(".local/bin/hiroute");
+ std::fs::create_dir_all(stable.parent().unwrap()).unwrap();
+ let first = home.join("v1/hiroute");
+ let second = home.join("v2/hiroute");
+ for binary in [&first, &second] {
+ std::fs::create_dir_all(binary.parent().unwrap()).unwrap();
+ std::fs::write(binary, b"binary").unwrap();
+ }
+ assert!(managed_cli_entry(&first, Some(home)).is_err());
+ std::os::unix::fs::symlink(&first, &stable).unwrap();
+ let persisted = managed_cli_entry(&first, Some(home)).unwrap();
+ assert_eq!(persisted, stable);
+ assert!(managed_cli_entry(&second, Some(home)).is_err());
+ std::fs::remove_file(&stable).unwrap();
+ std::os::unix::fs::symlink(&second, &stable).unwrap();
+ std::fs::remove_file(&first).unwrap();
+ assert_eq!(managed_cli_entry(&second, Some(home)).unwrap(), persisted);
+ assert_eq!(std::fs::canonicalize(persisted).unwrap(), second);
+ assert_eq!(managed_cli_entry(&second, None).unwrap(), second);
+ }
+
fn reserve_loopback() -> SocketAddr {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
diff --git a/crates/daemon/tests/pre_gateway_compute_routing.rs b/crates/daemon/tests/pre_gateway_compute_routing.rs
index d81c01a9..740e1986 100644
--- a/crates/daemon/tests/pre_gateway_compute_routing.rs
+++ b/crates/daemon/tests/pre_gateway_compute_routing.rs
@@ -1016,6 +1016,8 @@ fn production_compute_control_plane_is_transactional_and_routing_facts_fail_clos
.args(["--exact", TEST_NAME, "--nocapture"])
.env(CHILD_ROOT, directory.path())
.env("HOME", directory.path().join("home"))
+ .env_remove("CODEX_HOME")
+ .env_remove("CLAUDE_CONFIG_DIR")
.env("PATH", directory.path().join("bin"))
.env_remove("ANTHROPIC_BASE_URL")
.env_remove("ANTHROPIC_MODEL")
@@ -1047,6 +1049,8 @@ fn registered_claude_configuration_prepares_from_group_writable_install() {
.args(["--exact", DISCOVERED_PREPARE_TEST_NAME, "--nocapture"])
.env(DISCOVERED_PREPARE_CHILD_ROOT, directory.path())
.env("HOME", directory.path().join("home"))
+ .env_remove("CODEX_HOME")
+ .env_remove("CLAUDE_CONFIG_DIR")
.env("PATH", directory.path().join("bin"))
.env_remove("ANTHROPIC_BASE_URL")
.env_remove("ANTHROPIC_MODEL")
diff --git a/crates/domain/src/routing/gateway_execution.rs b/crates/domain/src/routing/gateway_execution.rs
index 441b3814..48bfbeef 100644
--- a/crates/domain/src/routing/gateway_execution.rs
+++ b/crates/domain/src/routing/gateway_execution.rs
@@ -215,18 +215,65 @@ pub enum GatewayNativeProviderStateEmissionV1 {
Unknown,
}
-#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum GatewayStreamingRefusalSemanticsV1 {
ExactDelta,
- TerminalClassified {
+ TerminalClassified,
+ /// Preserves authenticated historical digests; execution ignores both values.
+ #[doc(hidden)]
+ #[serde(rename = "terminal_classified")]
+ LegacyTerminalClassified {
max_buffered_bytes: u64,
- max_buffered_blocks: u32,
+ max_buffered_blocks: u64,
},
Unsupported,
Unknown,
}
+impl<'de> Deserialize<'de> for GatewayStreamingRefusalSemanticsV1 {
+ fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+ // Read-only recovery of persisted capability snapshots and operation
+ // journals. Keep their canonical bytes for digest validation; current
+ // producers construct TerminalClassified without these retired quotas.
+ #[derive(Deserialize)]
+ #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
+ enum Wire {
+ ExactDelta,
+ TerminalClassified {
+ #[serde(default, rename = "max_buffered_bytes")]
+ retired_bytes: Option<u64>,
+ #[serde(default, rename = "max_buffered_blocks")]
+ retired_blocks: Option<u64>,
+ },
+ Unsupported,
+ Unknown,
+ }
+ Ok(match Wire::deserialize(deserializer)? {
+ Wire::ExactDelta => Self::ExactDelta,
+ Wire::TerminalClassified {
+ retired_bytes,
+ retired_blocks,
+ } => match (retired_bytes, retired_blocks) {
+ (None, None) => Self::TerminalClassified,
+ (Some(max_buffered_bytes), Some(max_buffered_blocks)) => {
+ Self::LegacyTerminalClassified {
+ max_buffered_bytes,
+ max_buffered_blocks,
+ }
+ }
+ _ => {
+ return Err(serde::de::Error::custom(
+ "incomplete retired terminal classification descriptor",
+ ));
+ }
+ },
+ Wire::Unsupported => Self::Unsupported,
+ Wire::Unknown => Self::Unknown,
+ })
+ }
+}
+
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct GatewayRequestFeatureProfileV1 {
diff --git a/crates/gateway-core/src/runtime/body/memory.rs b/crates/gateway-core/src/runtime/body/memory.rs
index 02e02739..7e0b84d0 100644
--- a/crates/gateway-core/src/runtime/body/memory.rs
+++ b/crates/gateway-core/src/runtime/body/memory.rs
@@ -546,6 +546,31 @@ impl ChargedBodyQueue {
Ok(())
}
+ /// A protocol prefix has no message-count quota. Grow its queue against
+ /// the request budget while ordinary bounded handoff queues keep theirs.
+ pub fn push_back_growing(
+ &mut self,
+ budget: &StreamBudget,
+ chunk: ChargedBytes,
+ ) -> Result<(), BodyError> {
+ if self.chunks.len() == self.max_chunks {
+ let capacity = self
+ .max_chunks
+ .checked_mul(2)
+ .ok_or(BodyError::BodyLimitExceeded)?;
+ let bytes = capacity
+ .checked_mul(size_of::<ChargedBytes>())
+ .ok_or(BodyError::BodyLimitExceeded)?;
+ let reservation = budget.reserve(self.role, bytes)?;
+ let mut replacement = VecDeque::with_capacity(capacity);
+ replacement.extend(self.chunks.drain(..));
+ self.chunks = replacement;
+ self.metadata_reservation = Some(reservation);
+ self.max_chunks = capacity;
+ }
+ self.push_back(chunk)
+ }
+
pub fn pop_front(&mut self) -> Option<ChargedBytes> {
let chunk = self.chunks.pop_front()?;
self.visible_bytes -= chunk.bytes().len();
diff --git a/crates/gateway-core/src/runtime/driver/lifecycle.rs b/crates/gateway-core/src/runtime/driver/lifecycle.rs
index 327bd052..cb356fbb 100644
--- a/crates/gateway-core/src/runtime/driver/lifecycle.rs
+++ b/crates/gateway-core/src/runtime/driver/lifecycle.rs
@@ -85,6 +85,15 @@ pub struct GatewayCoreLifecycleLimits {
pub filter_blocking_queue: usize,
}
+impl GatewayCoreLifecycleLimits {
+ fn request_transport_chunk_bytes(&self) -> usize {
+ // Opaque codec backing and its exact charged copy coexist during admission.
+ // Derive the workspace from its memory owner, not an extra frame quota.
+ self.max_request_body_bytes
+ .min((self.stream_memory_bytes / 2).max(1))
+ }
+}
+
impl Default for GatewayCoreLifecycleLimits {
fn default() -> Self {
Self {
@@ -258,7 +267,7 @@ where
RequestPreparation::Completed(reuse) => return Ok(reuse),
RequestPreparation::Ready(request) => request,
};
- self.run_request(session, request_filters, request).await
+ Box::pin(self.run_request(session, request_filters, request)).await
}
/// Runs an already-authorized route through the same production execution
@@ -340,8 +349,10 @@ where
RequestPreparation::Completed(reuse) => return Ok(reuse),
RequestPreparation::Ready(request) => request,
};
- self.run_request(session, &mut request_filters, request)
- .await
+ // The execution future retains large provider and filter states.
+ // Keep it on the heap so wrapper polls do not duplicate its size
+ // on the listener thread's stack during a nested TLS handshake.
+ Box::pin(self.run_request(session, &mut request_filters, request)).await
}
.await;
let cleanup = request_filters
diff --git a/crates/gateway-core/src/runtime/driver/lifecycle/request.rs b/crates/gateway-core/src/runtime/driver/lifecycle/request.rs
index e7a96298..8079bc2c 100644
--- a/crates/gateway-core/src/runtime/driver/lifecycle/request.rs
+++ b/crates/gateway-core/src/runtime/driver/lifecycle/request.rs
@@ -453,7 +453,7 @@ where
// frames in core. Its head remains uncommitted to #15 until EOS, so a
// body filter can safely update held headers before exact framing.
let mut buffered_logical_frames = Vec::new();
- let request_transport_chunk_bytes = self.limits.max_request_body_bytes;
+ let request_transport_chunk_bytes = self.limits.request_transport_chunk_bytes();
loop {
// Header StopIteration remains body-driven. StopAll(Watermark)
diff --git a/crates/gateway/src/adapters/continuation.rs b/crates/gateway/src/adapters/continuation.rs
index 20c7ea9d..851578f1 100644
--- a/crates/gateway/src/adapters/continuation.rs
+++ b/crates/gateway/src/adapters/continuation.rs
@@ -2,6 +2,7 @@
use super::ProtocolAdapterError;
use crate::server::core_runtime::model_ir::{ExactProviderPathV1, ModelIrError};
use crate::server::request_plan::IngressProtocol;
+use hiroute_gateway_core::runtime::body::{MemoryRole, Reservation};
use sha2::{Digest, Sha256};
use std::future::Future;
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -25,8 +26,7 @@ pub(crate) fn project_tool_id(
&& !id
.bytes()
.all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-');
- let too_long = to == IngressProtocol::Responses && id.len() > 64;
- if !invalid && !too_long {
+ if !invalid {
return Ok(id.to_owned());
}
let prefix: String = id
@@ -66,11 +66,17 @@ impl ToolIdProjection {
}
}
+struct PendingTool {
+ id: String,
+ pattern: Vec<u8>,
+ charge: Arc<Reservation>,
+}
+
#[derive(Clone)]
pub(crate) struct ActiveResponseDelivery {
projection: ToolIdProjection,
provider_states: crate::provider_state::ActiveProviderStates,
- pending_tools: Arc<Mutex<Vec<Vec<u8>>>>,
+ pending_tools: Arc<Mutex<Vec<PendingTool>>>,
accepted_count: Arc<AtomicUsize>,
}
impl ActiveResponseDelivery {
@@ -89,6 +95,7 @@ impl ActiveResponseDelivery {
AcceptedResponseDeliveryScanner {
active: self.clone(),
carry: Vec::new(),
+ carry_charge: None,
provider_states: self.provider_states.scanner(),
}
}
@@ -115,28 +122,36 @@ pub(super) fn project_delivered_tool_id(
return Ok(native_id.to_owned());
};
let id = active.projection.project(native_id, owner)?;
- let field = match active.projection.downstream {
- IngressProtocol::Responses => "call_id",
- _ => "id",
- };
- let pattern = format!(
- "{}:{}",
- serde_json::to_string(field).unwrap(),
- serde_json::to_string(&id)
- .map_err(|e| ProtocolAdapterError::Serialization(e.to_string()))?
- )
- .into_bytes();
let mut pending = active
.pending_tools
.lock()
.unwrap_or_else(|e| e.into_inner());
- if !pending.contains(&pattern) {
- if pending.len() >= 128
- || pending.iter().map(Vec::len).sum::<usize>() + pattern.len() > 256 * 1024
- {
- return Err(ModelIrError::BufferLimit(256 * 1024).into());
- }
- pending.push(pattern);
+ if !pending.iter().any(|entry| entry.id == id) {
+ let prefix: &[u8] = match active.projection.downstream {
+ IngressProtocol::Responses => b"\"call_id\":",
+ _ => b"\"id\":",
+ };
+ let length = crate::provider_state::json_string_length(&id)?
+ .checked_add(prefix.len())
+ .ok_or(ModelIrError::BufferLimit(usize::MAX))?;
+ let bytes = length
+ .checked_mul(8)
+ .and_then(|n| n.checked_add(std::mem::size_of::<PendingTool>()))
+ .ok_or(ModelIrError::BufferLimit(usize::MAX))?;
+ let charge = active
+ .provider_states
+ .budget()
+ .reserve(MemoryRole::SemanticState, bytes)
+ .map_err(|_| ModelIrError::BufferLimit(bytes))?;
+ let mut pattern = Vec::with_capacity(length);
+ pattern.extend_from_slice(prefix);
+ serde_json::to_writer(&mut pattern, &id)
+ .map_err(|e| ProtocolAdapterError::Serialization(e.to_string()))?;
+ pending.push(PendingTool {
+ id: id.clone(),
+ pattern,
+ charge: Arc::new(charge),
+ });
}
Ok(id)
}
@@ -158,7 +173,7 @@ pub(super) fn record_provider_state_at_acceptance(
.try_with(|active| {
active
.provider_states
- .record_at_acceptance(state, owner, closing_event.to_vec())
+ .record_at_acceptance(state, owner, closing_event)
})
.unwrap_or(Ok(()))
.map_err(Into::into)
@@ -167,6 +182,7 @@ pub(super) fn record_provider_state_at_acceptance(
pub(crate) struct AcceptedResponseDeliveryScanner {
active: ActiveResponseDelivery,
carry: Vec<u8>,
+ carry_charge: Option<Arc<Reservation>>,
provider_states: crate::provider_state::AcceptedProviderStateScanner,
}
impl AcceptedResponseDeliveryScanner {
@@ -179,12 +195,17 @@ impl AcceptedResponseDeliveryScanner {
.unwrap_or_else(|e| e.into_inner());
let keep = pending
.iter()
- .map(|p| p.len().saturating_sub(1))
+ .map(|p| p.pattern.len().saturating_sub(1))
.max()
.unwrap_or(0);
+ let next_charge = pending
+ .iter()
+ .max_by_key(|p| p.pattern.len())
+ .map(|p| Arc::clone(&p.charge));
let mut boundary = self.carry.clone();
boundary.extend_from_slice(&bytes[..bytes.len().min(keep)]);
- pending.retain(|pattern| {
+ pending.retain(|entry| {
+ let pattern = &entry.pattern;
let accepted = bytes.windows(pattern.len()).any(|w| w == pattern)
|| boundary.windows(pattern.len()).any(|w| w == pattern);
if accepted {
@@ -199,6 +220,7 @@ impl AcceptedResponseDeliveryScanner {
let drop = self.carry.len().saturating_sub(keep);
self.carry.drain(..drop);
}
+ self.carry_charge = next_charge;
}
}
#[cfg(test)]
@@ -210,6 +232,63 @@ mod tests {
use crate::server::core_runtime::profiles::{CandidateProtocolProfile, fixed_reasoning};
use serde_json::json;
+ #[tokio::test]
+ async fn tool_delivery_uses_budget_without_fixed_pattern_size_or_count_limits() {
+ let budget =
+ hiroute_gateway_core::runtime::body::BudgetTree::new(8 * 1024 * 1024, 8 * 1024 * 1024)
+ .unwrap()
+ .stream(8 * 1024 * 1024)
+ .unwrap();
+ let scope = crate::provider_state::ProviderStateScopeV1 {
+ authority_id: "a".into(),
+ authority_epoch: 1,
+ grant_id: "g".into(),
+ grant_generation: 1,
+ served_model_id: "alias".into(),
+ route: hiroute_domain::ModelRequestRouteV2::Plan {
+ revision: 1,
+ semantic_digest: hiroute_domain::CanonicalDigest::of_bytes(b"route"),
+ },
+ };
+ let active = ActiveResponseDelivery::new(
+ IngressProtocol::Responses,
+ crate::provider_state::ActiveProviderStates::with_budget(
+ Arc::new(crate::provider_state::ProviderStateStore::default()),
+ scope,
+ IngressProtocol::Responses,
+ budget.clone(),
+ ),
+ );
+ let owner = CandidateProtocolProfile::exact_portable_path(
+ IngressProtocol::Responses,
+ IngressProtocol::Responses,
+ "native",
+ fixed_reasoning("fixed"),
+ )
+ .exact_provider_path()
+ .unwrap();
+ let long = "n".repeat(300 * 1024);
+ with_active_response_delivery(active.clone(), async {
+ assert_eq!(project_delivered_tool_id(&long, &owner).unwrap(), long);
+ for index in 0..129 {
+ let id = format!("call-{index}");
+ assert_eq!(project_delivered_tool_id(&id, &owner).unwrap(), id);
+ }
+ })
+ .await;
+ assert_eq!(active.pending_tools.lock().unwrap().len(), 130);
+ let mut scanner = active.scanner();
+ let wire = serde_json::to_vec(&json!({"call_id":long})).unwrap();
+ for chunk in wire.chunks(8191) {
+ scanner.accept_bytes(chunk, Instant::now());
+ }
+ assert_eq!(active.accepted_count(), 1);
+ assert!(budget.snapshot().unwrap().live > 0);
+ drop(scanner);
+ drop(active);
+ assert_eq!(budget.snapshot().unwrap().live, 0);
+ }
+
#[test]
fn repeated_native_ids_preserve_each_result_without_history_pairing() {
let document = json!({"model":"alias","input":[
@@ -314,7 +393,7 @@ mod tests {
.unwrap();
let response = decoder.finish().unwrap();
let wire_id = &response.response.tool_id_map[0].logical_id;
- assert_eq!(wire_id.len(), 64);
+ assert_eq!(wire_id, &"native".repeat(20));
let request = decode_ingress_request(IngressProtocol::Responses, &json!({"model":"alias",
"tools":[{"type":"function","name":"lookup","parameters":{"type":"object"}}],
"input":[{"type":"function_call","call_id":wire_id,"name":"lookup","arguments":"{}"},
@@ -339,7 +418,7 @@ mod tests {
}
let long = "x".repeat(100);
let shortened = project_tool_id(&long, Messages, Responses).unwrap();
- assert_eq!(shortened.len(), 64);
+ assert_eq!(shortened, long);
assert_eq!(
project_tool_id(&shortened, Messages, Responses).unwrap(),
shortened
diff --git a/crates/gateway/src/adapters/response/body_buffer.rs b/crates/gateway/src/adapters/response/body_buffer.rs
new file mode 100644
index 00000000..403fae15
--- /dev/null
+++ b/crates/gateway/src/adapters/response/body_buffer.rs
@@ -0,0 +1,96 @@
+//! Whole-JSON responses grow against their memory owner, not a protocol size cap.
+use hiroute_gateway_core::runtime::body::{MemoryRole, Reservation, StreamBudget};
+
+use super::{ModelIrError, ProtocolAdapterError};
+
+/// Reservations grow geometrically with retained semantic data. There is no
+/// independent protocol byte quota; allocation competes with the request.
+#[derive(Clone, Debug)]
+pub(super) struct Retention {
+ budget: StreamBudget,
+ bytes: usize,
+ charged: usize,
+ charges: Vec<std::sync::Arc<Reservation>>,
+}
+
+impl Retention {
+ pub(super) fn new(budget: StreamBudget) -> Self {
+ Self {
+ budget,
+ bytes: 0,
+ charged: 0,
+ charges: Vec::new(),
+ }
+ }
+
+ pub(super) fn add(&mut self, bytes: usize) -> Result<(), ProtocolAdapterError> {
+ let total = self
+ .bytes
+ .checked_add(bytes)
+ .ok_or(ModelIrError::BufferLimit(usize::MAX))?;
+ // Accumulator growth and a staged terminal reconciliation may coexist.
+ let required = total
+ .checked_mul(4)
+ .ok_or(ModelIrError::BufferLimit(usize::MAX))?;
+ if required > self.charged {
+ let target = required.checked_next_power_of_two().unwrap_or(required);
+ let charge = self
+ .budget
+ .reserve(MemoryRole::SemanticState, target - self.charged)
+ .map_err(|_| ModelIrError::BufferLimit(required))?;
+ self.charges.push(std::sync::Arc::new(charge));
+ self.charged = target;
+ }
+ self.bytes = total;
+ Ok(())
+ }
+}
+
+pub(super) fn standalone_budget() -> StreamBudget {
+ const MEMORY: usize = 256 * 1024 * 1024;
+ hiroute_gateway_core::runtime::body::BudgetTree::new(MEMORY, MEMORY)
+ .expect("fixed valid budget")
+ .stream(MEMORY)
+ .expect("fixed valid stream")
+}
+
+#[derive(Default)]
+pub(super) struct BodyBuffer {
+ bytes: Vec<u8>,
+ charge: Option<Reservation>,
+}
+
+impl BodyBuffer {
+ pub(super) fn append(
+ &mut self,
+ bytes: &[u8],
+ budget: &StreamBudget,
+ ) -> Result<(), ProtocolAdapterError> {
+ let needed = self
+ .bytes
+ .len()
+ .checked_add(bytes.len())
+ .ok_or(ModelIrError::BufferLimit(usize::MAX))?;
+ if needed > self.bytes.capacity() {
+ let capacity = needed.max(self.bytes.capacity().saturating_mul(2));
+ let charge = budget
+ .reserve(MemoryRole::ResponsePrefix, capacity)
+ .map_err(|_| ModelIrError::BufferLimit(capacity))?;
+ // Keep the old reservation alive until its allocation is dropped.
+ let mut replacement = Vec::with_capacity(capacity);
+ replacement.extend_from_slice(&self.bytes);
+ self.bytes = replacement;
+ self.charge = Some(charge);
+ }
+ self.bytes.extend_from_slice(bytes);
+ Ok(())
+ }
+
+ pub(super) fn bytes(&self) -> &[u8] {
+ &self.bytes
+ }
+
+ pub(super) fn len(&self) -> usize {
+ self.bytes.len()
+ }
+}
diff --git a/crates/gateway/src/adapters/response/client_stream.rs b/crates/gateway/src/adapters/response/client_stream.rs
index 8f69a3e1..2975e9a4 100644
--- a/crates/gateway/src/adapters/response/client_stream.rs
+++ b/crates/gateway/src/adapters/response/client_stream.rs
@@ -19,8 +19,6 @@ use super::{
#[path = "client_stream_tests.rs"]
mod tests;
-const MAX_RENDERER_METADATA_BYTES: usize = 1024 * 1024;
-
#[derive(Clone, Debug)]
struct ToolState {
logical_id: String,
@@ -57,10 +55,18 @@ pub struct IncrementalClientSseRenderer {
responses_metadata: BTreeMap<String, Value>,
open_messages_blocks: BTreeSet<u32>,
pending_messages_reasoning: BTreeSet<u32>,
- metadata_bytes: usize,
+ retention: super::body_buffer::Retention,
}
impl IncrementalClientSseRenderer {
+ pub(crate) fn with_budget(
+ mut self,
+ budget: hiroute_gateway_core::runtime::body::StreamBudget,
+ ) -> Self {
+ self.retention = super::body_buffer::Retention::new(budget);
+ self
+ }
+
pub fn new(
profile: ClientProtocolProfile,
served_model_alias: impl Into<String>,
@@ -73,7 +79,7 @@ impl IncrementalClientSseRenderer {
}
Ok(Self {
profile,
- metadata_bytes: alias.len(),
+ retention: super::body_buffer::Retention::new(super::body_buffer::standalone_budget()),
alias,
expected_sequence: 0,
native_sequence: 0,
@@ -829,7 +835,6 @@ impl IncrementalClientSseRenderer {
.map(str::to_owned)
.unwrap_or_else(|| format!("{prefix}_{index}"));
if item_id.is_empty()
- || item_id.len() > 256
|| item_id.chars().any(char::is_control)
|| self.item_ids.contains_key(&index)
|| self.item_ids.values().any(|current| current == &item_id)
@@ -1086,14 +1091,7 @@ impl IncrementalClientSseRenderer {
}
fn charge_metadata(&mut self, bytes: usize) -> Result<(), ProtocolAdapterError> {
- self.metadata_bytes = self
- .metadata_bytes
- .checked_add(bytes)
- .ok_or(ModelIrError::BufferLimit(MAX_RENDERER_METADATA_BYTES))?;
- if self.metadata_bytes > MAX_RENDERER_METADATA_BYTES {
- return Err(ModelIrError::BufferLimit(MAX_RENDERER_METADATA_BYTES).into());
- }
- Ok(())
+ self.retention.add(bytes)
}
}
diff --git a/crates/gateway/src/adapters/response/decoder.rs b/crates/gateway/src/adapters/response/decoder.rs
index e4cec098..8a972d9c 100644
--- a/crates/gateway/src/adapters/response/decoder.rs
+++ b/crates/gateway/src/adapters/response/decoder.rs
@@ -8,9 +8,8 @@ use hiroute_gateway_core::runtime::sse::{
use crate::server::core_runtime::model_ir::{ModelIrError, ModelResponseIRV1, ModelStreamEventV1};
use crate::server::core_runtime::profiles::{
- CandidateProtocolProfile, CapabilityError, Fidelity, MAX_TERMINAL_CLASSIFIED_REFUSAL_BLOCKS,
- MAX_TERMINAL_CLASSIFIED_REFUSAL_BYTES, NativeProviderStateEmission, StateAffinity,
- StreamingRefusalSemantics,
+ CandidateProtocolProfile, CapabilityError, Fidelity, NativeProviderStateEmission,
+ StateAffinity, StreamingRefusalSemantics,
};
use crate::server::request_plan::IngressProtocol;
@@ -18,10 +17,9 @@ use super::ProtocolAdapterError;
use super::protocols::ProtocolState;
use crate::server::core_runtime::adapters::ChatToolProjection;
-const MAX_NATIVE_BODY_BYTES: usize = 16 * 1024 * 1024;
-const MAX_SSE_EVENT_BYTES: usize = 256 * 1024;
+const RETAINED_SSE_CAPACITY: usize = 256 * 1024;
const MAX_QUEUED_EVENTS: usize = 32;
-const STREAM_MEMORY_BUDGET: usize = 64 * 1024 * 1024;
+const STREAM_MEMORY_BUDGET: usize = 256 * 1024 * 1024;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ResponseDecodeStatus {
@@ -41,7 +39,7 @@ pub struct NativeResponseDecoder {
protocol: IngressProtocol,
status: u16,
streaming: bool,
- body: Vec<u8>,
+ body: super::body_buffer::BodyBuffer,
state: ProtocolState,
events: VecDeque<ModelStreamEventV1>,
budget: StreamBudget,
@@ -64,6 +62,7 @@ impl NativeResponseDecoder {
profile.ingress_protocol,
)),
None,
+ None,
)
}
@@ -72,8 +71,16 @@ impl NativeResponseDecoder {
status: u16,
streaming: bool,
chat_tool_projection: Option<ChatToolProjection>,
+ budget: StreamBudget,
) -> Result<Self, ProtocolAdapterError> {
- Self::new_with_projections(profile, status, streaming, None, chat_tool_projection)
+ Self::new_with_projections(
+ profile,
+ status,
+ streaming,
+ None,
+ chat_tool_projection,
+ Some(budget),
+ )
}
/// Builds an observation-only decoder with an explicit request-bound Tool
@@ -92,6 +99,7 @@ impl NativeResponseDecoder {
streaming,
Some(tool_id_projection),
chat_tool_projection,
+ None,
)
}
@@ -101,6 +109,7 @@ impl NativeResponseDecoder {
streaming: bool,
tool_id_projection: Option<super::super::continuation::ToolIdProjection>,
chat_tool_projection: Option<ChatToolProjection>,
+ request_budget: Option<StreamBudget>,
) -> Result<Self, ProtocolAdapterError> {
if profile.capability.upstream_protocol != profile.connector.upstream_protocol
|| !profile.connector.critical_facts_are_exact()
@@ -119,56 +128,41 @@ impl NativeResponseDecoder {
);
}
let protocol = profile.capability.upstream_protocol;
- let (terminal_refusal_buffer, terminal_refusal_blocks) = if streaming {
- match (protocol, profile.capability.response.stream_refusal) {
+ if streaming
+ && !matches!(
+ (protocol, profile.capability.response.stream_refusal),
(
IngressProtocol::Messages,
- StreamingRefusalSemantics::TerminalClassified {
- max_buffered_bytes,
- max_buffered_blocks,
- },
- ) => {
- let bytes = usize::try_from(max_buffered_bytes)
- .ok()
- .filter(|limit| {
- *limit > 0
- && *limit <= super::MAX_CANONICAL_SEMANTIC_BYTES
- && max_buffered_bytes <= MAX_TERMINAL_CLASSIFIED_REFUSAL_BYTES
- })
- .ok_or(CapabilityError::StreamRefusalUnsupported)?;
- if max_buffered_blocks == 0
- || max_buffered_blocks > MAX_TERMINAL_CLASSIFIED_REFUSAL_BLOCKS
- {
- return Err(CapabilityError::StreamRefusalUnsupported.into());
- }
- (bytes, max_buffered_blocks)
- }
- (
+ StreamingRefusalSemantics::TerminalClassified
+ | StreamingRefusalSemantics::LegacyTerminalClassified { .. }
+ ) | (
IngressProtocol::Responses | IngressProtocol::ChatCompletions,
- StreamingRefusalSemantics::ExactDelta,
- ) => (0, 0),
- _ => return Err(CapabilityError::StreamRefusalUnsupported.into()),
- }
- } else {
- (0, 0)
- };
+ StreamingRefusalSemantics::ExactDelta
+ )
+ )
+ {
+ return Err(CapabilityError::StreamRefusalUnsupported.into());
+ }
let owner = profile.exact_provider_path()?;
let budget_tree = BudgetTree::new(STREAM_MEMORY_BUDGET, STREAM_MEMORY_BUDGET)
.map_err(|error| ModelIrError::InvalidSse(error.to_string()))?;
- let budget = budget_tree
- .stream(STREAM_MEMORY_BUDGET)
- .map_err(|error| ModelIrError::InvalidSse(error.to_string()))?;
+ let budget = match request_budget {
+ Some(budget) => budget,
+ None => budget_tree
+ .stream(STREAM_MEMORY_BUDGET)
+ .map_err(|error| ModelIrError::InvalidSse(error.to_string()))?,
+ };
let framer = if streaming {
Some(
SseFramer::new(
SseLimits {
- max_event_bytes: MAX_SSE_EVENT_BYTES,
- max_pending_bytes: MAX_SSE_EVENT_BYTES,
- max_output_event_bytes: MAX_SSE_EVENT_BYTES,
+ max_event_bytes: usize::MAX,
+ max_pending_bytes: usize::MAX,
+ max_output_event_bytes: usize::MAX,
expansion_ratio_numerator: 1,
expansion_ratio_denominator: 1,
expansion_slack_bytes: 0,
- retained_capacity_threshold: MAX_SSE_EVENT_BYTES,
+ retained_capacity_threshold: RETAINED_SSE_CAPACITY,
eof_policy: EofPolicy::Strict,
},
budget.clone(),
@@ -178,20 +172,23 @@ impl NativeResponseDecoder {
} else {
None
};
+ let mut state = ProtocolState::new(
+ protocol,
+ owner,
+ profile.capability.native_provider_state,
+ tool_id_projection,
+ chat_tool_projection,
+ );
+ state.core_mut().retention = super::body_buffer::Retention::new(budget.clone());
+ if let ProtocolState::Messages { state, .. } = &mut state {
+ state.retention = super::body_buffer::Retention::new(budget.clone());
+ }
Ok(Self {
protocol,
status,
streaming,
- body: Vec::new(),
- state: ProtocolState::new(
- protocol,
- owner,
- profile.capability.native_provider_state,
- terminal_refusal_buffer,
- terminal_refusal_blocks,
- tool_id_projection,
- chat_tool_projection,
- ),
+ body: super::body_buffer::BodyBuffer::default(),
+ state,
events: VecDeque::new(),
budget,
framer,
@@ -306,18 +303,10 @@ impl NativeResponseDecoder {
}
fn feed_json(&mut self, bytes: &[u8], end_stream: bool) -> Result<(), ProtocolAdapterError> {
- let next = self
- .body
- .len()
- .checked_add(bytes.len())
- .ok_or(ModelIrError::BufferLimit(MAX_NATIVE_BODY_BYTES))?;
- if next > MAX_NATIVE_BODY_BYTES {
- return Err(ModelIrError::BufferLimit(MAX_NATIVE_BODY_BYTES).into());
- }
- self.body.extend_from_slice(bytes);
+ self.body.append(bytes, &self.budget)?;
if end_stream {
self.state
- .decode_nonstream(self.status, &self.body, &mut self.events)?;
+ .decode_nonstream(self.status, self.body.bytes(), &mut self.events)?;
}
Ok(())
}
diff --git a/crates/gateway/src/adapters/response/messages.rs b/crates/gateway/src/adapters/response/messages.rs
index b6a207eb..947f791f 100644
--- a/crates/gateway/src/adapters/response/messages.rs
+++ b/crates/gateway/src/adapters/response/messages.rs
@@ -14,9 +14,7 @@ use super::{DecoderCore, ProtocolAdapterError};
pub(super) struct MessagesState {
blocks: BTreeMap<u32, NativeBlock>,
next_order: u32,
- max_deferred_bytes: usize,
- max_deferred_blocks: u32,
- deferred_bytes: usize,
+ pub(super) retention: super::body_buffer::Retention,
terminal_classified: bool,
}
@@ -43,24 +41,23 @@ enum NativeContent {
}
impl MessagesState {
- pub(super) fn new(max_deferred_bytes: usize, max_deferred_blocks: u32) -> Self {
+ pub(super) fn new() -> Self {
Self {
blocks: BTreeMap::new(),
next_order: 0,
- max_deferred_bytes,
- max_deferred_blocks,
- deferred_bytes: 0,
+ retention: super::body_buffer::Retention::new(super::body_buffer::standalone_budget()),
terminal_classified: false,
}
}
fn insert(&mut self, index: u32, content: NativeContent) -> Result<(), ProtocolAdapterError> {
- if self.terminal_classified
- || self.blocks.contains_key(&index)
- || u32::try_from(self.blocks.len()).unwrap_or(u32::MAX) >= self.max_deferred_blocks
- {
- return Err(ModelIrError::BufferLimit(self.max_deferred_blocks as usize).into());
+ if self.terminal_classified || self.blocks.contains_key(&index) {
+ return Err(ModelIrError::InvalidResponseLifecycle(
+ "duplicate or late Messages block".into(),
+ )
+ .into());
}
+ self.retention.add(256)?;
let order = self.next_order;
self.next_order = self.next_order.checked_add(1).ok_or_else(|| {
ModelIrError::InvalidResponseLifecycle("Messages block order overflow".into())
@@ -77,14 +74,7 @@ impl MessagesState {
}
fn charge(&mut self, bytes: usize) -> Result<(), ProtocolAdapterError> {
- self.deferred_bytes = self
- .deferred_bytes
- .checked_add(bytes)
- .ok_or(ModelIrError::BufferLimit(self.max_deferred_bytes))?;
- if self.deferred_bytes > self.max_deferred_bytes {
- return Err(ModelIrError::BufferLimit(self.max_deferred_bytes).into());
- }
- Ok(())
+ self.retention.add(bytes)
}
fn append_text(&mut self, index: u32, text: &str) -> Result<(), ProtocolAdapterError> {
@@ -263,7 +253,6 @@ impl MessagesState {
}
}
}
- self.deferred_bytes = 0;
self.terminal_classified = true;
Ok(())
}
diff --git a/crates/gateway/src/adapters/response/mod.rs b/crates/gateway/src/adapters/response/mod.rs
index 00e3ee57..96181c9c 100644
--- a/crates/gateway/src/adapters/response/mod.rs
+++ b/crates/gateway/src/adapters/response/mod.rs
@@ -1,5 +1,6 @@
//! Stateful native response decoding and client protocol rendering.
+mod body_buffer;
mod client_stream;
mod completion;
mod decoder;
@@ -16,6 +17,9 @@ mod wire;
#[path = "native_passthrough_tests.rs"]
mod native_passthrough_tests;
+#[cfg(test)]
+mod size_tests;
+
#[cfg(test)]
pub(super) fn test_tool_projection() -> super::continuation::ToolIdProjection {
// Production installs the process key before requests. Golden response tests must supply
@@ -41,9 +45,6 @@ use crate::server::request_plan::IngressProtocol;
use super::ProtocolAdapterError;
-const MAX_CANONICAL_SEMANTIC_BYTES: usize = 16 * 1024 * 1024;
-const MAX_CANONICAL_BLOCKS: u32 = 65_536;
-
/// Responses labels its ordinary assistant answer with `final_answer`. Messages and Chat
/// Completions have only that assistant-output channel, so the label is losslessly implicit in
/// those protocols. Every other named phase remains Responses-only and must fail closed.
@@ -111,6 +112,7 @@ struct DecoderCore {
next_block: u32,
next_sequence: u64,
semantic_bytes: usize,
+ retention: body_buffer::Retention,
}
impl DecoderCore {
@@ -135,6 +137,7 @@ impl DecoderCore {
next_block: 0,
next_sequence: 0,
semantic_bytes: 0,
+ retention: body_buffer::Retention::new(body_buffer::standalone_budget()),
}
}
@@ -166,9 +169,7 @@ impl DecoderCore {
return Ok((*index, false));
}
let index = self.next_block;
- if index >= MAX_CANONICAL_BLOCKS {
- return Err(ModelIrError::BufferLimit(MAX_CANONICAL_BLOCKS as usize).into());
- }
+ self.retention.add(256)?;
self.next_block = self.next_block.checked_add(1).ok_or_else(|| {
ModelIrError::InvalidResponseLifecycle("content block index overflow".into())
})?;
@@ -197,14 +198,14 @@ impl DecoderCore {
item_id: &str,
) -> Result<(), ProtocolAdapterError> {
if item_id.is_empty()
- || item_id.len() > 256
|| item_id.chars().any(char::is_control)
- || self.native_item_ids.len() >= MAX_CANONICAL_BLOCKS as usize
|| self.native_item_ids.contains_key(&native_index)
|| !self.native_item_id_values.insert(item_id.into())
{
return Err(ModelIrError::InvalidField("response output item id").into());
}
+ self.retention
+ .add(item_id.len().saturating_mul(2).saturating_add(128))?;
self.native_item_ids.insert(native_index, item_id.into());
Ok(())
}
@@ -243,11 +244,11 @@ impl DecoderCore {
let bytes = serde_json::to_vec(&value)
.map_err(|error| ModelIrError::InvalidJson(error.to_string()))?
.len();
+ self.retention.add(bytes)?;
self.responses_encrypted_buffered_bytes = self
.responses_encrypted_buffered_bytes
.checked_add(bytes)
- .filter(|next| *next <= MAX_CANONICAL_SEMANTIC_BYTES)
- .ok_or(ModelIrError::BufferLimit(MAX_CANONICAL_SEMANTIC_BYTES))?;
+ .ok_or(ModelIrError::BufferLimit(usize::MAX))?;
self.responses_encrypted_fallback
.insert(native_index, value);
Ok(())
@@ -321,7 +322,7 @@ impl DecoderCore {
}
return Ok(());
};
- if phase.is_empty() || phase.len() > 64 || phase.chars().any(char::is_control) {
+ if phase.is_empty() || phase.chars().any(char::is_control) {
return Err(ModelIrError::InvalidField("response output message phase").into());
}
if let Some(current) = self.native_message_phases.get(&native_index) {
@@ -345,14 +346,12 @@ impl DecoderCore {
)
.into());
}
- if self.native_message_phases.len() >= MAX_CANONICAL_BLOCKS as usize {
- return Err(ModelIrError::BufferLimit(MAX_CANONICAL_BLOCKS as usize).into());
- }
+
+ self.retention.add(phase.len())?;
self.semantic_bytes = self
.semantic_bytes
.checked_add(phase.len())
- .filter(|next| *next <= MAX_CANONICAL_SEMANTIC_BYTES)
- .ok_or(ModelIrError::BufferLimit(MAX_CANONICAL_SEMANTIC_BYTES))?;
+ .ok_or(ModelIrError::BufferLimit(usize::MAX))?;
self.native_message_phases
.insert(native_index, phase.to_owned());
Ok(())
@@ -416,10 +415,8 @@ impl DecoderCore {
let next = self
.semantic_bytes
.checked_add(added)
- .ok_or(ModelIrError::BufferLimit(MAX_CANONICAL_SEMANTIC_BYTES))?;
- if next > MAX_CANONICAL_SEMANTIC_BYTES {
- return Err(ModelIrError::BufferLimit(MAX_CANONICAL_SEMANTIC_BYTES).into());
- }
+ .ok_or(ModelIrError::BufferLimit(usize::MAX))?;
+ self.retention.add(added)?;
self.semantic_bytes = next;
Ok(())
}
@@ -743,7 +740,6 @@ impl DecoderCore {
return Ok(());
};
if item_id.is_empty()
- || item_id.len() > 256
|| item_id.chars().any(char::is_control)
|| self
.accumulator
diff --git a/crates/gateway/src/adapters/response/native_passthrough.rs b/crates/gateway/src/adapters/response/native_passthrough.rs
index acb545ac..727650bd 100644
--- a/crates/gateway/src/adapters/response/native_passthrough.rs
+++ b/crates/gateway/src/adapters/response/native_passthrough.rs
@@ -31,8 +31,7 @@ mod evidence;
#[path = "native_passthrough_state.rs"]
mod provider_state;
-const MAX_NATIVE_BODY_BYTES: usize = 16 * 1024 * 1024;
-const MAX_SSE_EVENT_BYTES: usize = 256 * 1024;
+const RETAINED_SSE_CAPACITY: usize = 256 * 1024;
const OBSERVATION_STREAM_BUDGET: usize = 1024 * 1024;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -54,7 +53,7 @@ pub(crate) struct NativeProjectedUnit {
pub(crate) struct NativeResponseProjector {
streaming: bool,
- body: Vec<u8>,
+ body: super::body_buffer::BodyBuffer,
budget: StreamBudget,
framer: Option<SseFramer>,
state: ProjectionState,
@@ -125,13 +124,15 @@ impl NativeResponseProjector {
.then(|| {
SseFramer::new(
SseLimits {
- max_event_bytes: MAX_SSE_EVENT_BYTES,
- max_pending_bytes: MAX_SSE_EVENT_BYTES,
- max_output_event_bytes: MAX_SSE_EVENT_BYTES,
+ max_event_bytes: usize::MAX,
+ max_pending_bytes: usize::MAX,
+ max_output_event_bytes: usize::MAX,
expansion_ratio_numerator: 2,
expansion_ratio_denominator: 1,
- expansion_slack_bytes: 4096,
- retained_capacity_threshold: MAX_SSE_EVENT_BYTES,
+ // Owned ID rewrites may expand many small native IDs.
+ // Charge their actual allocation instead of imposing a wire ratio.
+ expansion_slack_bytes: usize::MAX,
+ retained_capacity_threshold: RETAINED_SSE_CAPACITY,
eof_policy: EofPolicy::Strict,
},
budget.clone(),
@@ -143,7 +144,7 @@ impl NativeResponseProjector {
.transpose()?;
Ok(Self {
streaming,
- body: Vec::new(),
+ body: super::body_buffer::BodyBuffer::default(),
budget: budget.clone(),
framer,
state: ProjectionState {
@@ -190,20 +191,12 @@ impl NativeResponseProjector {
.into());
}
if !self.streaming {
- let next = self
- .body
- .len()
- .checked_add(bytes.len())
- .ok_or(ModelIrError::BufferLimit(MAX_NATIVE_BODY_BYTES))?;
- if next > MAX_NATIVE_BODY_BYTES {
- return Err(ModelIrError::BufferLimit(MAX_NATIVE_BODY_BYTES).into());
- }
- self.body.extend_from_slice(bytes);
+ self.body.append(bytes, &self.budget)?;
if !end_stream {
return Ok(Vec::new());
}
self.ended = true;
- let mut value: Value = serde_json::from_slice(&self.body)
+ let mut value: Value = serde_json::from_slice(self.body.bytes())
.map_err(|error| ModelIrError::InvalidJson(error.to_string()))?;
let metadata = self.state.project_nonstream(&mut value)?;
let bytes = serde_json::to_vec(&value)
diff --git a/crates/gateway/src/adapters/response/native_passthrough_state.rs b/crates/gateway/src/adapters/response/native_passthrough_state.rs
index fb3a50ce..9b749e34 100644
--- a/crates/gateway/src/adapters/response/native_passthrough_state.rs
+++ b/crates/gateway/src/adapters/response/native_passthrough_state.rs
@@ -42,11 +42,14 @@ impl ProjectionState {
match object.get("type").and_then(Value::as_str) {
Some("content_block_start") if object["content_block"]["type"] == "thinking" => {
let index = u32_field(object, "index")?;
- if self.messages_signatures.len() >= 128
- || self.messages_signatures.contains_key(&index)
- {
+ if self.messages_signatures.contains_key(&index) {
return Err(ModelIrError::InvalidField("thinking.index").into());
}
+ let charge = self
+ .budget
+ .reserve(MemoryRole::SemanticState, 128)
+ .map_err(|_| ModelIrError::BufferLimit(128))?;
+ self.retained.push(charge);
self.messages_signatures.insert(index, String::new());
if let Some(value) = object["content_block"]
.get("signature")
@@ -87,16 +90,14 @@ impl ProjectionState {
if delta.is_empty() {
return Ok(());
}
- if signature.len().saturating_add(delta.len()) > MAX_SSE_EVENT_BYTES {
- return Err(ModelIrError::BufferLimit(MAX_SSE_EVENT_BYTES).into());
- }
+ let bytes = delta
+ .len()
+ .checked_add(2 * std::mem::size_of::<Reservation>())
+ .ok_or(ModelIrError::BufferLimit(usize::MAX))?;
let charge = self
.budget
- .reserve(
- MemoryRole::SemanticState,
- delta.len() + 2 * std::mem::size_of::<Reservation>(),
- )
- .map_err(|_| ModelIrError::BufferLimit(MAX_SSE_EVENT_BYTES))?;
+ .reserve(MemoryRole::SemanticState, bytes)
+ .map_err(|_| ModelIrError::BufferLimit(bytes))?;
signature.reserve_exact(delta.len());
signature.push_str(delta);
self.retained.push(charge);
diff --git a/crates/gateway/src/adapters/response/protocols.rs b/crates/gateway/src/adapters/response/protocols.rs
index afc09c3a..f45288ba 100644
--- a/crates/gateway/src/adapters/response/protocols.rs
+++ b/crates/gateway/src/adapters/response/protocols.rs
@@ -53,8 +53,6 @@ impl ProtocolState {
protocol: IngressProtocol,
owner: ExactProviderPathV1,
state_emission: NativeProviderStateEmission,
- terminal_refusal_buffer: usize,
- terminal_refusal_blocks: u32,
tool_id_projection: Option<super::super::continuation::ToolIdProjection>,
chat_tool_projection: Option<ChatToolProjection>,
) -> Self {
@@ -71,7 +69,7 @@ impl ProtocolState {
},
IngressProtocol::Messages => Self::Messages {
core: DecoderCore::new(owner, state_emission, tool_id_projection),
- state: MessagesState::new(terminal_refusal_buffer, terminal_refusal_blocks),
+ state: MessagesState::new(),
},
}
}
@@ -157,7 +155,7 @@ impl ProtocolState {
}
}
- fn core_mut(&mut self) -> &mut DecoderCore {
+ pub(super) fn core_mut(&mut self) -> &mut DecoderCore {
match self {
Self::Responses { core, .. }
| Self::Chat { core, .. }
@@ -721,10 +719,7 @@ fn append_chat_tool_arguments(
total.checked_add(tool.wire_arguments.len())
})
.and_then(|total| total.checked_add(delta.len()))
- .filter(|total| *total <= super::MAX_CANONICAL_SEMANTIC_BYTES)
- .ok_or(ModelIrError::BufferLimit(
- super::MAX_CANONICAL_SEMANTIC_BYTES,
- ))?;
+ .ok_or(ModelIrError::BufferLimit(usize::MAX))?;
tools
.get_mut(&native_index)
.expect("Chat tool identity was established before its arguments")
@@ -881,6 +876,7 @@ fn decode_chat_sse(
output,
)?,
ToolKindV1::Custom => {
+ core.retention.add(delta.len())?;
append_chat_tool_arguments(tools, native_index, delta)?;
core.start_tool(
native_index,
diff --git a/crates/gateway/src/adapters/response/size_tests.rs b/crates/gateway/src/adapters/response/size_tests.rs
new file mode 100644
index 00000000..c8e87754
--- /dev/null
+++ b/crates/gateway/src/adapters/response/size_tests.rs
@@ -0,0 +1,207 @@
+use super::*;
+use hiroute_gateway_core::runtime::body::BudgetTree;
+use serde_json::json;
+
+fn projector(streaming: bool, bytes: usize) -> NativeResponseProjector {
+ let profile =
+ crate::server::core_runtime::profiles::CandidateProtocolProfile::exact_portable_path(
+ IngressProtocol::Responses,
+ IngressProtocol::Responses,
+ "physical",
+ crate::server::core_runtime::profiles::fixed_reasoning("fixed"),
+ );
+ let budget = BudgetTree::new(bytes, bytes)
+ .unwrap()
+ .stream(bytes)
+ .unwrap();
+ NativeResponseProjector::new_for_attempt(&profile, streaming, "alias".into(), None, budget)
+ .unwrap()
+}
+
+#[test]
+fn native_large_sse_event_preserves_bytes_across_transport_fragmentation() {
+ let content = "中文-content-".repeat(32 * 1024);
+ let event = json!({"type":"provider.extension","payload":content});
+ let wire = format!("event: provider.extension\ndata: {event}\n\n").into_bytes();
+ assert!(wire.len() > 256 * 1024);
+ for chunk_size in [1021, wire.len()] {
+ let mut projector = projector(true, 16 * 1024 * 1024);
+ let mut actual = Vec::new();
+ for chunk in wire.chunks(chunk_size) {
+ for unit in projector.feed(chunk, false).unwrap() {
+ actual.extend_from_slice(&unit.bytes);
+ }
+ }
+ assert_eq!(actual, wire);
+ }
+}
+
+#[test]
+fn native_json_response_above_sixteen_mib_preserves_content() {
+ let content = "x".repeat(17 * 1024 * 1024);
+ let body = json!({"id":"response", "model":"physical", "status":"completed",
+ "output":[{"type":"message", "id":"message", "status":"completed", "role":"assistant",
+ "content":[{"type":"output_text","text":content}]}]});
+ let wire = serde_json::to_vec(&body).unwrap();
+ let mut projector = projector(false, 128 * 1024 * 1024);
+ for chunk in wire.chunks(64 * 1024) {
+ assert!(projector.feed(chunk, false).unwrap().is_empty());
+ }
+ let units = projector.feed(&[], true).unwrap();
+ let mut expected = body;
+ expected["model"] = json!("alias");
+ assert_eq!(
+ serde_json::from_slice::<Value>(&units[0].bytes).unwrap(),
+ expected
+ );
+ assert_eq!(units[0].terminal, Some(NativeTerminalOutcome::Complete));
+}
+
+#[test]
+fn json_buffer_still_refuses_unavailable_memory_before_allocation() {
+ let mut projector = projector(false, 1024);
+ assert!(projector.feed(&vec![b'x'; 2048], false).is_err());
+}
+
+#[test]
+fn canonical_decode_and_incremental_render_preserve_large_content_and_ids() {
+ use crate::server::core_runtime::profiles::{
+ CandidateProtocolProfile, ClientProtocolProfile, fixed_reasoning,
+ };
+ let candidate = CandidateProtocolProfile::exact_portable_path(
+ IngressProtocol::Responses,
+ IngressProtocol::ChatCompletions,
+ "physical",
+ fixed_reasoning("fixed"),
+ );
+ let id = "r".repeat(1024 * 1024 + 1);
+ let content = "answer".repeat(3 * 1024 * 1024);
+ let wire = serde_json::to_vec(&json!({"id":id,"model":"physical",
+ "choices":[{"index":0,"message":{"role":"assistant","content":content},"finish_reason":"stop"}],
+ "usage":{"prompt_tokens":4,"completion_tokens":2,"total_tokens":6}})).unwrap();
+ let mut decoder = NativeResponseDecoder::new(&candidate, 200, false).unwrap();
+ decoder.feed(&wire, true).unwrap();
+ let decoded = decoder.finish().unwrap();
+ let mut renderer = client_stream::IncrementalClientSseRenderer::new(
+ ClientProtocolProfile::for_candidate(&candidate).unwrap(),
+ "alias",
+ )
+ .unwrap();
+ let mut rendered = Vec::new();
+ for event in &decoded.events {
+ rendered.extend(renderer.push(event).unwrap());
+ }
+ let final_response = &rendered.last().unwrap().data["response"];
+ assert_eq!(final_response["id"], id);
+ let delta = rendered
+ .iter()
+ .find(|event| event.event.as_deref() == Some("response.output_text.delta"))
+ .unwrap();
+ assert_eq!(delta.data["delta"], content);
+}
+
+#[test]
+fn messages_terminal_classification_accepts_more_than_eight_content_blocks() {
+ assert_messages_blocks(false);
+}
+
+#[test]
+fn recovered_terminal_descriptor_does_not_restore_retired_quotas() {
+ assert_messages_blocks(true);
+}
+
+fn assert_messages_blocks(recovered: bool) {
+ use crate::server::core_runtime::profiles::{CandidateProtocolProfile, fixed_reasoning};
+ let mut candidate = CandidateProtocolProfile::exact_portable_path(
+ IngressProtocol::Responses,
+ IngressProtocol::Messages,
+ "physical",
+ fixed_reasoning("fixed"),
+ );
+ if recovered {
+ candidate.capability.response.stream_refusal = serde_json::from_value(json!({
+ "kind":"terminal_classified", "max_buffered_bytes":1, "max_buffered_blocks":1
+ }))
+ .unwrap();
+ }
+ let mut decoder = NativeResponseDecoder::new(&candidate, 200, true).unwrap();
+ let mut events = vec![
+ json!({"type":"message_start","message":{"id":"m","type":"message",
+ "role":"assistant","model":"physical","content":[],"stop_reason":null,"stop_sequence":null,
+ "usage":{"input_tokens":1,"output_tokens":0}}}),
+ ];
+ for index in 0..9 {
+ events.push(json!({"type":"content_block_start","index":index,"content_block":{"type":"text","text":format!("part-{index}")}}));
+ events.push(json!({"type":"content_block_stop","index":index}));
+ }
+ events.push(json!({"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":9}}));
+ events.push(json!({"type":"message_stop"}));
+ for event in events {
+ let wire = format!(
+ "event: {}\ndata: {event}\n\n",
+ event["type"].as_str().unwrap()
+ );
+ let mut status = decoder.feed(wire.as_bytes(), false).unwrap();
+ decoder.take_events();
+ while status == ResponseDecodeStatus::NeedDrain {
+ status = decoder.resume().unwrap();
+ decoder.take_events();
+ }
+ }
+ decoder.feed(&[], true).unwrap();
+ let result = decoder.finish().unwrap();
+ assert_eq!(result.response.blocks.len(), 9);
+}
+
+#[test]
+fn native_messages_signatures_use_budget_without_byte_or_block_quotas() {
+ use crate::server::core_runtime::profiles::{CandidateProtocolProfile, fixed_reasoning};
+ let candidate = CandidateProtocolProfile::exact_portable_path(
+ IngressProtocol::Messages,
+ IngressProtocol::Messages,
+ "physical",
+ fixed_reasoning("fixed"),
+ );
+ let budget = BudgetTree::new(16 * 1024 * 1024, 16 * 1024 * 1024)
+ .unwrap()
+ .stream(16 * 1024 * 1024)
+ .unwrap();
+ let mut projector = NativeResponseProjector::new_for_attempt(
+ &candidate,
+ true,
+ "alias".into(),
+ None,
+ budget.clone(),
+ )
+ .unwrap();
+ let mut events = Vec::new();
+ for index in 0..129 {
+ events.push(json!({"type":"content_block_start","index":index,
+ "content_block":{"type":"thinking","thinking":""}}));
+ }
+ let fragment = "s".repeat(160 * 1024);
+ for _ in 0..2 {
+ events.push(json!({"type":"content_block_delta","index":0,
+ "delta":{"type":"signature_delta","signature":fragment}}));
+ }
+ for index in 0..129 {
+ events.push(json!({"type":"content_block_stop","index":index}));
+ }
+ for event in events {
+ let wire = format!(
+ "event: {}\ndata: {event}\n\n",
+ event["type"].as_str().unwrap()
+ );
+ let units = projector.feed(wire.as_bytes(), false).unwrap();
+ assert_eq!(
+ units
+ .into_iter()
+ .flat_map(|unit| unit.bytes)
+ .collect::<Vec<_>>(),
+ wire.as_bytes()
+ );
+ }
+ assert!(budget.snapshot().unwrap().live > 0);
+ drop(projector);
+ assert_eq!(budget.snapshot().unwrap().live, 0);
+}
diff --git a/crates/gateway/src/adapters/responses_options.rs b/crates/gateway/src/adapters/responses_options.rs
index d320bfa2..cad79e1c 100644
--- a/crates/gateway/src/adapters/responses_options.rs
+++ b/crates/gateway/src/adapters/responses_options.rs
@@ -23,12 +23,7 @@ pub(super) fn annotations(
if !annotations.is_empty() {
if item.get("role").and_then(Value::as_str) != Some("assistant")
|| part.get("type").and_then(Value::as_str) != Some("output_text")
- || annotations.len() > 128
- || annotations.iter().any(|a| {
- a.start_index > a.end_index
- || a.title.len() > 4096
- || a.url.len() > 8192
- })
+ || annotations.iter().any(|a| a.start_index > a.end_index)
{
return Err(ModelIrError::InvalidField("annotations"));
}
@@ -60,7 +55,7 @@ pub(super) fn item_ids(
let id = id
.as_str()
.ok_or(ModelIrError::InvalidField("input[].id"))?;
- if id.is_empty() || id.len() > 256 || id.chars().any(char::is_control) {
+ if id.is_empty() || id.chars().any(char::is_control) {
return Err(ModelIrError::InvalidField("input[].id"));
}
ids.insert(message_index, id.to_owned());
@@ -102,7 +97,7 @@ pub(super) fn internal_chat_message_metadata(
"internal_chat_message_metadata_passthrough",
)?;
let turn_id = required_string(metadata, "turn_id")?;
- if turn_id.is_empty() || turn_id.len() > 256 || turn_id.chars().any(char::is_control) {
+ if turn_id.is_empty() || turn_id.chars().any(char::is_control) {
return Err(ModelIrError::InvalidField(
"internal_chat_message_metadata_passthrough.turn_id",
));
@@ -158,15 +153,16 @@ pub(super) fn decode(
)?;
for field in ["effort", "summary"] {
if let Some(value) = optional_string(reasoning, field)?
- && (value.is_empty() || value.len() > 64 || value.chars().any(char::is_control))
+ && (value.is_empty() || value.chars().any(char::is_control))
{
return Err(ModelIrError::InvalidField(field));
}
}
let context = optional_string(reasoning, "context")?;
- if context.as_deref().is_some_and(|value| {
- value.is_empty() || value.len() > 256 || value.chars().any(char::is_control)
- }) {
+ if context
+ .as_deref()
+ .is_some_and(|value| value.is_empty() || value.chars().any(char::is_control))
+ {
return Err(ModelIrError::InvalidField("reasoning.context"));
}
Ok((optional_string(reasoning, "summary")?, context))
@@ -185,17 +181,13 @@ pub(super) fn decode(
let values = value
.as_array()
.ok_or(ModelIrError::InvalidField("include"))?;
- if values.len() > 16 {
- return Err(ModelIrError::InvalidField("include"));
- }
values
.iter()
.map(|value| {
let value = value
.as_str()
.ok_or(ModelIrError::InvalidField("include"))?;
- if value.is_empty() || value.len() > 256 || value.chars().any(char::is_control)
- {
+ if value.is_empty() || value.chars().any(char::is_control) {
return Err(ModelIrError::InvalidField("include"));
}
Ok(value.to_owned())
@@ -206,7 +198,7 @@ pub(super) fn decode(
let prompt_cache_key = optional_string(object, "prompt_cache_key")?;
if prompt_cache_key
.as_ref()
- .is_some_and(|key| key.len() > 256 || key.chars().any(char::is_control))
+ .is_some_and(|key| key.chars().any(char::is_control))
{
return Err(ModelIrError::InvalidField("prompt_cache_key"));
}
@@ -216,16 +208,13 @@ pub(super) fn decode(
let values = value
.as_object()
.ok_or(ModelIrError::InvalidField("client_metadata"))?;
- if values.len() > 16 {
- return Err(ModelIrError::InvalidField("client_metadata"));
- }
values
.iter()
.map(|(key, value)| {
let value = value
.as_str()
.ok_or(ModelIrError::InvalidField("client_metadata"))?;
- if key.len() > 128 || value.len() > 2048 || key.chars().any(char::is_control) {
+ if key.chars().any(char::is_control) {
return Err(ModelIrError::InvalidField("client_metadata"));
}
Ok((key.clone(), value.to_owned()))
@@ -256,6 +245,56 @@ pub(super) fn decode(
mod tests {
use super::*;
use serde_json::json;
+
+ #[test]
+ fn native_options_and_history_preserve_values_above_former_proxy_limits() {
+ let long = "native".repeat(2048);
+ let metadata = (0..20)
+ .map(|n| (format!("{long}{n}"), json!(long)))
+ .collect::<Map<String, Value>>();
+ let document = json!({
+ "reasoning":{"effort":long,"summary":long,"context":long},
+ "prompt_cache_key":long,
+ "include":vec![long.clone();20],
+ "client_metadata":metadata
+ });
+ let options = decode(document.as_object().unwrap()).unwrap().unwrap();
+ assert_eq!(options.reasoning_summary.as_deref(), Some(long.as_str()));
+ assert_eq!(options.reasoning_context.as_deref(), Some(long.as_str()));
+ assert_eq!(options.prompt_cache_key.as_deref(), Some(long.as_str()));
+ assert_eq!(options.include.unwrap(), vec![long.clone(); 20]);
+ assert_eq!(
+ options.client_metadata.unwrap(),
+ metadata
+ .iter()
+ .map(|(k, v)| (k.clone(), v.as_str().unwrap().to_owned()))
+ .collect()
+ );
+ let citation = json!({"type":"url_citation","start_index":0,"end_index":1,
+ "title":long,"url":format!("https://example.test/{long}")});
+ let history = json!([{"type":"message","role":"assistant","id":long,
+ "content":[{"type":"output_text","text":"a","annotations":vec![citation;129]}],
+ "internal_chat_message_metadata_passthrough":{"turn_id":long}}]);
+ assert_eq!(item_ids(Some(&history)).unwrap()[&0], long);
+ assert_eq!(
+ internal_chat_message_metadata(&history[0])
+ .unwrap()
+ .unwrap()
+ .turn_id,
+ long
+ );
+ let citations = annotations(Some(&history)).unwrap();
+ assert_eq!(citations[&0][&0].len(), 129);
+ assert_eq!(citations[&0][&0][0].title, long);
+ assert_eq!(
+ citations[&0][&0][0].url,
+ format!("https://example.test/{long}")
+ );
+ let mut malformed = history;
+ malformed[0]["content"][0]["annotations"][0]["start_index"] = json!(2);
+ assert!(annotations(Some(&malformed)).is_err());
+ }
+
#[test]
fn reasoning_context_roundtrips_only_on_responses_without_dropping_history_policy() {
use super::super::super::project_candidate_request;
@@ -288,20 +327,14 @@ mod tests {
assert!(projection.is_err());
}
}
- for invalid in [
- json!(null),
- json!(false),
- json!(""),
- json!("x".repeat(257)),
- json!("line\nbreak"),
- ] {
+ for invalid in [json!(null), json!(false), json!(""), json!("line\nbreak")] {
let mut document = document.clone();
document["reasoning"]["context"] = invalid;
assert!(decode_ingress_request(IngressProtocol::Responses, &document).is_err());
}
}
#[test]
- fn bounded_native_responses_options_are_provider_decided_not_gateway_enumerated() {
+ fn native_responses_options_are_provider_decided_not_gateway_enumerated() {
use super::super::super::project_candidate_request;
use super::super::decode_ingress_request;
use crate::server::core_runtime::profiles::{CandidateProtocolProfile, fixed_reasoning};
@@ -437,7 +470,7 @@ mod tests {
);
let projected = project_candidate_request(&restored, &profile).unwrap();
assert_eq!(projected.body["input"], document["input"]);
- for invalid in [Value::Null, json!(5), json!(""), json!("x".repeat(257))] {
+ for invalid in [Value::Null, json!(5), json!("")] {
let mut document = document.clone();
document["input"][0]["id"] = invalid;
assert!(decode_ingress_request(IngressProtocol::Responses, &document).is_err());
@@ -547,7 +580,7 @@ mod tests {
}
#[test]
- fn installed_codex_item_metadata_and_phase_remain_bounded_and_fail_closed() {
+ fn installed_codex_item_metadata_and_phase_keep_type_and_semantic_validation() {
use super::super::decode_ingress_request;
let valid = json!({"model":"alias","input":[{
"type":"message","role":"assistant","phase":"commentary",
@@ -560,7 +593,6 @@ mod tests {
json!(null),
json!({}),
json!({"turn_id":""}),
- json!({"turn_id":"x".repeat(257)}),
json!({"turn_id":"turn-1","unknown":true}),
] {
let mut document = valid.clone();
@@ -947,16 +979,13 @@ mod tests {
assert_eq!(decode(json!({}).as_object().unwrap()).unwrap(), None);
}
#[test]
- fn stateful_unknown_and_unbounded_controls_fail_closed() {
+ fn stateful_unknown_and_malformed_controls_fail_closed() {
for value in [
json!({"store":true}),
json!({"store":"false"}),
json!({"include":[false]}),
json!({"include":[""]}),
- json!({"include":["x".repeat(257)]}),
- json!({"include":vec!["x"; 17]}),
json!({"client_metadata":{"nested":{}}}),
- json!({"prompt_cache_key":"x".repeat(257)}),
] {
assert!(decode(value.as_object().unwrap()).is_err());
}
diff --git a/crates/gateway/src/content_ref.rs b/crates/gateway/src/content_ref.rs
index 6a56c2ee..62e1d707 100644
--- a/crates/gateway/src/content_ref.rs
+++ b/crates/gateway/src/content_ref.rs
@@ -11,7 +11,6 @@ use crate::server::core_runtime::model_ir::{
pub(crate) const MARKER_PREFIX: &str = "__hiroute_content_ref_v2_";
const MARKER_SUFFIX: &str = "__";
const JSON_MARKER_KEY: &str = "__hiroute_json_content_ref_v2";
-pub const MAX_CONTENT_FIELDS: usize = 16_384;
const CONTENT_FIELD_METADATA_BYTES: usize = 192;
mod ingress;
@@ -151,9 +150,6 @@ pub fn externalize_model_request(
inline_limit: usize,
) -> Result<(), ReplayError> {
let field_count = content_field_count(request)?;
- if field_count > MAX_CONTENT_FIELDS {
- return Err(ReplayError::StructureLimit);
- }
replay.charge_metadata(
field_count
.checked_mul(CONTENT_FIELD_METADATA_BYTES)
diff --git a/crates/gateway/src/content_ref/ingress.rs b/crates/gateway/src/content_ref/ingress.rs
index c84859cf..37f9561d 100644
--- a/crates/gateway/src/content_ref/ingress.rs
+++ b/crates/gateway/src/content_ref/ingress.rs
@@ -6,10 +6,9 @@ use serde_json::Value;
use crate::replay::{ReplayContentWriter, ReplayError, ReplayReader, ReplayStore};
use crate::server::request_plan::IngressProtocol;
-use super::{MARKER_PREFIX, MAX_CONTENT_FIELDS};
+use super::MARKER_PREFIX;
const MAX_JSON_DEPTH: usize = 128;
-const MAX_CONTROL_STRING_BYTES: usize = 8 * 1024;
const INLINE_CONTENT_BYTES: usize = 8 * 1024;
const STRUCTURE_RESERVATION_BYTES: usize = 128;
@@ -77,7 +76,6 @@ pub(crate) fn compact_ingress_document(
replay,
pool: None,
inline_remaining: INLINE_CONTENT_BYTES,
- fields: 0,
};
compactor.walk(document, None, 0)?;
if let Some(pool) = compactor.pool {
@@ -91,7 +89,6 @@ struct Compactor<'a> {
replay: &'a ReplayStore,
pool: Option<ReplayContentWriter>,
inline_remaining: usize,
- fields: usize,
}
impl Compactor<'_> {
@@ -111,8 +108,6 @@ impl Compactor<'_> {
Value::String(text) => {
if field.is_some_and(content_string_field) {
self.externalize_string(text)
- } else if text.len() > MAX_CONTROL_STRING_BYTES {
- Err(ReplayError::StructureLimit)
} else {
Ok(())
}
@@ -225,7 +220,6 @@ impl Compactor<'_> {
value: &mut String,
payload_start: usize,
) -> Result<(), ReplayError> {
- self.count_field()?;
let payload = value.get(payload_start..).ok_or(ReplayError::InvalidUtf8)?;
if payload.len() <= self.inline_remaining && !payload.contains(MARKER_PREFIX) {
self.inline_remaining -= payload.len();
@@ -238,7 +232,6 @@ impl Compactor<'_> {
}
fn externalize_string(&mut self, value: &mut String) -> Result<(), ReplayError> {
- self.count_field()?;
if value.len() <= self.inline_remaining && !value.contains(MARKER_PREFIX) {
self.inline_remaining -= value.len();
return Ok(());
@@ -249,7 +242,6 @@ impl Compactor<'_> {
}
fn externalize_json(&mut self, value: &mut Value) -> Result<(), ReplayError> {
- self.count_field()?;
let reference = self.pool()?.append_json(value, false)?;
*value = reference.json_marker();
Ok(())
@@ -261,17 +253,6 @@ impl Compactor<'_> {
}
self.pool.as_mut().ok_or(ReplayError::Integrity)
}
-
- fn count_field(&mut self) -> Result<(), ReplayError> {
- self.fields = self
- .fields
- .checked_add(1)
- .ok_or(ReplayError::LengthOverflow)?;
- if self.fields > MAX_CONTENT_FIELDS {
- return Err(ReplayError::StructureLimit);
- }
- Ok(())
- }
}
fn content_string_field(field: &str) -> bool {
@@ -286,6 +267,9 @@ fn content_string_field(field: &str) -> bool {
| "output"
| "system"
| "text"
+ | "thinking"
+ | "signature"
+ | "encrypted_content"
)
}
@@ -314,6 +298,85 @@ mod tests {
use hiroute_gateway_core::runtime::body::BudgetTree;
use serde_json::json;
+ #[test]
+ fn long_control_strings_are_preserved_without_a_generic_length_limit() {
+ let root = std::env::temp_dir().join(format!(
+ "hiroute-long-control-{}-{}",
+ std::process::id(),
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap()
+ .as_nanos()
+ ));
+ let manager = crate::replay::ReplayManager::open(crate::replay::ReplayConfig {
+ root: root.clone(),
+ memory_threshold_bytes: 128,
+ record_bytes: 31,
+ orphan_ttl: std::time::Duration::from_secs(60),
+ })
+ .unwrap();
+ let original = "control 中文 ".repeat(4096);
+ for protocol in [
+ IngressProtocol::Responses,
+ IngressProtocol::Messages,
+ IngressProtocol::ChatCompletions,
+ ] {
+ let tree = BudgetTree::new(1024 * 1024, 1024 * 1024).unwrap();
+ let store = manager
+ .begin_request(tree.stream(1024 * 1024).unwrap())
+ .unwrap();
+ let mut document = json!({"model":original,"metadata":{"extension":original}});
+ let expected = document.clone();
+ compact_ingress_document(protocol, &mut document, &store).unwrap();
+ assert_eq!(document, expected);
+ }
+ drop(manager);
+ std::fs::remove_dir_all(root).unwrap();
+ }
+
+ #[test]
+ fn long_native_reasoning_history_is_content_not_control() {
+ let root = std::env::temp_dir().join(format!(
+ "hiroute-long-reasoning-{}-{}",
+ std::process::id(),
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap()
+ .as_nanos()
+ ));
+ let manager = crate::replay::ReplayManager::open(crate::replay::ReplayConfig {
+ root: root.clone(),
+ memory_threshold_bytes: 128,
+ record_bytes: 31,
+ orphan_ttl: std::time::Duration::from_secs(60),
+ })
+ .unwrap();
+ for (protocol, field) in [
+ (IngressProtocol::Messages, "thinking"),
+ (IngressProtocol::Messages, "signature"),
+ (IngressProtocol::Responses, "encrypted_content"),
+ ] {
+ let tree = BudgetTree::new(1024 * 1024, 1024 * 1024).unwrap();
+ let store = manager
+ .begin_request(tree.stream(1024 * 1024).unwrap())
+ .unwrap();
+ let original = "reasoning 中文 ".repeat(1024);
+ let mut document = json!({"model":"alias", "history":[{field:original}]});
+ compact_ingress_document(protocol, &mut document, &store).unwrap();
+ let marker = document["history"][0][field].as_str().unwrap();
+ let reference = crate::content_ref::ContentRef::from_wire_marker(marker).unwrap();
+ let mut actual = String::new();
+ store
+ .reader(&reference)
+ .unwrap()
+ .read_to_string(&mut actual)
+ .unwrap();
+ assert_eq!(actual, original);
+ }
+ drop(manager);
+ std::fs::remove_dir_all(root).unwrap();
+ }
+
#[test]
fn responses_reasoning_stays_visible_for_typed_summary_decode() {
let root = std::env::temp_dir().join(format!(
@@ -415,9 +478,6 @@ impl StructuralScanner {
.structure_units
.checked_add(1)
.ok_or(ReplayError::StructureLimit)?;
- if self.structure_units > MAX_CONTENT_FIELDS {
- return Err(ReplayError::StructureLimit);
- }
Ok(())
}
diff --git a/crates/gateway/src/core_runtime.rs b/crates/gateway/src/core_runtime.rs
index bde22b39..63d1150e 100644
--- a/crates/gateway/src/core_runtime.rs
+++ b/crates/gateway/src/core_runtime.rs
@@ -22,7 +22,7 @@ use std::time::Instant;
use async_trait::async_trait;
use bytes::Bytes;
use hiroute_gateway_core::runtime::body::{
- BodyDirection, BodyError, BodyPlanExecutor, MemoryRole, StreamBudget,
+ BodyDirection, BodyError, BodyPlanExecutor, ChargedBytes, MemoryRole, StreamBudget,
};
use hiroute_gateway_core::runtime::driver::{
BoundRequestAdmission, DecisionCandidateAuthority, GatewayCoreLifecycle,
@@ -128,7 +128,12 @@ impl ProductionGatewayRuntime {
ObservedProductionProvider::new(ProductionProvider::new(&ports)),
NoopGatewayFilterManager,
connector.clone(),
- GatewayCoreLifecycleLimits::default(),
+ GatewayCoreLifecycleLimits {
+ // Replay and retained state use the shared memory owner;
+ // the model proxy adds no cumulative request byte quota.
+ max_request_body_bytes: usize::MAX,
+ ..GatewayCoreLifecycleLimits::default()
+ },
)
.expect("static production lifecycle limits are valid"),
);
@@ -281,12 +286,25 @@ impl GatewayLifecycle for ProductionGatewayRuntime {
Ok(authenticated) => authenticated,
Err(error) => return write_dispatch_error(session, error).await,
};
+ let budget = match self.execution.allocate_bound_stream_budget() {
+ Ok(budget) => budget,
+ Err(_) => {
+ return write_typed_error_phase(
+ session,
+ StatusCode::SERVICE_UNAVAILABLE,
+ "REPLAY_BUDGET_UNAVAILABLE",
+ "canonical_request",
+ )
+ .await;
+ }
+ };
let selector_deadline = authenticated.selector_deadline();
let mut selector = ModelSelector::new(self.authority().selector_limit());
// Retain every opaque chunk already polled while the bounded selector
// sees only its prefix, then replay those exact Bytes into the
// canonical decoder before polling the rest of the body.
let mut ingress_prefix = Vec::new();
+ let mut selector_charges = Vec::new();
let served_model_id = loop {
match session.read_request_body_before(selector_deadline).await {
Err(TransportError::DeadlineExceeded) => {
@@ -295,8 +313,26 @@ impl GatewayLifecycle for ProductionGatewayRuntime {
}
Err(error) => return Err(error),
Ok(Some(chunk)) => {
+ let charge_bytes = chunk.len().checked_mul(3).and_then(|n| n.checked_add(128));
+ let charge = charge_bytes
+ .and_then(|n| budget.reserve(MemoryRole::ModelIrBacking, n).ok());
+ let Some(charge) = charge else {
+ return write_typed_error_phase(
+ session,
+ StatusCode::SERVICE_UNAVAILABLE,
+ "REPLAY_BUDGET_UNAVAILABLE",
+ "model_selector",
+ )
+ .await;
+ };
+ selector_charges.push(charge);
let selected = selector.feed(&chunk);
- ingress_prefix.push(chunk);
+ let retained =
+ ChargedBytes::copy_from_opaque(&budget, MemoryRole::RawRequest, &chunk)
+ .map_err(|_| {
+ TransportError::Io("selector memory budget unavailable".into())
+ })?;
+ ingress_prefix.push(retained);
match selected {
Ok(Some(alias)) => break alias,
Ok(None) => {}
@@ -309,6 +345,8 @@ impl GatewayLifecycle for ProductionGatewayRuntime {
},
}
};
+ drop(selector);
+ drop(selector_charges);
let authorized = match authenticated
.authorize_alias(&served_model_id, Instant::now())
.map(Box::new)
@@ -324,6 +362,7 @@ impl GatewayLifecycle for ProductionGatewayRuntime {
declared_content_length,
ingress_prefix,
authorized,
+ budget,
)
.await
}
@@ -337,23 +376,12 @@ impl ProductionGatewayRuntime {
request: Box<GatewayRequestHead>,
protocol: IngressProtocol,
declared_content_length: Option<usize>,
- ingress_prefix: Vec<Bytes>,
+ ingress_prefix: Vec<ChargedBytes>,
authorized: Box<AuthorizedRequestPlan>,
+ budget: StreamBudget,
) -> Pin<Box<dyn Future<Output = Result<SessionReuse, TransportError>> + Send + 'a>> {
Box::pin(async move {
let continuation_scope = tool_continuation_scope(&authorized);
- let budget = match self.execution.allocate_bound_stream_budget() {
- Ok(budget) => budget,
- Err(_) => {
- return write_typed_error_phase(
- session,
- StatusCode::SERVICE_UNAVAILABLE,
- "REPLAY_BUDGET_UNAVAILABLE",
- "canonical_request",
- )
- .await;
- }
- };
let Some(replay_manager) = &self.replay else {
return write_typed_error_phase(
session,
@@ -1128,10 +1156,11 @@ impl ProductionGatewayRuntime {
let route_binding = frozen_candidates[0].binding;
let active_continuation = adapters::ActiveResponseDelivery::new(
protocol,
- crate::provider_state::ActiveProviderStates::new(
+ crate::provider_state::ActiveProviderStates::with_budget(
Arc::clone(&self.provider_states),
continuation_scope,
protocol,
+ budget.clone(),
),
);
request_observation.bind_tool_id_projection(active_continuation.tool_id_projection());
@@ -1522,7 +1551,7 @@ enum ReplayBodyError {
async fn read_replay_body(
session: &mut dyn GatewaySession,
- ingress_prefix: Vec<Bytes>,
+ ingress_prefix: Vec<ChargedBytes>,
declared_content_length: Option<usize>,
authorized: &AuthorizedRequestPlan,
replay: &ReplayStore,
@@ -1544,8 +1573,11 @@ async fn read_replay_body(
}
let mut body = replay.begin_raw().map_err(|_| ReplayBodyError::Replay)?;
for chunk in ingress_prefix {
- owner.admit_chunk(chunk.len()).map_err(map_body_error)?;
- body.append(&chunk).map_err(|_| ReplayBodyError::Replay)?;
+ owner
+ .admit_chunk(chunk.bytes().len())
+ .map_err(map_body_error)?;
+ body.append(chunk.bytes())
+ .map_err(|_| ReplayBodyError::Replay)?;
}
loop {
let chunk = session
@@ -1653,75 +1685,8 @@ async fn write_response(
}
#[cfg(test)]
-mod inbound_auth_tests {
- use super::*;
-
- #[test]
- fn x_api_key_never_authenticates_and_bearer_wins_when_both_are_present() {
- let mut headers = HeaderMap::new();
- headers.insert("x-api-key", HeaderValue::from_static("not-an-agent-grant"));
- assert_eq!(inbound_authorization("/v1/messages", &headers), None);
-
- headers.insert(
- AUTHORIZATION,
- HeaderValue::from_static("Bearer agent-grant"),
- );
- assert_eq!(
- inbound_authorization("/v1/messages", &headers).as_deref(),
- Some("Bearer agent-grant")
- );
- }
-
- #[test]
- fn codex_requires_independent_grant_without_native_bearer_fallback() {
- let mut headers = HeaderMap::new();
- headers.insert(
- AUTHORIZATION,
- HeaderValue::from_static("Bearer native-oauth"),
- );
- assert!(inbound_authorization("/v1/responses", &headers).is_none());
- headers.insert("x-hiroute-token", HeaderValue::from_static("model-grant"));
- assert_eq!(
- inbound_authorization("/v1/responses", &headers).as_deref(),
- Some("Bearer model-grant")
- );
- assert!(inbound_authorization("/v1/messages", &headers).is_none());
- headers.insert("x-hiroute-token", HeaderValue::from_static(""));
- assert!(inbound_authorization("/v1/responses", &headers).is_none());
- }
-
- #[test]
- fn repeated_or_combined_grant_headers_are_rejected() {
- let mut headers = HeaderMap::new();
- headers.append("x-hiroute-token", HeaderValue::from_static("first"));
- headers.append("x-hiroute-token", HeaderValue::from_static("second"));
- assert!(inbound_authorization("/v1/responses", &headers).is_none());
- headers.insert("x-hiroute-token", HeaderValue::from_static("first,second"));
- assert!(inbound_authorization("/v1/responses", &headers).is_none());
- headers.remove("x-hiroute-token");
- headers.append(AUTHORIZATION, HeaderValue::from_static("Bearer first"));
- headers.append(AUTHORIZATION, HeaderValue::from_static("Bearer second"));
- assert!(inbound_authorization("/v1/messages", &headers).is_none());
- }
-
- #[test]
- fn worker_run_authority_retains_its_separate_bearer_channel() {
- let mut headers = HeaderMap::new();
- headers.insert(
- AUTHORIZATION,
- HeaderValue::from_static("Bearer hr_run_model_fixture"),
- );
- assert_eq!(
- inbound_authorization("/v1/responses", &headers).as_deref(),
- Some("Bearer hr_run_model_fixture")
- );
- headers.insert(
- "x-hiroute-token",
- HeaderValue::from_static("hr_run_model_fixture"),
- );
- assert!(inbound_authorization("/v1/responses", &headers).is_none());
- }
-}
+#[path = "core_runtime/inbound_auth_tests.rs"]
+mod inbound_auth_tests;
#[cfg(test)]
#[path = "core_runtime/acceptance_tests.rs"]
diff --git a/crates/gateway/src/core_runtime/inbound_auth_tests.rs b/crates/gateway/src/core_runtime/inbound_auth_tests.rs
new file mode 100644
index 00000000..5341d4a7
--- /dev/null
+++ b/crates/gateway/src/core_runtime/inbound_auth_tests.rs
@@ -0,0 +1,67 @@
+use super::*;
+
+#[test]
+fn x_api_key_never_authenticates_and_bearer_wins_when_both_are_present() {
+ let mut headers = HeaderMap::new();
+ headers.insert("x-api-key", HeaderValue::from_static("not-an-agent-grant"));
+ assert_eq!(inbound_authorization("/v1/messages", &headers), None);
+
+ headers.insert(
+ AUTHORIZATION,
+ HeaderValue::from_static("Bearer agent-grant"),
+ );
+ assert_eq!(
+ inbound_authorization("/v1/messages", &headers).as_deref(),
+ Some("Bearer agent-grant")
+ );
+}
+
+#[test]
+fn codex_requires_independent_grant_without_native_bearer_fallback() {
+ let mut headers = HeaderMap::new();
+ headers.insert(
+ AUTHORIZATION,
+ HeaderValue::from_static("Bearer native-oauth"),
+ );
+ assert!(inbound_authorization("/v1/responses", &headers).is_none());
+ headers.insert("x-hiroute-token", HeaderValue::from_static("model-grant"));
+ assert_eq!(
+ inbound_authorization("/v1/responses", &headers).as_deref(),
+ Some("Bearer model-grant")
+ );
+ assert!(inbound_authorization("/v1/messages", &headers).is_none());
+ headers.insert("x-hiroute-token", HeaderValue::from_static(""));
+ assert!(inbound_authorization("/v1/responses", &headers).is_none());
+}
+
+#[test]
+fn repeated_or_combined_grant_headers_are_rejected() {
+ let mut headers = HeaderMap::new();
+ headers.append("x-hiroute-token", HeaderValue::from_static("first"));
+ headers.append("x-hiroute-token", HeaderValue::from_static("second"));
+ assert!(inbound_authorization("/v1/responses", &headers).is_none());
+ headers.insert("x-hiroute-token", HeaderValue::from_static("first,second"));
+ assert!(inbound_authorization("/v1/responses", &headers).is_none());
+ headers.remove("x-hiroute-token");
+ headers.append(AUTHORIZATION, HeaderValue::from_static("Bearer first"));
+ headers.append(AUTHORIZATION, HeaderValue::from_static("Bearer second"));
+ assert!(inbound_authorization("/v1/messages", &headers).is_none());
+}
+
+#[test]
+fn worker_run_authority_retains_its_separate_bearer_channel() {
+ let mut headers = HeaderMap::new();
+ headers.insert(
+ AUTHORIZATION,
+ HeaderValue::from_static("Bearer hr_run_model_fixture"),
+ );
+ assert_eq!(
+ inbound_authorization("/v1/responses", &headers).as_deref(),
+ Some("Bearer hr_run_model_fixture")
+ );
+ headers.insert(
+ "x-hiroute-token",
+ HeaderValue::from_static("hr_run_model_fixture"),
+ );
+ assert!(inbound_authorization("/v1/responses", &headers).is_none());
+}
diff --git a/crates/gateway/src/dispatch/authority.rs b/crates/gateway/src/dispatch/authority.rs
index 53e98978..36cc26a1 100644
--- a/crates/gateway/src/dispatch/authority.rs
+++ b/crates/gateway/src/dispatch/authority.rs
@@ -15,7 +15,8 @@ use crate::server::request_plan::{
AuthorizedRequestPlan, IngressProtocol, RequestAuthorityReceipt,
};
-const DEFAULT_SELECTOR_LIMIT: usize = 16 * 1024;
+// Selection must not depend on the byte position of the JSON model field.
+const DEFAULT_SELECTOR_LIMIT: usize = usize::MAX;
/// Time allowed to identify the top-level model alias. This ingress bound is
/// deliberately independent of every AgentPlan deadline: adding or removing
/// another alias cannot change how long an incomplete selector pins the
diff --git a/crates/gateway/src/dispatch/selector.rs b/crates/gateway/src/dispatch/selector.rs
index 743b65c0..1b92130c 100644
--- a/crates/gateway/src/dispatch/selector.rs
+++ b/crates/gateway/src/dispatch/selector.rs
@@ -33,7 +33,7 @@ impl ModelSelector {
Ok(None)
}
- pub fn finish(self) -> Result<String, SelectorError> {
+ pub fn finish(&self) -> Result<String, SelectorError> {
if let Extract::Found(model) = extract_top_level_model(&self.bytes)? {
return validate_model(model);
}
diff --git a/crates/gateway/src/dispatch/tests.rs b/crates/gateway/src/dispatch/tests.rs
index 099b80e4..f986fb6f 100644
--- a/crates/gateway/src/dispatch/tests.rs
+++ b/crates/gateway/src/dispatch/tests.rs
@@ -18,6 +18,25 @@ use crate::server::request_plan::IngressProtocol;
static DIRECTORY_SEQUENCE: AtomicUsize = AtomicUsize::new(0);
+#[test]
+fn model_selection_does_not_require_model_in_first_sixteen_kib() {
+ let (runtime, _, _directory) = runtime("127.0.0.1:45678".parse().unwrap());
+ let body = format!(
+ "{{\"input\":\"{}\",\"model\":\"fast\"}}",
+ "x".repeat(128 * 1024)
+ );
+ assert!(
+ runtime
+ .authorize_bytes(
+ IngressProtocol::Responses,
+ Some("Bearer token"),
+ body.as_bytes(),
+ Instant::now()
+ )
+ .is_ok()
+ );
+}
+
struct TestDirectory(PathBuf);
impl TestDirectory {
diff --git a/crates/gateway/src/model_ir/search.rs b/crates/gateway/src/model_ir/search.rs
index 3f5700e7..0b965171 100644
--- a/crates/gateway/src/model_ir/search.rs
+++ b/crates/gateway/src/model_ir/search.rs
@@ -117,7 +117,7 @@ impl WebSearchCallV1 {
}
let item: Self = serde_json::from_value(value)
.map_err(|_| super::ModelIrError::InvalidField("web_search_call"))?;
- if item.id.is_empty() || item.id.len() > 256 || item.id.chars().any(char::is_control) {
+ if item.id.is_empty() || item.id.chars().any(char::is_control) {
return Err(super::ModelIrError::InvalidField("web_search_call.id"));
}
Ok(item)
diff --git a/crates/gateway/src/planning/eligibility.rs b/crates/gateway/src/planning/eligibility.rs
index 0084ffd1..7855d527 100644
--- a/crates/gateway/src/planning/eligibility.rs
+++ b/crates/gateway/src/planning/eligibility.rs
@@ -3,8 +3,7 @@ use crate::server::core_runtime::model_ir::{
};
use crate::server::core_runtime::profiles::{
CandidateContextDemand, CandidateProtocolProfile, ContextProjectionError, ContextProjector,
- Fidelity, MAX_TERMINAL_CLASSIFIED_REFUSAL_BLOCKS, MAX_TERMINAL_CLASSIFIED_REFUSAL_BYTES,
- NativeProviderStateEmission, StateAffinity, StreamingRefusalSemantics,
+ Fidelity, NativeProviderStateEmission, StateAffinity, StreamingRefusalSemantics,
};
use crate::server::request_plan::IngressProtocol;
@@ -376,10 +375,8 @@ fn valid_stream_refusal(protocol: IngressProtocol, semantics: StreamingRefusalSe
(protocol, semantics),
(
IngressProtocol::Messages,
- StreamingRefusalSemantics::TerminalClassified {
- max_buffered_bytes: 1..=MAX_TERMINAL_CLASSIFIED_REFUSAL_BYTES,
- max_buffered_blocks: 1..=MAX_TERMINAL_CLASSIFIED_REFUSAL_BLOCKS,
- }
+ StreamingRefusalSemantics::TerminalClassified
+ | StreamingRefusalSemantics::LegacyTerminalClassified { .. }
) | (
IngressProtocol::Responses | IngressProtocol::ChatCompletions,
StreamingRefusalSemantics::ExactDelta
diff --git a/crates/gateway/src/profiles/capability.rs b/crates/gateway/src/profiles/capability.rs
index 2644fdc3..d64b66a8 100644
--- a/crates/gateway/src/profiles/capability.rs
+++ b/crates/gateway/src/profiles/capability.rs
@@ -16,8 +16,6 @@ use super::{
/// Exact bounded-delay contract for protocols that reveal refusal only in
/// their terminal metadata. The block bound also keeps one terminal replay
/// below the decoder's fixed event-drain window.
-pub const MAX_TERMINAL_CLASSIFIED_REFUSAL_BYTES: u64 = 16 * 1024 * 1024;
-pub const MAX_TERMINAL_CLASSIFIED_REFUSAL_BLOCKS: u32 = 8;
#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
@@ -129,17 +127,7 @@ pub enum NativeProviderStateEmission {
Unknown,
}
-#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
-#[serde(tag = "kind", rename_all = "snake_case")]
-pub enum StreamingRefusalSemantics {
- ExactDelta,
- TerminalClassified {
- max_buffered_bytes: u64,
- max_buffered_blocks: u32,
- },
- Unsupported,
- Unknown,
-}
+pub use hiroute_domain::GatewayStreamingRefusalSemanticsV1 as StreamingRefusalSemantics;
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
@@ -236,13 +224,7 @@ impl CandidateProtocolProfile {
provider_state: Fidelity::Unsupported,
state_affinity: StateAffinity::Unsupported,
stream_refusal: if upstream_protocol == IngressProtocol::Messages {
- StreamingRefusalSemantics::TerminalClassified {
- max_buffered_bytes: MAX_TERMINAL_CLASSIFIED_REFUSAL_BYTES,
- // Messages supplies refusal only with its terminal
- // stop reason, so the decoder defers a bounded
- // native block set until that classification.
- max_buffered_blocks: MAX_TERMINAL_CLASSIFIED_REFUSAL_BLOCKS,
- }
+ StreamingRefusalSemantics::TerminalClassified
} else {
StreamingRefusalSemantics::ExactDelta
},
@@ -479,10 +461,8 @@ impl CandidateProtocolProfile {
(self.capability.upstream_protocol, response.stream_refusal),
(
IngressProtocol::Messages,
- StreamingRefusalSemantics::TerminalClassified {
- max_buffered_bytes: 1..=MAX_TERMINAL_CLASSIFIED_REFUSAL_BYTES,
- max_buffered_blocks: 1..=MAX_TERMINAL_CLASSIFIED_REFUSAL_BLOCKS
- }
+ StreamingRefusalSemantics::TerminalClassified
+ | StreamingRefusalSemantics::LegacyTerminalClassified { .. }
) | (
IngressProtocol::Responses | IngressProtocol::ChatCompletions,
StreamingRefusalSemantics::ExactDelta
@@ -724,6 +704,37 @@ pub fn fixed_reasoning(profile_id: impl Into<String>) -> ReasoningProfileCapabil
mod tests {
use super::*;
+ #[test]
+ fn terminal_classification_contract_contains_no_proxy_size_quota() {
+ let expected = serde_json::json!({"kind":"terminal_classified"});
+ assert_eq!(
+ serde_json::to_value(StreamingRefusalSemantics::TerminalClassified).unwrap(),
+ expected
+ );
+ assert_eq!(
+ serde_json::to_value(
+ hiroute_domain::GatewayStreamingRefusalSemanticsV1::TerminalClassified
+ )
+ .unwrap(),
+ expected
+ );
+ let obsolete = serde_json::json!({"kind":"terminal_classified","max_buffered_bytes":16777216,"max_buffered_blocks":8});
+ let recovered =
+ serde_json::from_value::<StreamingRefusalSemantics>(obsolete.clone()).unwrap();
+ assert!(matches!(
+ recovered,
+ StreamingRefusalSemantics::LegacyTerminalClassified { .. }
+ ));
+ // Recovery must preserve the original capability digest.
+ assert_eq!(serde_json::to_value(recovered).unwrap(), obsolete);
+ assert!(
+ serde_json::from_value::<StreamingRefusalSemantics>(
+ serde_json::json!({"kind":"terminal_classified","unregistered_quota":8})
+ )
+ .is_err()
+ );
+ }
+
#[test]
fn client_profile_preserves_only_explicitly_representable_exact_owner_state() {
let mut candidate = CandidateProtocolProfile::exact_portable_path(
diff --git a/crates/gateway/src/provider_state.rs b/crates/gateway/src/provider_state.rs
index 1d1ec781..57f1218f 100644
--- a/crates/gateway/src/provider_state.rs
+++ b/crates/gateway/src/provider_state.rs
@@ -4,6 +4,7 @@ use std::collections::BTreeMap;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
+use hiroute_gateway_core::runtime::body::{MemoryRole, Reservation, StreamBudget};
use serde_json::Value;
use sha2::{Digest, Sha256};
@@ -12,8 +13,6 @@ use crate::server::request_plan::IngressProtocol;
const CAPACITY: usize = 4096;
const IDLE_TTL: Duration = Duration::from_secs(3600);
-const MAX_PATTERN: usize = 256 * 1024;
-const MAX_PENDING_BYTES: usize = 4 * 1024 * 1024;
/// Authorization scope for opaque provider state, never for ordinary tool IDs.
#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub(crate) struct ProviderStateScopeV1 {
@@ -127,10 +126,34 @@ fn digest(state: &str) -> [u8; 32] {
Sha256::digest(state.as_bytes()).into()
}
+// Counting writer uses the same escaping rules as the actual serializer.
+struct JsonLength(usize);
+
+impl std::io::Write for JsonLength {
+ fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
+ self.0 = self
+ .0
+ .checked_add(bytes.len())
+ .ok_or_else(|| std::io::Error::other("serialized provider state length overflow"))?;
+ Ok(bytes.len())
+ }
+
+ fn flush(&mut self) -> std::io::Result<()> {
+ Ok(())
+ }
+}
+
+pub(crate) fn json_string_length(value: &str) -> Result<usize, ModelIrError> {
+ let mut length = JsonLength(0);
+ serde_json::to_writer(&mut length, value).map_err(|_| ModelIrError::BufferLimit(usize::MAX))?;
+ Ok(length.0)
+}
+
struct Pending {
digest: [u8; 32],
owner: ExactProviderPathV1,
pattern: Vec<u8>,
+ charge: Arc<Reservation>,
}
#[derive(Clone)]
@@ -138,23 +161,49 @@ pub(crate) struct ActiveProviderStates {
store: Arc<ProviderStateStore>,
scope: ProviderStateScopeV1,
pending: Arc<Mutex<Vec<Pending>>>,
+ budget: StreamBudget,
downstream: IngressProtocol,
}
impl ActiveProviderStates {
+ #[cfg(test)]
pub(crate) fn new(
store: Arc<ProviderStateStore>,
scope: ProviderStateScopeV1,
downstream: IngressProtocol,
+ ) -> Self {
+ let tree = hiroute_gateway_core::runtime::body::BudgetTree::new(
+ 64 * 1024 * 1024,
+ 64 * 1024 * 1024,
+ )
+ .unwrap();
+ Self::with_budget(
+ store,
+ scope,
+ downstream,
+ tree.stream(64 * 1024 * 1024).unwrap(),
+ )
+ }
+
+ pub(crate) fn with_budget(
+ store: Arc<ProviderStateStore>,
+ scope: ProviderStateScopeV1,
+ downstream: IngressProtocol,
+ budget: StreamBudget,
) -> Self {
Self {
store,
scope,
pending: Arc::new(Mutex::new(Vec::new())),
+ budget,
downstream,
}
}
+ pub(crate) fn budget(&self) -> &StreamBudget {
+ &self.budget
+ }
+
pub(crate) fn record(
&self,
state: &Value,
@@ -164,35 +213,43 @@ impl ActiveProviderStates {
.as_str()
.filter(|s| !s.is_empty())
.ok_or(ModelIrError::InvalidField("encrypted_content"))?;
- // Responses ciphertext is surfaced as a thinking signature for Messages.
- // Bind the actual downstream representation, not the upstream field name.
- // Quotes inside user text are escaped and cannot activate this fragment.
- let field = match self.downstream {
- IngressProtocol::Responses => "encrypted_content",
- IngressProtocol::Messages => "signature",
+ let prefix: &[u8] = match self.downstream {
+ IngressProtocol::Responses => b"\"encrypted_content\":",
+ IngressProtocol::Messages => b"\"signature\":",
_ => return Err(ModelIrError::ProviderStateNotPortable),
};
- let pattern = format!(
- "\"{field}\":{}",
- serde_json::to_string(state)
- .map_err(|_| ModelIrError::InvalidField("encrypted_content"))?
- )
- .into_bytes();
- self.record_at_acceptance(state, owner, pattern)
+ let length = json_string_length(state)?
+ .checked_add(prefix.len())
+ .ok_or(ModelIrError::BufferLimit(usize::MAX))?;
+ self.record_pattern(state, owner, length, |pattern| {
+ pattern.extend_from_slice(prefix);
+ serde_json::to_writer(pattern, state)
+ .map_err(|_| ModelIrError::InvalidField("encrypted_content"))
+ })
}
pub(crate) fn record_at_acceptance(
&self,
state: &str,
owner: &ExactProviderPathV1,
- pattern: Vec<u8>,
+ pattern: &[u8],
) -> Result<(), ModelIrError> {
- if state.is_empty() || state.len() > MAX_PATTERN || pattern.is_empty() {
+ if state.is_empty() || pattern.is_empty() {
return Err(ModelIrError::InvalidField("provider_state"));
}
- if pattern.len() > MAX_PATTERN {
- return Err(ModelIrError::BufferLimit(MAX_PATTERN));
- }
+ self.record_pattern(state, owner, pattern.len(), |out| {
+ out.extend_from_slice(pattern);
+ Ok(())
+ })
+ }
+
+ fn record_pattern(
+ &self,
+ state: &str,
+ owner: &ExactProviderPathV1,
+ length: usize,
+ fill: impl FnOnce(&mut Vec<u8>) -> Result<(), ModelIrError>,
+ ) -> Result<(), ModelIrError> {
let digest = digest(state);
let mut pending = self.pending.lock().unwrap_or_else(|e| e.into_inner());
if pending
@@ -201,16 +258,22 @@ impl ActiveProviderStates {
{
return Ok(());
}
- if pending.len() >= 128
- || pending.iter().map(|p| p.pattern.len()).sum::<usize>() + pattern.len()
- > MAX_PENDING_BYTES
- {
- return Err(ModelIrError::BufferLimit(MAX_PENDING_BYTES));
- }
+ // Charge pattern and scanner carry/boundary copies before allocating.
+ let bytes = length
+ .checked_mul(8)
+ .and_then(|bytes| bytes.checked_add(std::mem::size_of::<Pending>()))
+ .ok_or(ModelIrError::BufferLimit(usize::MAX))?;
+ let charge = self
+ .budget
+ .reserve(MemoryRole::SemanticState, bytes)
+ .map_err(|_| ModelIrError::BufferLimit(bytes))?;
+ let mut pattern = Vec::with_capacity(length);
+ fill(&mut pattern)?;
pending.push(Pending {
digest,
owner: owner.clone(),
pattern,
+ charge: Arc::new(charge),
});
Ok(())
}
@@ -219,6 +282,7 @@ impl ActiveProviderStates {
AcceptedProviderStateScanner {
active: self.clone(),
carry: Vec::new(),
+ carry_charge: None,
}
}
}
@@ -226,6 +290,7 @@ impl ActiveProviderStates {
pub(crate) struct AcceptedProviderStateScanner {
active: ActiveProviderStates,
carry: Vec<u8>,
+ carry_charge: Option<Arc<Reservation>>,
}
impl AcceptedProviderStateScanner {
@@ -240,6 +305,10 @@ impl AcceptedProviderStateScanner {
.map(|p| p.pattern.len().saturating_sub(1))
.max()
.unwrap_or(0);
+ let next_charge = pending
+ .iter()
+ .max_by_key(|p| p.pattern.len())
+ .map(|p| Arc::clone(&p.charge));
let mut boundary = self.carry.clone();
boundary.extend_from_slice(&bytes[..bytes.len().min(keep)]);
pending.retain(|p| {
@@ -257,6 +326,7 @@ impl AcceptedProviderStateScanner {
let start = boundary.len().saturating_sub(keep);
self.carry = boundary[start..].to_vec();
}
+ self.carry_charge = next_charge;
}
}
@@ -296,6 +366,70 @@ mod tests {
json!({"input":[{"type":"reasoning","encrypted_content":state}]})
}
+ #[test]
+ fn provider_state_admission_counts_escaping_and_deduplicates() {
+ let state = "\"\\\n\u{0000}雪".repeat(128);
+ let mut length = JsonLength(0);
+ serde_json::to_writer(&mut length, &state).unwrap();
+ assert_eq!(length.0, serde_json::to_vec(&state).unwrap().len());
+ let bytes =
+ (b"\"encrypted_content\":".len() + length.0) * 8 + std::mem::size_of::<Pending>();
+ let tree =
+ hiroute_gateway_core::runtime::body::BudgetTree::new(bytes * 2, bytes * 2).unwrap();
+ let store = Arc::new(ProviderStateStore::default());
+ let rejected = ActiveProviderStates::with_budget(
+ store.clone(),
+ scope(),
+ IngressProtocol::Responses,
+ tree.stream(bytes - 1).unwrap(),
+ );
+ assert!(matches!(
+ rejected.record(&json!(state), &owner("luna")),
+ Err(ModelIrError::BufferLimit(_))
+ ));
+ assert!(rejected.pending.lock().unwrap().is_empty());
+ let accepted = ActiveProviderStates::with_budget(
+ store,
+ scope(),
+ IngressProtocol::Responses,
+ tree.stream(bytes).unwrap(),
+ );
+ accepted.record(&json!(state), &owner("luna")).unwrap();
+ accepted.record(&json!(state), &owner("luna")).unwrap();
+ assert_eq!(accepted.pending.lock().unwrap().len(), 1);
+ }
+
+ #[test]
+ fn native_signature_acceptance_has_no_fixed_state_or_pending_count_limit() {
+ let store = Arc::new(ProviderStateStore::default());
+ let active = ActiveProviderStates::new(store.clone(), scope(), IngressProtocol::Messages);
+ let now = Instant::now();
+ let large = "s".repeat(300 * 1024);
+ for index in 0..129 {
+ let state = format!("{large}{index}");
+ let closing = format!(
+ "event: content_block_stop\ndata: {{\"index\":{index},\"type\":\"content_block_stop\"}}\n\n"
+ );
+ active
+ .record_at_acceptance(&state, &owner("luna"), closing.as_bytes())
+ .unwrap();
+ }
+ assert_eq!(active.pending.lock().unwrap().len(), 129);
+ let state = format!("{large}128");
+ let replay = json!({"messages":[{"role":"assistant","content":[
+ {"type":"thinking","thinking":"","signature":state}
+ ]}]});
+ assert!(store.resolve(&scope(), &replay, now).is_err());
+ active.scanner().accept_bytes(
+ b"event: content_block_stop\ndata: {\"index\":128,\"type\":\"content_block_stop\"}\n\n",
+ now,
+ );
+ assert_eq!(
+ store.resolve(&scope(), &replay, now).unwrap(),
+ Some(owner("luna"))
+ );
+ }
+
#[test]
fn fragmented_native_signature_requires_accepted_matching_block_close() {
let store = Arc::new(ProviderStateStore::default());
@@ -306,7 +440,7 @@ mod tests {
]}]});
let closing = br#"{ "index": 0, "type": "content_block_stop" }"#;
active
- .record_at_acceptance("combined-signature", &owner("luna"), closing.to_vec())
+ .record_at_acceptance("combined-signature", &owner("luna"), closing)
.unwrap();
let mut scanner = active.scanner();
scanner.accept_bytes(br#"{"signature":"combined-signature"}"#, now);
@@ -410,10 +544,17 @@ mod tests {
store.resolve(&scope(), &request("one"), now),
Err(ModelIrError::ProviderStateNotPortable)
);
- assert!(
- ActiveProviderStates::new(store, scope(), IngressProtocol::Responses)
- .record(&json!("x".repeat(MAX_PATTERN)), &owner("luna"))
- .is_err()
+ let large = "x".repeat(300 * 1024);
+ let active = ActiveProviderStates::new(store.clone(), scope(), IngressProtocol::Responses);
+ active.record(&json!(large), &owner("luna")).unwrap();
+ let wire = serde_json::to_vec(&json!({"encrypted_content":large})).unwrap();
+ let mut scanner = active.scanner();
+ for chunk in wire.chunks(8191) {
+ scanner.accept_bytes(chunk, now);
+ }
+ assert_eq!(
+ store.resolve(&scope(), &request(&large), now).unwrap(),
+ Some(owner("luna"))
);
}
diff --git a/crates/gateway/src/publication/compiler.rs b/crates/gateway/src/publication/compiler.rs
index e8477fee..8b875c31 100644
--- a/crates/gateway/src/publication/compiler.rs
+++ b/crates/gateway/src/publication/compiler.rs
@@ -39,7 +39,9 @@ use crate::server::request_plan::{
ProviderCandidateAuthority, RequestPriceBindingV1, RestBranchClassifierAuthorityV1,
};
-const LOGICAL_BODY_LIMIT: usize = 1024 * 1024;
+// Replay spills; allocation is governed by the request memory budget.
+// The body plan carries no additional protocol byte limit.
+const LOGICAL_BODY_LIMIT: usize = usize::MAX;
const FRAME_LIMIT: usize = 64 * 1024;
const DEFAULT_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(20);
diff --git a/crates/gateway/src/publication/tests.rs b/crates/gateway/src/publication/tests.rs
index 2028ae6c..1af19c35 100644
--- a/crates/gateway/src/publication/tests.rs
+++ b/crates/gateway/src/publication/tests.rs
@@ -11,6 +11,25 @@ use crate::server::request_plan::IngressProtocol;
static DIRECTORY_SEQUENCE: AtomicUsize = AtomicUsize::new(0);
+#[test]
+fn production_body_plan_admits_large_requests_without_a_proxy_byte_quota() {
+ use hiroute_gateway_core::runtime::body::{BodyDirection, BodyPlanExecutor};
+ let aggregate = super::compiler::compile(&snapshot(1, "renderer")).unwrap();
+ let plan = &aggregate.aliases["alpha"]
+ .execution
+ .request_plan
+ .logical_request
+ .body_plan;
+ let mut owner =
+ BodyPlanExecutor::new(BodyDirection::LogicalRequest, plan.clone(), usize::MAX).unwrap();
+ let bytes = 32 * 1024 * 1024;
+ owner.preflight_content_length(bytes).unwrap();
+ for _ in 0..512 {
+ owner.admit_chunk(64 * 1024).unwrap();
+ }
+ assert_eq!(owner.finish().unwrap(), bytes);
+}
+
pub(super) struct TestDirectory(PathBuf);
impl TestDirectory {
diff --git a/crates/gateway/src/replay.rs b/crates/gateway/src/replay.rs
index 25b82ab6..3329aa10 100644
--- a/crates/gateway/src/replay.rs
+++ b/crates/gateway/src/replay.rs
@@ -31,8 +31,6 @@ use stream::{ReplayStream, ReplayStreamBuilder};
const DEFAULT_MEMORY_THRESHOLD: usize = 10 * 1024 * 1024;
const DEFAULT_RECORD_BYTES: usize = 16 * 1024;
const DEFAULT_ORPHAN_TTL: Duration = Duration::from_secs(24 * 60 * 60);
-const MAX_STREAMS_PER_REQUEST: u64 = 8;
-const MAX_RANGES_PER_STREAM: usize = 16_384;
const STREAM_METADATA_BYTES: usize = 1024;
const RANGE_METADATA_BYTES: usize = 256;
@@ -46,8 +44,13 @@ pub struct ReplayConfig {
impl ReplayConfig {
pub fn production_default() -> Self {
+ // System temp directories may be aliases (for example /tmp on Linux or /var
+ // on macOS). Resolve that base, while leaving the replay leaf itself subject
+ // to SecureRoot's owner, permission, and no-symlink checks.
+ let temporary = std::env::temp_dir();
+ let temporary = std::fs::canonicalize(&temporary).unwrap_or(temporary);
Self {
- root: std::env::temp_dir().join("hiroute-replay"),
+ root: temporary.join("hiroute-replay"),
memory_threshold_bytes: DEFAULT_MEMORY_THRESHOLD,
record_bytes: DEFAULT_RECORD_BYTES,
orphan_ttl: DEFAULT_ORPHAN_TTL,
@@ -314,9 +317,6 @@ impl ReplayStore {
.fetch_add(1, Ordering::AcqRel)
.checked_add(1)
.ok_or(ReplayError::LengthOverflow)?;
- if ordinal > MAX_STREAMS_PER_REQUEST {
- return Err(ReplayError::StructureLimit);
- }
ReplayStreamBuilder::new(
Arc::clone(&self.inner.directory),
ordinal,
@@ -425,9 +425,6 @@ impl ReplayContentWriter {
}
fn reserve_range(&self) -> Result<(), ReplayError> {
- if self.ranges.len() >= MAX_RANGES_PER_STREAM {
- return Err(ReplayError::StructureLimit);
- }
self.store.charge_metadata(RANGE_METADATA_BYTES)
}
diff --git a/crates/gateway/src/replay/tests.rs b/crates/gateway/src/replay/tests.rs
index 86f77701..4fd6a013 100644
--- a/crates/gateway/src/replay/tests.rs
+++ b/crates/gateway/src/replay/tests.rs
@@ -67,6 +67,46 @@ fn production_default_retains_up_to_ten_mib() {
assert_eq!(config.record_bytes, 16 * 1024);
}
+#[cfg(unix)]
+#[test]
+fn production_default_supports_symlinked_system_temp() {
+ const CHILD: &str = "HIROUTE_REPLAY_SYMLINK_TEMP_TEST";
+ if std::env::var_os(CHILD).is_some() {
+ let manager = ReplayManager::from_environment().expect("default replay root");
+ let store = manager.begin_request(budget()).expect("request backing");
+ let mut writer = store.begin_raw().expect("writer");
+ writer.append(b"native client request").expect("append");
+ let reference = writer.seal().expect("seal");
+ assert_eq!(read_all(&store, &reference), b"native client request");
+ return;
+ }
+ let root = TestRoot::new("system-temp-link");
+ fs::create_dir_all(root.0.join("actual")).expect("temporary directory");
+ let actual = fs::canonicalize(root.0.join("actual")).expect("resolved temp");
+ let link = root.0.join("system-temp");
+ std::os::unix::fs::symlink(&actual, &link).expect("system temp alias");
+ let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
+ .args([
+ "--exact",
+ "replay::tests::production_default_supports_symlinked_system_temp",
+ "--nocapture",
+ ])
+ .env(CHILD, "1")
+ .env("TMPDIR", &link)
+ .env_remove("HIROUTE_REPLAY_ROOT")
+ .env_remove("HIROUTE_REPLAY_MEMORY_THRESHOLD")
+ .env_remove("HIROUTE_REPLAY_RECORD_BYTES")
+ .env_remove("HIROUTE_REPLAY_ORPHAN_TTL_MS")
+ .output()
+ .expect("isolated default-path check");
+ assert!(
+ output.status.success(),
+ "default replay failed with a system temp alias: {} {}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+}
+
#[test]
fn small_stream_retains_charged_capacity_and_has_independent_readers() {
let root = TestRoot::new("memory");
@@ -116,6 +156,30 @@ fn small_stream_retains_charged_capacity_and_has_independent_readers() {
assert_eq!(stream_budget.snapshot().expect("snapshot").live, 0);
}
+#[test]
+fn replay_stream_and_range_counts_use_owned_memory_without_fixed_quotas() {
+ let root = TestRoot::new("metadata-counts");
+ let manager = ReplayManager::open(config(&root.0, 64 * 1024, 4 * 1024)).unwrap();
+ let stream_budget = budget();
+ let store = manager.begin_request(stream_budget.clone()).unwrap();
+ for _ in 0..9 {
+ let mut writer = store.begin_raw().unwrap();
+ writer.append(b"raw").unwrap();
+ let reference = writer.seal().unwrap();
+ assert_eq!(read_all(&store, &reference), b"raw");
+ }
+ let mut writer = store.begin_content_pool().unwrap();
+ let mut last = None;
+ for _ in 0..16_385 {
+ last = Some(writer.append(b"range").unwrap());
+ }
+ writer.seal().unwrap();
+ assert_eq!(read_all(&store, &last.unwrap()), b"range");
+ assert!(stream_budget.snapshot().unwrap().live > 16_384 * super::RANGE_METADATA_BYTES);
+ drop(store);
+ assert_eq!(stream_budget.snapshot().unwrap().live, 0);
+}
+
#[test]
fn large_stream_uses_one_plaintext_backing_for_multiple_readers() {
let root = TestRoot::new("disk");
@@ -344,7 +408,7 @@ fn replay_handle_relative_rename_and_symlink_race_cannot_escape() {
}
#[test]
-fn replay_ingress_structure_gate_rejects_ten_thousand_fields_before_content_pool() {
+fn replay_ingress_structure_is_budgeted_without_a_fixed_field_count_limit() {
let root = TestRoot::new("structure-gate");
let manager = ReplayManager::open(config(&root.0, 128, 64)).expect("manager");
let store = manager.begin_request(budget()).expect("store");
@@ -362,9 +426,29 @@ fn replay_ingress_structure_gate_rejects_ten_thousand_fields_before_content_pool
writer.append(&body).expect("append raw");
let raw = writer.seal().expect("seal raw");
- let error = crate::content_ref::scan_ingress_document(store.reader(&raw).expect("raw reader"))
- .expect_err("structure limit");
- assert!(matches!(error, ReplayError::StructureLimit));
+ let stats = crate::content_ref::scan_ingress_document(store.reader(&raw).expect("raw reader"))
+ .expect("many fields are not a protocol error");
+ let large = manager
+ .begin_request(
+ BudgetTree::new(64 * 1024 * 1024, 64 * 1024 * 1024)
+ .unwrap()
+ .stream(64 * 1024 * 1024)
+ .unwrap(),
+ )
+ .unwrap();
+ let workspace = stats
+ .reserve_workspace(&large, raw.byte_len())
+ .expect("sufficient memory");
+ assert!(workspace.bytes() > 16_384 * 128);
+ let small = manager
+ .begin_request(
+ BudgetTree::new(1024 * 1024, 1024 * 1024)
+ .unwrap()
+ .stream(1024 * 1024)
+ .unwrap(),
+ )
+ .unwrap();
+ assert!(stats.reserve_workspace(&small, raw.byte_len()).is_err());
assert_eq!(
store.snapshot().live_streams,
1,
diff --git a/crates/gateway/src/runtime/driver.rs b/crates/gateway/src/runtime/driver.rs
index ccca6343..6df89b6a 100644
--- a/crates/gateway/src/runtime/driver.rs
+++ b/crates/gateway/src/runtime/driver.rs
@@ -56,7 +56,6 @@ const MATERIALIZATION_AUTHORITY_FAILED: &str = "authority_failed";
const RUNTIME_STATE_AUTHORITY_FAILED: &str = "runtime_state_authority_failed";
const MATERIALIZATION_DNS_FAILED: &str = "dns_failed";
const MATERIALIZATION_PROTOCOL_FAILED: &str = "protocol_failed";
-const MAX_RENDERED_PRECOMMIT_BYTES: usize = 256 * 1024;
pub struct ProductionProvider {
credentials: Arc<dyn ProductionCredentialResolver>,
diff --git a/crates/gateway/src/runtime/driver/materialization.rs b/crates/gateway/src/runtime/driver/materialization.rs
index fd353318..b56f534c 100644
--- a/crates/gateway/src/runtime/driver/materialization.rs
+++ b/crates/gateway/src/runtime/driver/materialization.rs
@@ -43,8 +43,8 @@ use super::{
MATERIALIZATION_BINDING_UNAVAILABLE, MATERIALIZATION_CREDENTIAL_COOLING_PREFIX,
MATERIALIZATION_CREDENTIAL_DISABLED, MATERIALIZATION_CREDENTIAL_EXHAUSTED,
MATERIALIZATION_CREDENTIAL_PROBE_BUSY, MATERIALIZATION_DNS_FAILED,
- MATERIALIZATION_PROTOCOL_FAILED, MAX_RENDERED_PRECOMMIT_BYTES, MechanicalFailureSignal,
- PermitAvailability, ProductionAttemptState, ProductionLogicalRequest, ProductionProvider,
+ MATERIALIZATION_PROTOCOL_FAILED, MechanicalFailureSignal, PermitAvailability,
+ ProductionAttemptState, ProductionLogicalRequest, ProductionProvider,
ProductionReplaySeedEnvelope, RUNTIME_STATE_AUTHORITY_FAILED, RuntimeCooldownPolicy,
StatePermit, acquire_target_permit_status, confirm_success, label, record_failure, safe_error,
validate_attempt_permits,
@@ -279,6 +279,7 @@ pub(super) async fn materialize_attempt(
client_profile.clone(),
served_model_alias.clone(),
)
+ .map(|renderer| renderer.with_budget(context.budget.clone()))
})
.transpose()
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?;
@@ -286,7 +287,7 @@ pub(super) async fn materialize_attempt(
context.budget,
MemoryRole::ResponsePrefix,
&plan.body_plans.attempt_response_precommit,
- MAX_RENDERED_PRECOMMIT_BYTES,
+ usize::MAX,
plan.precommit_event_capacity,
)
.map_err(safe_error)?;
@@ -903,198 +904,5 @@ fn profile_digest(profile: &CandidateProtocolProfile) -> Result<String, Arc<str>
}
#[cfg(test)]
-mod tests {
- use std::time::Duration;
-
- use crate::attempt_outcome::AttemptFailureClass;
- use crate::ports::{CasOutcome, InMemoryRuntimeStateStore, ProbeLeaseOutcome, RuntimeHealth};
- use crate::runtime::state::acquire_target_permit;
- use crate::server::core_runtime::profiles::fixed_reasoning;
-
- use super::*;
-
- #[test]
- fn dns_lookup_uses_scheme_default_and_preserves_explicit_port() {
- assert_eq!(
- dns_lookup_target("provider.example", TransportScheme::Https).unwrap(),
- ("provider.example".to_owned(), 443)
- );
- assert_eq!(
- dns_lookup_target("provider.example:8443", TransportScheme::Https).unwrap(),
- ("provider.example".to_owned(), 8443)
- );
- assert_eq!(
- dns_lookup_target("provider.example", TransportScheme::Http).unwrap(),
- ("provider.example".to_owned(), 80)
- );
- }
-
- async fn active_permits(
- store: &dyn RuntimeStateStore,
- ) -> super::super::OwnedAttemptStatePermits {
- let scope = ExecutionScope::new(
- Instant::now() + Duration::from_secs(1),
- tokio_util::sync::CancellationToken::new(),
- );
- let binding_key = RuntimeStateKey::binding("mechanical-binding");
- let binding_permit = acquire_target_permit(store, &binding_key, &scope)
- .await
- .unwrap()
- .unwrap();
- let credential_key =
- RuntimeStateKey::credential("mechanical-binding", "credential", "key", 1);
- let credential_permit = acquire_target_permit(store, &credential_key, &scope)
- .await
- .unwrap()
- .unwrap();
- super::super::OwnedAttemptStatePermits::new(
- binding_key,
- binding_permit,
- credential_key,
- credential_permit,
- )
- }
-
- fn production_profile() -> CandidateProtocolProfile {
- let mut profile = CandidateProtocolProfile::exact_portable_path(
- crate::server::request_plan::IngressProtocol::Responses,
- crate::server::request_plan::IngressProtocol::Responses,
- "native-model",
- fixed_reasoning("fixed"),
- );
- profile.connector.connector_id = "builtin-openai".into();
- profile
- }
-
- #[tokio::test]
- async fn connect_first_byte_and_idle_failures_cool_exact_binding() {
- for (class, timeout, expected_class) in [
- (
- CoreAttemptFailureClass::Connect,
- None,
- AttemptFailureClass::Transient,
- ),
- (
- CoreAttemptFailureClass::Connect,
- Some(AttemptTimeoutKind::Connect),
- AttemptFailureClass::Timeout(crate::attempt_outcome::AttemptTimeoutPhase::Connect),
- ),
- (
- CoreAttemptFailureClass::FirstByte,
- Some(AttemptTimeoutKind::FirstByte),
- AttemptFailureClass::Timeout(
- crate::attempt_outcome::AttemptTimeoutPhase::FirstByte,
- ),
- ),
- (
- CoreAttemptFailureClass::StreamIdle,
- Some(AttemptTimeoutKind::StreamIdle),
- AttemptFailureClass::Timeout(
- crate::attempt_outcome::AttemptTimeoutPhase::StreamIdle,
- ),
- ),
- ] {
- let store = InMemoryRuntimeStateStore::default();
- let permits = active_permits(&store).await;
- let cancellation = tokio_util::sync::CancellationToken::new();
- let authority = super::super::RuntimeStateAuthoritySignal::default();
- let confirmed = confirm_mechanical_state(
- &store,
- &permits,
- &production_profile(),
- &RuntimeCooldownPolicy::default(),
- MechanicalFailureSignal {
- class,
- timeout,
- runtime_state_authority: authority,
- },
- Instant::now() + Duration::from_secs(1),
- &cancellation,
- )
- .await
- .unwrap()
- .expect("mechanical failure is classified");
-
- assert_eq!(confirmed.1.retryability, RetryabilityFact::Retryable);
- assert_eq!(confirmed.0.class, expected_class);
- let binding = store.entry(&RuntimeStateKey::binding("mechanical-binding"));
- assert_eq!(binding.generation, 1);
- assert!(matches!(binding.health, RuntimeHealth::CoolingDown { .. }));
- assert_eq!(
- store
- .entry(&RuntimeStateKey::credential(
- "mechanical-binding",
- "credential",
- "key",
- 1,
- ))
- .generation,
- 0,
- "mechanical failures must not cool the credential"
- );
- }
- }
-
- struct FailingCasStore;
-
- #[async_trait]
- impl ProductionStore for FailingCasStore {
- async fn read_exact(
- &self,
- _key: &RuntimeStateKey,
- _scope: &ExecutionScope,
- ) -> Result<RuntimeStateEntry, PortError> {
- Ok(RuntimeStateEntry::default())
- }
-
- async fn compare_and_swap_exact(
- &self,
- _key: &RuntimeStateKey,
- _expected_generation: u64,
- _next: RuntimeStateEntry,
- _scope: &ExecutionScope,
- ) -> Result<CasOutcome, PortError> {
- Err(PortError::Unavailable("external RuntimeStateStore"))
- }
-
- async fn acquire_probe_lease_exact(
- &self,
- _key: &RuntimeStateKey,
- _expected_generation: u64,
- _now: Instant,
- _lease_duration: Duration,
- _scope: &ExecutionScope,
- ) -> Result<ProbeLeaseOutcome, PortError> {
- Err(PortError::Unavailable("external RuntimeStateStore"))
- }
- }
-
- #[tokio::test]
- async fn mechanical_failure_state_write_error_is_fail_closed() {
- let store = ProductionStateAdapter {
- inner: Arc::new(FailingCasStore),
- };
- let permits = active_permits(&store).await;
- let cancellation = tokio_util::sync::CancellationToken::new();
- let authority = super::super::RuntimeStateAuthoritySignal::default();
-
- let error = confirm_mechanical_state(
- &store,
- &permits,
- &production_profile(),
- &RuntimeCooldownPolicy::default(),
- MechanicalFailureSignal {
- class: CoreAttemptFailureClass::Connect,
- timeout: None,
- runtime_state_authority: authority.clone(),
- },
- Instant::now() + Duration::from_secs(1),
- &cancellation,
- )
- .await
- .unwrap_err();
-
- assert_eq!(error.as_ref(), RUNTIME_STATE_AUTHORITY_FAILED);
- assert!(authority.failed());
- }
-}
+#[path = "materialization_tests.rs"]
+mod tests;
diff --git a/crates/gateway/src/runtime/driver/materialization_tests.rs b/crates/gateway/src/runtime/driver/materialization_tests.rs
new file mode 100644
index 00000000..c38e23e0
--- /dev/null
+++ b/crates/gateway/src/runtime/driver/materialization_tests.rs
@@ -0,0 +1,186 @@
+use std::time::Duration;
+
+use crate::attempt_outcome::AttemptFailureClass;
+use crate::ports::{CasOutcome, InMemoryRuntimeStateStore, ProbeLeaseOutcome, RuntimeHealth};
+use crate::runtime::state::acquire_target_permit;
+use crate::server::core_runtime::profiles::fixed_reasoning;
+
+use super::*;
+
+#[test]
+fn dns_lookup_uses_scheme_default_and_preserves_explicit_port() {
+ assert_eq!(
+ dns_lookup_target("provider.example", TransportScheme::Https).unwrap(),
+ ("provider.example".to_owned(), 443)
+ );
+ assert_eq!(
+ dns_lookup_target("provider.example:8443", TransportScheme::Https).unwrap(),
+ ("provider.example".to_owned(), 8443)
+ );
+ assert_eq!(
+ dns_lookup_target("provider.example", TransportScheme::Http).unwrap(),
+ ("provider.example".to_owned(), 80)
+ );
+}
+
+async fn active_permits(store: &dyn RuntimeStateStore) -> super::super::OwnedAttemptStatePermits {
+ let scope = ExecutionScope::new(
+ Instant::now() + Duration::from_secs(1),
+ tokio_util::sync::CancellationToken::new(),
+ );
+ let binding_key = RuntimeStateKey::binding("mechanical-binding");
+ let binding_permit = acquire_target_permit(store, &binding_key, &scope)
+ .await
+ .unwrap()
+ .unwrap();
+ let credential_key = RuntimeStateKey::credential("mechanical-binding", "credential", "key", 1);
+ let credential_permit = acquire_target_permit(store, &credential_key, &scope)
+ .await
+ .unwrap()
+ .unwrap();
+ super::super::OwnedAttemptStatePermits::new(
+ binding_key,
+ binding_permit,
+ credential_key,
+ credential_permit,
+ )
+}
+
+fn production_profile() -> CandidateProtocolProfile {
+ let mut profile = CandidateProtocolProfile::exact_portable_path(
+ crate::server::request_plan::IngressProtocol::Responses,
+ crate::server::request_plan::IngressProtocol::Responses,
+ "native-model",
+ fixed_reasoning("fixed"),
+ );
+ profile.connector.connector_id = "builtin-openai".into();
+ profile
+}
+
+#[tokio::test]
+async fn connect_first_byte_and_idle_failures_cool_exact_binding() {
+ for (class, timeout, expected_class) in [
+ (
+ CoreAttemptFailureClass::Connect,
+ None,
+ AttemptFailureClass::Transient,
+ ),
+ (
+ CoreAttemptFailureClass::Connect,
+ Some(AttemptTimeoutKind::Connect),
+ AttemptFailureClass::Timeout(crate::attempt_outcome::AttemptTimeoutPhase::Connect),
+ ),
+ (
+ CoreAttemptFailureClass::FirstByte,
+ Some(AttemptTimeoutKind::FirstByte),
+ AttemptFailureClass::Timeout(crate::attempt_outcome::AttemptTimeoutPhase::FirstByte),
+ ),
+ (
+ CoreAttemptFailureClass::StreamIdle,
+ Some(AttemptTimeoutKind::StreamIdle),
+ AttemptFailureClass::Timeout(crate::attempt_outcome::AttemptTimeoutPhase::StreamIdle),
+ ),
+ ] {
+ let store = InMemoryRuntimeStateStore::default();
+ let permits = active_permits(&store).await;
+ let cancellation = tokio_util::sync::CancellationToken::new();
+ let authority = super::super::RuntimeStateAuthoritySignal::default();
+ let confirmed = confirm_mechanical_state(
+ &store,
+ &permits,
+ &production_profile(),
+ &RuntimeCooldownPolicy::default(),
+ MechanicalFailureSignal {
+ class,
+ timeout,
+ runtime_state_authority: authority,
+ },
+ Instant::now() + Duration::from_secs(1),
+ &cancellation,
+ )
+ .await
+ .unwrap()
+ .expect("mechanical failure is classified");
+
+ assert_eq!(confirmed.1.retryability, RetryabilityFact::Retryable);
+ assert_eq!(confirmed.0.class, expected_class);
+ let binding = store.entry(&RuntimeStateKey::binding("mechanical-binding"));
+ assert_eq!(binding.generation, 1);
+ assert!(matches!(binding.health, RuntimeHealth::CoolingDown { .. }));
+ assert_eq!(
+ store
+ .entry(&RuntimeStateKey::credential(
+ "mechanical-binding",
+ "credential",
+ "key",
+ 1,
+ ))
+ .generation,
+ 0,
+ "mechanical failures must not cool the credential"
+ );
+ }
+}
+
+struct FailingCasStore;
+
+#[async_trait]
+impl ProductionStore for FailingCasStore {
+ async fn read_exact(
+ &self,
+ _key: &RuntimeStateKey,
+ _scope: &ExecutionScope,
+ ) -> Result<RuntimeStateEntry, PortError> {
+ Ok(RuntimeStateEntry::default())
+ }
+
+ async fn compare_and_swap_exact(
+ &self,
+ _key: &RuntimeStateKey,
+ _expected_generation: u64,
+ _next: RuntimeStateEntry,
+ _scope: &ExecutionScope,
+ ) -> Result<CasOutcome, PortError> {
+ Err(PortError::Unavailable("external RuntimeStateStore"))
+ }
+
+ async fn acquire_probe_lease_exact(
+ &self,
+ _key: &RuntimeStateKey,
+ _expected_generation: u64,
+ _now: Instant,
+ _lease_duration: Duration,
+ _scope: &ExecutionScope,
+ ) -> Result<ProbeLeaseOutcome, PortError> {
+ Err(PortError::Unavailable("external RuntimeStateStore"))
+ }
+}
+
+#[tokio::test]
+async fn mechanical_failure_state_write_error_is_fail_closed() {
+ let store = ProductionStateAdapter {
+ inner: Arc::new(FailingCasStore),
+ };
+ let permits = active_permits(&store).await;
+ let cancellation = tokio_util::sync::CancellationToken::new();
+ let authority = super::super::RuntimeStateAuthoritySignal::default();
+
+ let error = confirm_mechanical_state(
+ &store,
+ &permits,
+ &production_profile(),
+ &RuntimeCooldownPolicy::default(),
+ MechanicalFailureSignal {
+ class: CoreAttemptFailureClass::Connect,
+ timeout: None,
+ runtime_state_authority: authority.clone(),
+ },
+ Instant::now() + Duration::from_secs(1),
+ &cancellation,
+ )
+ .await
+ .unwrap_err();
+
+ assert_eq!(error.as_ref(), RUNTIME_STATE_AUTHORITY_FAILED);
+ assert!(authority.failed());
+}
diff --git a/crates/gateway/src/runtime/driver/response.rs b/crates/gateway/src/runtime/driver/response.rs
index 1282f6ed..30950343 100644
--- a/crates/gateway/src/runtime/driver/response.rs
+++ b/crates/gateway/src/runtime/driver/response.rs
@@ -30,8 +30,8 @@ use crate::server::core_runtime::profiles::CandidateProtocolProfile;
use crate::server::request_plan::IngressProtocol;
use super::{
- MATERIALIZATION_PROTOCOL_FAILED, MAX_RENDERED_PRECOMMIT_BYTES, ProductionAttemptState,
- ProductionDecodedSse, ProductionReadiness, SemanticTerminalOutcome, label, safe_error,
+ MATERIALIZATION_PROTOCOL_FAILED, ProductionAttemptState, ProductionDecodedSse,
+ ProductionReadiness, SemanticTerminalOutcome, label, safe_error,
};
pub(super) struct PrecommitDecoderBudget {
@@ -67,9 +67,6 @@ impl PrecommitDecoderBudget {
.raw_bytes
.checked_add(bytes)
.ok_or_else(|| Arc::from("native precommit decoder byte limit overflow"))?;
- if raw_bytes > MAX_RENDERED_PRECOMMIT_BYTES {
- return Err(Arc::from("native precommit decoder byte limit exceeded"));
- }
// Vec's amortized retained capacity is strictly below twice the live
// byte length after its minimum allocation. Shadow-charge that upper
// bound before the native decoder is allowed to copy this frame.
@@ -85,15 +82,10 @@ impl PrecommitDecoderBudget {
// Transport chunk boundaries are arbitrary and must not become a
// correctness limit. Grow the accounting reservation geometrically
// so a long sequence of tiny chunks uses bounded metadata while the
- // exact byte ceiling remains enforced.
- let maximum_charge = MAX_RENDERED_PRECOMMIT_BYTES
- .checked_mul(2)
- .and_then(|bytes| bytes.checked_add(8))
- .ok_or_else(|| Arc::from("native precommit decoder charge overflow"))?;
+ // request memory budget remains enforced.
let next_charge = target_charge
.checked_next_power_of_two()
- .unwrap_or(maximum_charge)
- .min(maximum_charge);
+ .unwrap_or(target_charge);
self.charges.push(
self.budget
.reserve(MemoryRole::ResponsePrefix, next_charge - self.charged_bytes)
@@ -157,6 +149,7 @@ pub(super) fn classify_precommit(
status.as_u16(),
state.streaming && status.is_success(),
state.chat_tool_projection.take(),
+ state.budget.clone(),
)
.map_err(|_| Arc::from(MATERIALIZATION_PROTOCOL_FAILED))?,
);
@@ -1116,7 +1109,8 @@ fn push_queue_bytes(
let quantum = queue.max_chunk_bytes();
for chunk in bytes.chunks(quantum) {
queue
- .push_back(
+ .push_back_growing(
+ budget,
ChargedBytes::copy_from_opaque(budget, MemoryRole::ResponsePrefix, chunk)
.map_err(safe_error)?,
)
@@ -1180,45 +1174,5 @@ pub(super) fn failure_facts(failure: &AttemptFailure) -> ProviderClassificationF
mod terminal_tests;
#[cfg(test)]
-mod tests {
- use super::*;
- use crate::server::core_runtime::profiles::{CandidateProtocolProfile, fixed_reasoning};
- use hiroute_gateway_core::runtime::body::BudgetTree;
-
- #[test]
- fn precommit_decoder_is_bounded_by_bytes_not_transport_chunk_count() {
- let budget = BudgetTree::new(4 * 1024 * 1024, 4 * 1024 * 1024)
- .unwrap()
- .stream(4 * 1024 * 1024)
- .unwrap();
- let mut decoder = PrecommitDecoderBudget::new(&budget).unwrap();
- for _ in 0..1_000 {
- decoder.charge_frame(128).unwrap();
- }
- assert!(
- decoder.charge_frame(MAX_RENDERED_PRECOMMIT_BYTES).is_err(),
- "the byte ceiling remains authoritative"
- );
- }
-
- #[test]
- fn connector_error_semantics_are_sealed_by_profile_not_connector_allowlist() {
- let mut profile = CandidateProtocolProfile::exact_portable_path(
- IngressProtocol::Responses,
- IngressProtocol::Responses,
- "gpt-codex",
- fixed_reasoning("fixed"),
- );
- profile.connector.connector_id = "connector.cpa.codex".into();
- assert!(connector_error_profile(&profile).is_ok());
-
- profile.connector.request_path = "/registered/provider/responses".into();
- assert!(
- connector_error_profile(&profile).is_ok(),
- "a catalog-bound exact provider path need not equal the public ingress path"
- );
-
- profile.connector.schema_version = "hiroute.connector-profile/v2".into();
- assert!(connector_error_profile(&profile).is_err());
- }
-}
+#[path = "response_budget_tests.rs"]
+mod tests;
diff --git a/crates/gateway/src/runtime/driver/response_budget_tests.rs b/crates/gateway/src/runtime/driver/response_budget_tests.rs
new file mode 100644
index 00000000..da89800f
--- /dev/null
+++ b/crates/gateway/src/runtime/driver/response_budget_tests.rs
@@ -0,0 +1,63 @@
+use super::*;
+use crate::server::core_runtime::profiles::{CandidateProtocolProfile, fixed_reasoning};
+use hiroute_gateway_core::runtime::body::BudgetTree;
+
+#[test]
+fn protocol_prefix_grows_beyond_initial_queue_capacity_and_releases_memory() {
+ let budget = BudgetTree::new(16 * 1024 * 1024, 16 * 1024 * 1024)
+ .unwrap()
+ .stream(16 * 1024 * 1024)
+ .unwrap();
+ let plan = hiroute_gateway_core::runtime::body::BodyPlan::PassThrough {
+ max_chunk_bytes: 64 * 1024,
+ };
+ let mut queue =
+ ChargedBodyQueue::new(&budget, MemoryRole::ResponsePrefix, &plan, usize::MAX, 1).unwrap();
+ let expected = vec![b'x'; 2 * 1024 * 1024];
+ push_queue_bytes(&mut queue, &budget, expected.clone()).unwrap();
+ let mut actual = Vec::new();
+ while let Some(chunk) = queue.pop_front() {
+ actual.extend_from_slice(chunk.bytes());
+ }
+ assert_eq!(actual, expected);
+ queue.clear_and_release();
+ assert_eq!(budget.snapshot().unwrap().live, 0);
+}
+
+#[test]
+fn precommit_decoder_is_bounded_by_bytes_not_transport_chunk_count() {
+ let budget = BudgetTree::new(4 * 1024 * 1024, 4 * 1024 * 1024)
+ .unwrap()
+ .stream(4 * 1024 * 1024)
+ .unwrap();
+ let mut decoder = PrecommitDecoderBudget::new(&budget).unwrap();
+ for _ in 0..1_000 {
+ decoder.charge_frame(128).unwrap();
+ }
+ decoder.charge_frame(256 * 1024).unwrap();
+ assert!(
+ decoder.charge_frame(4 * 1024 * 1024).is_err(),
+ "the actual memory budget remains authoritative"
+ );
+}
+
+#[test]
+fn connector_error_semantics_are_sealed_by_profile_not_connector_allowlist() {
+ let mut profile = CandidateProtocolProfile::exact_portable_path(
+ IngressProtocol::Responses,
+ IngressProtocol::Responses,
+ "gpt-codex",
+ fixed_reasoning("fixed"),
+ );
+ profile.connector.connector_id = "connector.cpa.codex".into();
+ assert!(connector_error_profile(&profile).is_ok());
+
+ profile.connector.request_path = "/registered/provider/responses".into();
+ assert!(
+ connector_error_profile(&profile).is_ok(),
+ "a catalog-bound exact provider path need not equal the public ingress path"
+ );
+
+ profile.connector.schema_version = "hiroute.connector-profile/v2".into();
+ assert!(connector_error_profile(&profile).is_err());
+}
diff --git a/crates/integrations/src/agents/native_ingress_probe.rs b/crates/integrations/src/agents/native_ingress_probe.rs
index b5e7bb50..b7425390 100644
--- a/crates/integrations/src/agents/native_ingress_probe.rs
+++ b/crates/integrations/src/agents/native_ingress_probe.rs
@@ -434,6 +434,7 @@ mod tests {
fs::write(&binary, b"#!/bin/sh\nprintf 'codex-cli 99.99.99\\n'\n").unwrap();
fs::set_permissions(&binary, fs::Permissions::from_mode(0o700)).unwrap();
let mut layout = super::super::AgentFilesystemLayoutV1::from_process(&home, root.path());
+ layout.codex_user_config = config;
layout.codex_executable = binary;
layout.codex_desktop_executable = None;
let registry: ConnectorRegistryBundleV1 = serde_json::from_slice(include_bytes!(
diff --git a/docs/standalone-cli.md b/docs/standalone-cli.md
index f2106698..3f1a5b69 100644
--- a/docs/standalone-cli.md
+++ b/docs/standalone-cli.md
@@ -384,6 +384,7 @@ Copy the Codex `context_id` from scan results and connect it to a published `PLA
"intent": "configure",
"settings": {
"mode": "codex_default",
+ "native_model_mode": "hiroute_only",
"fixed_models": [],
"allowed_plan_ids": ["PLAN_ID"],
"default_selection": {"kind": "plan", "plan_id": "PLAN_ID"}
diff --git a/docs/standalone-cli.zh-CN.md b/docs/standalone-cli.zh-CN.md
index d767f0b9..3ed789d1 100644
--- a/docs/standalone-cli.zh-CN.md
+++ b/docs/standalone-cli.zh-CN.md
@@ -362,6 +362,7 @@ hiroute agents check agent_codex_default \
"intent": "configure",
"settings": {
"mode": "codex_default",
+ "native_model_mode": "hiroute_only",
"fixed_models": [],
"allowed_plan_ids": ["PLAN_ID"],
"default_selection": {"kind": "plan", "plan_id": "PLAN_ID"}
diff --git a/e2e/matrix/p0-gateway-coverage.json b/e2e/matrix/p0-gateway-coverage.json
index 4b9c4dc4..88dbb90c 100644
--- a/e2e/matrix/p0-gateway-coverage.json
+++ b/e2e/matrix/p0-gateway-coverage.json
@@ -220,7 +220,7 @@
"execution": "real_hirouted_native_provider",
"id": "replay.request_capacity",
"source": "tools/e2e-harness/tests/p0_gateway_replay.rs",
- "symbol": "real_hirouted_rejects_request_body_plan_capacity_before_provider_call",
+ "symbol": "real_hirouted_rejects_unavailable_request_workspace_before_provider_call",
"test_target": "p0_gateway_replay"
},
{
@@ -301,7 +301,7 @@
"test_target": "p0_gateway_privacy"
}
],
- "registry_digest": "sha256:061cda488951676f3f8a60b38467ba345ee519249f999cb21f75aac9ab793d62",
+ "registry_digest": "sha256:32438a229a8518a05a1f02ad0ebfa8ed031a6ddc1410120bbc3b6ed5668dad8a",
"rows": [
{
"id": "publication.incompatible_or_gap",
diff --git a/tools/e2e-harness/src/p0/coverage.rs b/tools/e2e-harness/src/p0/coverage.rs
index af3663f5..ee2654f9 100644
--- a/tools/e2e-harness/src/p0/coverage.rs
+++ b/tools/e2e-harness/src/p0/coverage.rs
@@ -300,7 +300,7 @@ pub const RECEIPTS: &[FrozenReceipt] = &[
receipt!(
"replay.request_capacity",
"p0_gateway_replay",
- "real_hirouted_rejects_request_body_plan_capacity_before_provider_call",
+ "real_hirouted_rejects_unavailable_request_workspace_before_provider_call",
"tools/e2e-harness/tests/p0_gateway_replay.rs",
[
"replay.request_capacity_limit",
diff --git a/tools/e2e-harness/tests/p0_gateway_protocol/production_matrix/native_protocol_paths.rs b/tools/e2e-harness/tests/p0_gateway_protocol/production_matrix/native_protocol_paths.rs
index a7d5112f..6fa02193 100644
--- a/tools/e2e-harness/tests/p0_gateway_protocol/production_matrix/native_protocol_paths.rs
+++ b/tools/e2e-harness/tests/p0_gateway_protocol/production_matrix/native_protocol_paths.rs
@@ -438,11 +438,11 @@ fn real_glm_messages_native_path_handles_repeated_small_and_large_requests() {
format!("Reply with OK after reading: {}", "alpha ".repeat(13_120)),
),
] {
- // Keep model in the first 16 KiB selector window. A map-backed JSON
- // serializer may sort it after a large messages value instead.
+ // Model selection must work after a large messages value, as with
+ // serializers that order model after messages.
let content_json = serde_json::to_string(&content).unwrap();
let body = format!(
- r#"{{"model":"{alias}","max_tokens":128,"messages":[{{"role":"user","content":{content_json}}}],"stream":false}}"#
+ r#"{{"max_tokens":128,"messages":[{{"role":"user","content":{content_json}}}],"model":"{alias}","stream":false}}"#
)
.into_bytes();
let response = single_write_request(address, "/v1/messages", &body);
diff --git a/tools/e2e-harness/tests/p0_gateway_protocol/reviewer.rs b/tools/e2e-harness/tests/p0_gateway_protocol/reviewer.rs
index fbf59316..99251efd 100644
--- a/tools/e2e-harness/tests/p0_gateway_protocol/reviewer.rs
+++ b/tools/e2e-harness/tests/p0_gateway_protocol/reviewer.rs
@@ -8,10 +8,9 @@ use hiroute_gateway::server::core_runtime::model_ir::{
ResponseBlockKind,
};
use hiroute_gateway::server::core_runtime::profiles::{
- CandidateProtocolProfile, ClientProtocolProfile, Fidelity,
- MAX_TERMINAL_CLASSIFIED_REFUSAL_BLOCKS, NativeReasoningFieldAssignment, NativeReasoningRender,
- NativeReasoningValue, ReasoningAccounting, ReasoningControlKind, ReasoningProfileCapability,
- StateAffinity, StreamingRefusalSemantics, fixed_reasoning,
+ CandidateProtocolProfile, ClientProtocolProfile, Fidelity, NativeReasoningFieldAssignment,
+ NativeReasoningRender, NativeReasoningValue, ReasoningAccounting, ReasoningControlKind,
+ ReasoningProfileCapability, StateAffinity, StreamingRefusalSemantics, fixed_reasoning,
};
use hiroute_gateway::server::request_plan::IngressProtocol;
use serde_json::{Value, json};
@@ -505,17 +504,11 @@ fn protocol_reviewer_messages_terminal_refusal_is_typed_before_client_emission()
.unwrap();
assert!(matches!(
exact_profile.capability.response.stream_refusal,
- StreamingRefusalSemantics::TerminalClassified {
- max_buffered_bytes: 1..,
- max_buffered_blocks: 1..=8,
- }
+ StreamingRefusalSemantics::TerminalClassified
));
for semantics in [
StreamingRefusalSemantics::Unknown,
- StreamingRefusalSemantics::TerminalClassified {
- max_buffered_bytes: 16 * 1024 * 1024,
- max_buffered_blocks: MAX_TERMINAL_CLASSIFIED_REFUSAL_BLOCKS + 1,
- },
+ StreamingRefusalSemantics::Unsupported,
] {
let mut invalid = exact_profile.clone();
invalid.capability.response.stream_refusal = semantics;
diff --git a/tools/e2e-harness/tests/p0_gateway_replay.rs b/tools/e2e-harness/tests/p0_gateway_replay.rs
index 4c94e44b..e0783c94 100644
--- a/tools/e2e-harness/tests/p0_gateway_replay.rs
+++ b/tools/e2e-harness/tests/p0_gateway_replay.rs
@@ -462,7 +462,7 @@ fn real_hirouted_rejects_non_owner_only_replay_root() {
}
#[test]
-fn real_hirouted_rejects_request_body_plan_capacity_before_provider_call() {
+fn real_hirouted_rejects_unavailable_request_workspace_before_provider_call() {
hiroute_e2e::p0_execution_receipt!(
"replay.request_capacity",
[
@@ -472,19 +472,19 @@ fn real_hirouted_rejects_request_body_plan_capacity_before_provider_call() {
);
let forbidden = NativeProvider::start(vec![success()]);
let fixture = RuntimeFixture::launch_with_replay(&[&forbidden], 1, 4_096, 1_024);
- let body = request_document("capacity boundary ".repeat(65_536));
+ let body = request_document("capacity boundary ".repeat(500_000));
assert!(
- body.len() > 1024 * 1024,
- "test must exceed the frozen body plan"
+ body.len() > 8 * 1024 * 1024,
+ "parsed JSON workspace must exceed the actual stream memory budget"
);
let response = fixture.request_body(&body);
- assert_eq!(response.status, 413);
+ assert_eq!(response.status, 503);
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&response.body).unwrap(),
serde_json::json!({
"schema_version": "hiroute.gateway.error/v1",
- "code": "REQUEST_BODY_PLAN_LIMIT_EXCEEDED",
+ "code": "REPLAY_BUDGET_UNAVAILABLE",
"phase": "canonical_request"
})
);
diff --git a/tools/e2e-harness/tests/p0_gateway_request_authority.rs b/tools/e2e-harness/tests/p0_gateway_request_authority.rs
index fee5a899..6f73e9e9 100644
--- a/tools/e2e-harness/tests/p0_gateway_request_authority.rs
+++ b/tools/e2e-harness/tests/p0_gateway_request_authority.rs
@@ -23,6 +23,10 @@ use hiroute_gateway::server::publication::{
use hiroute_gateway::server::request_plan::IngressProtocol;
use serde_json::{Value, json};
+#[path = "p0_gateway_request_authority/fixtures.rs"]
+mod fixtures;
+use fixtures::*;
+
fn candidate(local_id: u32) -> CandidateBindingV1 {
sealed_native_candidate(
local_id,
@@ -292,12 +296,16 @@ fn production_hirouted_wire_authority_catalog_deadline_and_actual_restart() {
"Bearer wire-provider-secret-1",
provider_response("wire-provider", "ok"),
);
+ // Padding keeps the semantic request small while proving the production
+ // entry and core impose no former 1 MiB cumulative request ceiling.
+ let mut accepted_body = br#"{"model":"wire-fast","input":"opaque"}"#.to_vec();
+ accepted_body.resize(1024 * 1024 + 1024, b' ');
let accepted = request(
address,
"POST",
"/v1/responses",
&[("X-HiRoute-Token", "wire-token")],
- br#"{"model":"wire-fast","input":"opaque"}"#,
+ &accepted_body,
);
provider_thread.join().unwrap();
assert_json_response(&accepted, 200);
@@ -512,164 +520,6 @@ fn production_hirouted_live_publication_control_pins_cutover_and_preserves_lkg_o
restarted.crash();
}
-fn wire_snapshot(provider_authority: &str) -> GatewayPublicationSnapshotV3 {
- let candidate = |local_id| {
- sealed_native_candidate(
- local_id,
- &format!("wire-target-{local_id}"),
- &[format!("wire-credential-{local_id}")],
- provider_authority,
- &format!("wire-native-model-{local_id}"),
- &[
- (IngressProtocol::Responses, IngressProtocol::Responses),
- (IngressProtocol::Messages, IngressProtocol::Responses),
- ],
- )
- };
- GatewayPublicationSnapshotV3::seal(
- "personal/default",
- "wire-authority",
- 9,
- 31,
- "wire-renderer/v1",
- vec![
- AliasPlanV1 {
- served_model_id: "wire-fast".into(),
- purpose: "wire fast".into(),
- agent_plan_revision: 41,
- protocols: vec![IngressProtocol::Responses],
- overall_timeout_ms: 1_500,
- max_attempts: 1,
- routing: None,
- candidates: vec![candidate(1)],
- },
- AliasPlanV1 {
- served_model_id: "wire-deep".into(),
- purpose: "wire deep".into(),
- agent_plan_revision: 42,
- protocols: vec![IngressProtocol::Responses, IngressProtocol::Messages],
- overall_timeout_ms: MAX_LOGICAL_REQUEST_DURATION_MS,
- max_attempts: 4,
- routing: None,
- candidates: vec![candidate(2)],
- },
- AliasPlanV1 {
- served_model_id: "wire-private".into(),
- purpose: "wire private".into(),
- agent_plan_revision: 43,
- protocols: vec![IngressProtocol::Responses],
- overall_timeout_ms: 200,
- max_attempts: 1,
- routing: None,
- candidates: vec![candidate(3)],
- },
- ],
- vec![
- GrantV1 {
- grant_id: "wire-grant".into(),
- generation: 5,
- bearer_token_sha256: token_sha256("wire-token"),
- protocol: IngressProtocol::Responses,
- routes: plan_routes(&[("wire-fast", 41), ("wire-deep", 42)]),
- },
- GrantV1 {
- grant_id: "wire-messages-grant".into(),
- generation: 5,
- bearer_token_sha256: token_sha256("wire-messages-token"),
- protocol: IngressProtocol::Messages,
- routes: plan_routes(&[("wire-deep", 42)]),
- },
- ],
- )
- .unwrap()
-}
-
-fn live_snapshot(
- publication_revision: u64,
- plan_revision: u64,
- provider_authority: &str,
- credential_ref: &str,
-) -> GatewayPublicationSnapshotV3 {
- GatewayPublicationSnapshotV3::seal(
- "personal/default",
- "live-authority",
- 17,
- publication_revision,
- format!("live-renderer/v{publication_revision}"),
- vec![AliasPlanV1 {
- served_model_id: "wire-live".into(),
- purpose: "live publication cutover".into(),
- agent_plan_revision: plan_revision,
- protocols: vec![IngressProtocol::Responses],
- overall_timeout_ms: 60_000,
- max_attempts: 1,
- routing: None,
- candidates: vec![sealed_native_candidate(
- 1,
- "live-target",
- &[credential_ref.into()],
- provider_authority,
- "live-native-model",
- &[(IngressProtocol::Responses, IngressProtocol::Responses)],
- )],
- }],
- vec![GrantV1 {
- grant_id: "live-grant".into(),
- generation: 1,
- bearer_token_sha256: token_sha256("live-wire-token"),
- protocol: IngressProtocol::Responses,
- routes: plan_routes(&[("wire-live", plan_revision)]),
- }],
- )
- .unwrap()
-}
-
-fn write_live_credentials(directory: &Path) -> PathBuf {
- std::fs::write(
- directory.join("live-credential.json"),
- serde_json::to_vec_pretty(&json!({
- "schema_version": "hiroute.gateway.credential-leases/v1",
- "credential_ref": "live-credential",
- "keys": [{
- "key_id": "live-key",
- "generation": 1,
- "authorization": "Bearer live-provider",
- }],
- }))
- .unwrap(),
- )
- .unwrap();
- let path = directory.join("live-credentials.json");
- std::fs::write(
- &path,
- serde_json::to_vec_pretty(&json!({
- "schema_version": "hiroute.gateway.credentials/v1",
- "credentials": {
- "live-credential": "live-credential.json",
- }
- }))
- .unwrap(),
- )
- .unwrap();
- path
-}
-fn write_control_nonce(directory: &Path, label: &str, mode: u32) -> (PathBuf, String) {
- let mut random = [0_u8; 32];
- getrandom::fill(&mut random).unwrap();
- let nonce = random
- .iter()
- .map(|byte| format!("{byte:02x}"))
- .collect::<String>();
- let path = directory.join(format!("e2e-control-{label}.nonce"));
- let mut options = std::fs::OpenOptions::new();
- options.write(true).create_new(true);
- #[cfg(unix)]
- options.mode(mode);
- let mut file = options.open(&path).unwrap();
- file.write_all(nonce.as_bytes()).unwrap();
- file.sync_all().unwrap();
- (path, nonce)
-}
fn request_models(address: SocketAddr, token: &str) -> WireResponse {
request(
address,
diff --git a/tools/e2e-harness/tests/p0_gateway_request_authority/fixtures.rs b/tools/e2e-harness/tests/p0_gateway_request_authority/fixtures.rs
new file mode 100644
index 00000000..afef832f
--- /dev/null
+++ b/tools/e2e-harness/tests/p0_gateway_request_authority/fixtures.rs
@@ -0,0 +1,160 @@
+use super::*;
+
+pub(super) fn wire_snapshot(provider_authority: &str) -> GatewayPublicationSnapshotV3 {
+ let candidate = |local_id| {
+ sealed_native_candidate(
+ local_id,
+ &format!("wire-target-{local_id}"),
+ &[format!("wire-credential-{local_id}")],
+ provider_authority,
+ &format!("wire-native-model-{local_id}"),
+ &[
+ (IngressProtocol::Responses, IngressProtocol::Responses),
+ (IngressProtocol::Messages, IngressProtocol::Responses),
+ ],
+ )
+ };
+ GatewayPublicationSnapshotV3::seal(
+ "personal/default",
+ "wire-authority",
+ 9,
+ 31,
+ "wire-renderer/v1",
+ vec![
+ AliasPlanV1 {
+ served_model_id: "wire-fast".into(),
+ purpose: "wire fast".into(),
+ agent_plan_revision: 41,
+ protocols: vec![IngressProtocol::Responses],
+ overall_timeout_ms: 1_500,
+ max_attempts: 1,
+ routing: None,
+ candidates: vec![candidate(1)],
+ },
+ AliasPlanV1 {
+ served_model_id: "wire-deep".into(),
+ purpose: "wire deep".into(),
+ agent_plan_revision: 42,
+ protocols: vec![IngressProtocol::Responses, IngressProtocol::Messages],
+ overall_timeout_ms: MAX_LOGICAL_REQUEST_DURATION_MS,
+ max_attempts: 4,
+ routing: None,
+ candidates: vec![candidate(2)],
+ },
+ AliasPlanV1 {
+ served_model_id: "wire-private".into(),
+ purpose: "wire private".into(),
+ agent_plan_revision: 43,
+ protocols: vec![IngressProtocol::Responses],
+ overall_timeout_ms: 200,
+ max_attempts: 1,
+ routing: None,
+ candidates: vec![candidate(3)],
+ },
+ ],
+ vec![
+ GrantV1 {
+ grant_id: "wire-grant".into(),
+ generation: 5,
+ bearer_token_sha256: token_sha256("wire-token"),
+ protocol: IngressProtocol::Responses,
+ routes: plan_routes(&[("wire-fast", 41), ("wire-deep", 42)]),
+ },
+ GrantV1 {
+ grant_id: "wire-messages-grant".into(),
+ generation: 5,
+ bearer_token_sha256: token_sha256("wire-messages-token"),
+ protocol: IngressProtocol::Messages,
+ routes: plan_routes(&[("wire-deep", 42)]),
+ },
+ ],
+ )
+ .unwrap()
+}
+
+pub(super) fn live_snapshot(
+ publication_revision: u64,
+ plan_revision: u64,
+ provider_authority: &str,
+ credential_ref: &str,
+) -> GatewayPublicationSnapshotV3 {
+ GatewayPublicationSnapshotV3::seal(
+ "personal/default",
+ "live-authority",
+ 17,
+ publication_revision,
+ format!("live-renderer/v{publication_revision}"),
+ vec![AliasPlanV1 {
+ served_model_id: "wire-live".into(),
+ purpose: "live publication cutover".into(),
+ agent_plan_revision: plan_revision,
+ protocols: vec![IngressProtocol::Responses],
+ overall_timeout_ms: 60_000,
+ max_attempts: 1,
+ routing: None,
+ candidates: vec![sealed_native_candidate(
+ 1,
+ "live-target",
+ &[credential_ref.into()],
+ provider_authority,
+ "live-native-model",
+ &[(IngressProtocol::Responses, IngressProtocol::Responses)],
+ )],
+ }],
+ vec![GrantV1 {
+ grant_id: "live-grant".into(),
+ generation: 1,
+ bearer_token_sha256: token_sha256("live-wire-token"),
+ protocol: IngressProtocol::Responses,
+ routes: plan_routes(&[("wire-live", plan_revision)]),
+ }],
+ )
+ .unwrap()
+}
+
+pub(super) fn write_live_credentials(directory: &Path) -> PathBuf {
+ std::fs::write(
+ directory.join("live-credential.json"),
+ serde_json::to_vec_pretty(&json!({
+ "schema_version": "hiroute.gateway.credential-leases/v1",
+ "credential_ref": "live-credential",
+ "keys": [{
+ "key_id": "live-key",
+ "generation": 1,
+ "authorization": "Bearer live-provider",
+ }],
+ }))
+ .unwrap(),
+ )
+ .unwrap();
+ let path = directory.join("live-credentials.json");
+ std::fs::write(
+ &path,
+ serde_json::to_vec_pretty(&json!({
+ "schema_version": "hiroute.gateway.credentials/v1",
+ "credentials": {
+ "live-credential": "live-credential.json",
+ }
+ }))
+ .unwrap(),
+ )
+ .unwrap();
+ path
+}
+pub(super) fn write_control_nonce(directory: &Path, label: &str, mode: u32) -> (PathBuf, String) {
+ let mut random = [0_u8; 32];
+ getrandom::fill(&mut random).unwrap();
+ let nonce = random
+ .iter()
+ .map(|byte| format!("{byte:02x}"))
+ .collect::<String>();
+ let path = directory.join(format!("e2e-control-{label}.nonce"));
+ let mut options = std::fs::OpenOptions::new();
+ options.write(true).create_new(true);
+ #[cfg(unix)]
+ options.mode(mode);
+ let mut file = options.open(&path).unwrap();
+ file.write_all(nonce.as_bytes()).unwrap();
+ file.sync_all().unwrap();
+ (path, nonce)
+}
diff --git a/tools/e2e-harness/tests/p0_gateway_runtime/reasoning.rs b/tools/e2e-harness/tests/p0_gateway_runtime/reasoning.rs
index 5e5873f3..eaa1af13 100644
--- a/tools/e2e-harness/tests/p0_gateway_runtime/reasoning.rs
+++ b/tools/e2e-harness/tests/p0_gateway_runtime/reasoning.rs
@@ -81,8 +81,12 @@ fn fixed_effort_large_input_preserves_choice_after_content_externalization() {
}
#[test]
-fn fixed_effort_late_model_still_obeys_selector_limit() {
- let primary = NativeProvider::start(Vec::new());
+fn fixed_effort_late_model_reaches_upstream_after_large_input() {
+ let primary = NativeProvider::start(vec![ProviderReply::Complete {
+ status: 200,
+ error_kind: None,
+ body: RESPONSES_OK,
+ }]);
let other = NativeProvider::start(Vec::new());
let fixture = RuntimeFixture::launch_reasoning(&[&primary, &other], true);
let input = serde_json::to_string(&"hello".repeat(4096)).unwrap();
@@ -92,15 +96,12 @@ fn fixed_effort_late_model_still_obeys_selector_limit() {
let response = fixture.request_body(body.as_bytes());
assert_eq!(
response.status,
- 400,
+ 200,
"{}",
String::from_utf8_lossy(&response.body)
);
- assert_eq!(
- serde_json::from_slice::<serde_json::Value>(&response.body).unwrap()["code"],
- "MODEL_SELECTOR_LIMIT_EXCEEDED"
- );
- assert_eq!((primary.calls(), other.calls()), (0, 0));
+ assert_eq!((primary.calls(), other.calls()), (1, 0));
+ assert_eq!(reasoning_wire_value(&primary.requests()[0]), "high");
}
#[test]
diff --git a/tools/product-e2e/tests/control_shell.rs b/tools/product-e2e/tests/control_shell.rs
index 9e395109..0358e1c6 100644
--- a/tools/product-e2e/tests/control_shell.rs
+++ b/tools/product-e2e/tests/control_shell.rs
@@ -97,6 +97,8 @@ fn control_shell_uses_real_hiroute_and_hirouted_processes() {
runtime_root.to_str().unwrap(),
])
.env("HOME", &agent_home)
+ .env_remove("CODEX_HOME")
+ .env_remove("CLAUDE_CONFIG_DIR")
.env("PATH", agent_path)
.env_remove("ANTHROPIC_BASE_URL")
.env_remove("ANTHROPIC_MODEL")
diff --git a/tools/product-e2e/tests/pre_gateway_compute_routing.rs b/tools/product-e2e/tests/pre_gateway_compute_routing.rs
index 9ddc8fc1..4fc94446 100644
--- a/tools/product-e2e/tests/pre_gateway_compute_routing.rs
+++ b/tools/product-e2e/tests/pre_gateway_compute_routing.rs
@@ -101,6 +101,8 @@ fn production_subprocess_uses_embedded_catalog_and_ignores_storage_tampering() {
runtime_root.to_str().unwrap(),
])
.env("HOME", &home)
+ .env_remove("CODEX_HOME")
+ .env_remove("CLAUDE_CONFIG_DIR")
.env_remove("ANTHROPIC_AUTH_TOKEN")
.env_remove("ANTHROPIC_BASE_URL")
.env_remove("ANTHROPIC_MODEL")