🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(ai-gateway): raise request limit to 16 MiB and make execution limits configurable (#18820)
Tobias Wochinger6b6b103c98d0
diff --git a/.env.dev.example b/.env.dev.exampleindex 255a3bc42a..789e2fc44a 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -211,6 +211,12 @@ ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000# Independent per-instance concurrency limits; provisional defaults, tune with load tests.# LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS=128# LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS=128+# Largest request body, and the body bytes all in-flight requests may buffer together.+# LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES=16777216+# LANGFUSE_AI_GATEWAY_SHARED_REQUEST_BUFFER_BYTES=1073741824+# Execution ceiling and the longest wait without stream progress.+# LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS=600+# LANGFUSE_AI_GATEWAY_STALL_TIMEOUT_SECONDS=120# AI gateway control-plane key rotation. Generate service keys with# `openssl rand -hex 32` and ES256 PEM keys withdiff --git a/.env.prod.example b/.env.prod.exampleindex dbfc328f45..af35519fe5 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -585,6 +585,12 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# Independent per-instance concurrency limits; provisional defaults, tune with load tests.# LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS=128# LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS=128+# Largest request body, and the body bytes all in-flight requests may buffer together.+# LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES=16777216+# LANGFUSE_AI_GATEWAY_SHARED_REQUEST_BUFFER_BYTES=1073741824+# Execution ceiling and the longest wait without stream progress.+# LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS=600+# LANGFUSE_AI_GATEWAY_STALL_TIMEOUT_SECONDS=120# Internal cloud trace-batch reader controls; these do not enable processing.# ClickHouse socket idle timeout during streaming, including summary backpressure.diff --git a/ai-gateway/Cargo.lock b/ai-gateway/Cargo.lockindex ea0e37c9b9..72a0fe70e6 100644--- a/ai-gateway/Cargo.lock+++ b/ai-gateway/Cargo.lock@@ -1779,6 +1779,7 @@ dependencies = ["http-body","percent-encoding","pin-project-lite",+ "tokio","tower-layer","tower-service","tracing",diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.tomlindex c48fa1fc3f..6e3a8bfb15 100644--- a/ai-gateway/Cargo.toml+++ b/ai-gateway/Cargo.toml@@ -24,7 +24,7 @@ serde = { version = "1", features = ["derive"] }serde_json = { version = "1", features = ["raw_value"] }sha2 = "0.11"tokio = { version = "1.53.1", features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] }-tower-http = { version = "0.7", features = ["trace"] }+tower-http = { version = "0.7", features = ["timeout", "trace"] }tracing = "0.1.44"tracing-opentelemetry = { version = "0.34", default-features = false }tracing-subscriber = { version = "0.3.23", default-features = false, features = ["env-filter", "fmt", "json"] }diff --git a/ai-gateway/README.md b/ai-gateway/README.mdindex e083b797a3..6040c4a1b9 100644--- a/ai-gateway/README.md+++ b/ai-gateway/README.md@@ -81,6 +81,10 @@ do not load dotenv files:| `LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS` | `128` | Positive integer up to Tokio's semaphore capacity; buffered and active requests per instance || `LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS` | `128` | Positive integer up to Tokio's semaphore capacity; separate model-catalog and Web-resolution budgets per instance || `LANGFUSE_AI_GATEWAY_TELEMETRY_BUFFER_BYTES` | `67108864` (64 MiB) | Integer from 4194304 (4 MiB) to 4294967296 (4 GiB); span and credential bytes an instance holds for upload before dropping new records |+| `LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES` | `16777216` (16 MiB) | Integer from 1048576 (1 MiB) to 4294967295; largest accepted request body |+| `LANGFUSE_AI_GATEWAY_SHARED_REQUEST_BUFFER_BYTES` | `1073741824` (1 GiB) | Integer from the request limit up to Tokio's semaphore capacity; request body bytes all in-flight requests of an instance may buffer together |+| `LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS` | `600` | Integer of at least 60; ceiling from execution admission until the response completes. Web ingestion tokens last 15 minutes, so traces of longer executions may be dropped |+| `LANGFUSE_AI_GATEWAY_STALL_TIMEOUT_SECONDS` | `120` | Integer from 1 to the execution ceiling; longest wait without progress |The gateway shares `LANGFUSE_LOG_LEVEL` with Web and worker. Values are lowercase;`fatal` maps to Rust's `error` level and therefore includes ordinary error logs.@@ -372,28 +376,35 @@ error type follows the status (`authentication_error`, `permission_error`,`api_error`).After validating the credential header, the gateway buffers and validates the request-under the active-request admission budget, then resolves the credential and-model-aware route through Web under a separate budget. It holds the body slot through-the provider request. Model catalogs use another budget so slow catalog discovery+under the active-request admission budget and a shared request-byte budget, then resolves the credential and+model-aware route through Web under a separate budget. A declared `Content-Length` is+checked against the request limit before reading. The byte budget covers the buffer's+allocation and grows as bytes arrive, so a client that declares a large body and stalls+holds only what it sent. The body slot and its bytes stay reserved with the execution+until the response ends. Model catalogs use another budget so slow catalog discoverycannot block inference resolution.Execution capacity remains reserved through response completion. An exhausted budgetreturns 503 immediately without a waiting queue. Request-body and execution budgets-use the active-request limit; model-catalog and Web-call budgets use the resolution+use the active-request limit, and buffered bodies also share+`LANGFUSE_AI_GATEWAY_SHARED_REQUEST_BUFFER_BYTES`; model-catalog and Web-call budgets use the resolutionlimit. Their defaults of 128 are provisionalguardrails, not measured capacity: tune them using load tests for the instanceresources, request sizes and stream durations. These limits bound work; they do notguarantee fairness between clients or tenants.-Other limits are 10 MiB request bodies in both namespaces, 30 seconds to read a request, 5 seconds to-connect, 120 seconds for provider response headers or an individual upstream read,-and 600 seconds overall from execution admission in both namespaces. Response size is not capped: a single task pumps chunks+Request bodies are limited to 16 MiB by default in both namespaces. The body read deadline+scales with that limit: at least 30 seconds, and enough for the maximal body at 10 MiB per+30 seconds (49 seconds at 16 MiB).+Connecting to a provider may take 5 seconds. Executions end at a ten-minute ceiling from+admission by default, and earlier when they stall: 120 seconds by default without an+upstream chunk, without the downstream accepting a chunk, or, for streamed calls+(`"stream": true`) and token counting, without provider response+headers. A non-streamed generation waits for headers until the ceiling, because+providers send nothing until the response is generated. Response size is not capped: a single task pumps chunksthrough a one-slot channel, with chunks at most 64 KiB. It stops reading when that-channel fills. Completion, disconnect and deadline release admission and context;-the deadline runs even when the downstream stops polling. A failure after headers+channel fills. Completion, disconnect, stall and ceiling release admission and context;+both timeouts run even when the downstream stops polling. A failure after headersterminates the response body without adding an error event or retrying.-There is no separate downstream stall timeout; a client that stops reading can-retain execution capacity until the overall deadline. A progress-based downstream-stall policy is deferred to a separate change.Only request content type/encoding and accept cross the provider boundary, plus theresolved credential in its provider's header (`Authorization: Bearer` for OpenAI,diff --git a/ai-gateway/src/capture/anthropic_messages/tests.rs b/ai-gateway/src/capture/anthropic_messages/tests.rsindex b29e40b1d2..0e26b4637e 100644--- a/ai-gateway/src/capture/anthropic_messages/tests.rs+++ b/ai-gateway/src/capture/anthropic_messages/tests.rs@@ -557,8 +557,12 @@ async fn streamed_messages_upload_one_generation_with_native_usage() {provider.try_admit().unwrap(),context,&headers,- Bytes::from_static(REQUEST),- crate::providers::model_range(REQUEST).unwrap(),+ crate::providers::RequestBody {+ bytes: Bytes::from_static(REQUEST),+ model_range: crate::providers::model_range(REQUEST).unwrap(),+ streaming: true,+ reservation: None,+ },Route::AnthropicMessages,&mut crate::correlation::RequestCorrelation::default(),)diff --git a/ai-gateway/src/capture/openai_responses/tests.rs b/ai-gateway/src/capture/openai_responses/tests.rsindex dcc45fc962..55f49121ed 100644--- a/ai-gateway/src/capture/openai_responses/tests.rs+++ b/ai-gateway/src/capture/openai_responses/tests.rs@@ -3,6 +3,7 @@ use crate::capture::{ExecutionCapture, InputOmission, InputOmissionReason, MAX_INPUT_CAPTURE_BYTES, ProtocolCapture,RelayOutcome, sse::SseDecoder,};+use crate::config::ExecutionLimits;use crate::resolution::ApiFormat;use crate::{providers::{ProviderLimits, ProviderTransport},@@ -513,8 +514,10 @@ async fn upstream_read_timeout_is_logged_as_timeout() {format!("{}/v1", upstream.url),ProviderLimits {active: 1,- read_timeout: Duration::from_millis(50),- ..ProviderLimits::default()+ execution: ExecutionLimits {+ stall_timeout: Duration::from_millis(50),+ ..ExecutionLimits::default()+ },},);let writer = LogWriter::default();@@ -568,8 +571,10 @@ async fn native_http_relay_logs_once_on_eof_drop_and_unpolled_deadline() {format!("{}/v1", upstream.url),ProviderLimits {active: 1,- execution_timeout: Duration::from_secs(1),- ..ProviderLimits::default()+ execution: ExecutionLimits {+ max_execution_time: Duration::from_secs(1),+ ..ExecutionLimits::default()+ },},);let writer = LogWriter::default();@@ -632,8 +637,10 @@ async fn provider_future_finalizes_on_timeout_and_cancellation_before_headers()format!("{}/v1", upstream.url),ProviderLimits {active: 1,- headers_timeout: Duration::from_millis(100),- ..ProviderLimits::default()+ execution: ExecutionLimits {+ stall_timeout: Duration::from_millis(100),+ ..ExecutionLimits::default()+ },},);let writer = LogWriter::default();diff --git a/ai-gateway/src/config.rs b/ai-gateway/src/config.rsindex 3061b9768c..5f59f685bf 100644--- a/ai-gateway/src/config.rs+++ b/ai-gateway/src/config.rs@@ -1,5 +1,5 @@use crate::resolution::ControlPlaneConfig;-use std::{env, error::Error, fmt, net::SocketAddr, time::Duration};+use std::{env, error::Error, fmt, net::SocketAddr, ops::RangeInclusive, time::Duration};use tracing::level_filters::LevelFilter;pub struct GatewayConfig {@@ -12,6 +12,50 @@ pub struct GatewayConfig {pub max_active_requests: usize,pub max_concurrent_resolutions: usize,pub telemetry_buffer_bytes: usize,+ pub request_limits: RequestLimits,+ pub execution_limits: ExecutionLimits,+}++const MIB: usize = 1024 * 1024;+const DEFAULT_CONCURRENCY: usize = 128;+const SEMAPHORE_CAPACITY: usize = tokio::sync::Semaphore::MAX_PERMITS;+pub const DEFAULT_MAX_REQUEST_BYTES: usize = 16 * MIB;+const DEFAULT_SHARED_REQUEST_BUFFER_BYTES: usize = 1024 * MIB;+const DEFAULT_MAX_EXECUTION_SECONDS: u32 = 10 * 60;+const DEFAULT_STALL_TIMEOUT_SECONDS: u32 = 120;+/// Buffered request body bounds per instance.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub struct RequestLimits {+ pub max_request_bytes: usize,+ /// Request body bytes all in-flight requests may buffer together.+ pub shared_buffer_bytes: usize,+}++impl Default for RequestLimits {+ fn default() -> Self {+ Self {+ max_request_bytes: DEFAULT_MAX_REQUEST_BYTES,+ shared_buffer_bytes: DEFAULT_SHARED_REQUEST_BUFFER_BYTES,+ }+ }+}++/// Provider execution deadlines.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub struct ExecutionLimits {+ /// Ceiling from execution admission until the response completes.+ pub max_execution_time: Duration,+ /// Longest wait without progress: stream response headers, upstream reads, downstream writes.+ pub stall_timeout: Duration,+}++impl Default for ExecutionLimits {+ fn default() -> Self {+ Self {+ max_execution_time: Duration::from_secs(DEFAULT_MAX_EXECUTION_SECONDS.into()),+ stall_timeout: Duration::from_secs(DEFAULT_STALL_TIMEOUT_SECONDS.into()),+ }+ }}#[derive(Debug, PartialEq, Eq)]@@ -51,6 +95,14 @@ impl GatewayConfig {config.telemetry_buffer_bytes = telemetry_buffer_bytes(read_env("LANGFUSE_AI_GATEWAY_TELEMETRY_BUFFER_BYTES")?.as_deref(),)?;+ config.request_limits = request_limits(+ read_env("LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES")?.as_deref(),+ read_env("LANGFUSE_AI_GATEWAY_SHARED_REQUEST_BUFFER_BYTES")?.as_deref(),+ )?;+ config.execution_limits = execution_limits(+ read_env("LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS")?.as_deref(),+ read_env("LANGFUSE_AI_GATEWAY_STALL_TIMEOUT_SECONDS")?.as_deref(),+ )?;if let Some(web_url) =read_env("LANGFUSE_AI_GATEWAY_WEB_URL")?.filter(|url| !url.is_empty()){@@ -98,16 +150,12 @@ impl GatewayConfig {.ok_or(GatewayConfigError("LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT must be true or false",))?;- let seconds: u64 = shutdown_timeout.unwrap_or("10").parse().map_err(|_| {- GatewayConfigError(- "LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",- )- })?;- if !(1..=300).contains(&seconds) {- return Err(GatewayConfigError(- "LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",- ));- }+ let seconds = integer_setting(+ shutdown_timeout,+ 10,+ 1..=300,+ "LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",+ )?;let log_level = match log_level.unwrap_or("info") {"debug" => LevelFilter::DEBUG,"info" => LevelFilter::INFO,@@ -136,43 +184,95 @@ impl GatewayConfig {shutdown_timeout: Duration::from_secs(seconds),log_level,log_format,- max_active_requests: concurrency_limit(+ max_active_requests: integer_setting(max_active_requests,+ DEFAULT_CONCURRENCY,+ 1..=SEMAPHORE_CAPACITY,"LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS must be a positive integer within the semaphore capacity",)?,- max_concurrent_resolutions: concurrency_limit(+ max_concurrent_resolutions: integer_setting(max_concurrent_resolutions,+ DEFAULT_CONCURRENCY,+ 1..=SEMAPHORE_CAPACITY,"LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS must be a positive integer within the semaphore capacity",)?,telemetry_buffer_bytes: crate::telemetry::DEFAULT_RETAINED_BYTES,+ request_limits: RequestLimits::default(),+ execution_limits: ExecutionLimits::default(),})}}-fn concurrency_limit(- value: Option<&str>,- message: &'static str,-) -> Result<usize, GatewayConfigError> {- value- .unwrap_or("128")- .parse::<usize>()- .ok()- .filter(|limit| (1..=tokio::sync::Semaphore::MAX_PERMITS).contains(limit))- .ok_or(GatewayConfigError(message))+fn request_limits(+ max_request_bytes: Option<&str>,+ shared_buffer_bytes: Option<&str>,+) -> Result<RequestLimits, GatewayConfigError> {+ let defaults = RequestLimits::default();+ // One request reserves its body in a single semaphore acquisition, which takes a u32.+ let max_reservation = usize::try_from(u32::MAX)+ .unwrap_or(usize::MAX)+ .min(SEMAPHORE_CAPACITY);+ let max_request_bytes = integer_setting(+ max_request_bytes,+ defaults.max_request_bytes,+ MIB..=max_reservation,+ "LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES must be an integer of at least 1048576 (1 MiB) and within a single semaphore reservation",+ )?;+ let shared_buffer_bytes = integer_setting(+ shared_buffer_bytes,+ defaults.shared_buffer_bytes,+ max_request_bytes..=SEMAPHORE_CAPACITY,+ "LANGFUSE_AI_GATEWAY_SHARED_REQUEST_BUFFER_BYTES must be an integer of at least LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES within the semaphore capacity",+ )?;+ Ok(RequestLimits {+ max_request_bytes,+ shared_buffer_bytes,+ })+}++fn execution_limits(+ max_execution_time: Option<&str>,+ stall_timeout: Option<&str>,+) -> Result<ExecutionLimits, GatewayConfigError> {+ // `u32` seconds keep every deadline far from `Instant` overflow.+ let max_execution_time: u32 = integer_setting(+ max_execution_time,+ DEFAULT_MAX_EXECUTION_SECONDS,+ 60..=u32::MAX,+ "LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS must be an integer of at least 60",+ )?;+ let stall_timeout: u32 = integer_setting(+ stall_timeout,+ DEFAULT_STALL_TIMEOUT_SECONDS,+ 1..=max_execution_time,+ "LANGFUSE_AI_GATEWAY_STALL_TIMEOUT_SECONDS must be an integer from 1 to LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS",+ )?;+ Ok(ExecutionLimits {+ max_execution_time: Duration::from_secs(max_execution_time.into()),+ stall_timeout: Duration::from_secs(stall_timeout.into()),+ })}fn telemetry_buffer_bytes(value: Option<&str>) -> Result<usize, GatewayConfigError> {- const MIB: usize = 1024 * 1024;- let Some(value) = value else {- return Ok(crate::telemetry::DEFAULT_RETAINED_BYTES);- };+ integer_setting(+ value,+ crate::telemetry::DEFAULT_RETAINED_BYTES,+ 4 * MIB..=4096 * MIB,+ "LANGFUSE_AI_GATEWAY_TELEMETRY_BUFFER_BYTES must be an integer from 4194304 (4 MiB) to 4294967296 (4 GiB)",+ )+}++/// An unset value takes `default`. Errors carry only `message`, never the supplied value.+fn integer_setting<T: std::str::FromStr + PartialOrd>(+ value: Option<&str>,+ default: T,+ range: RangeInclusive<T>,+ message: &'static str,+) -> Result<T, GatewayConfigError> {value- .parse::<usize>()- .ok()- .filter(|bytes| (4 * MIB..=4096 * MIB).contains(bytes))- .ok_or(GatewayConfigError(- "LANGFUSE_AI_GATEWAY_TELEMETRY_BUFFER_BYTES must be an integer from 4194304 (4 MiB) to 4294967296 (4 GiB)",- ))+ .map_or(Some(default), |value| value.parse().ok())+ .filter(|value| range.contains(value))+ .ok_or(GatewayConfigError(message))}fn read_env(name: &'static str) -> Result<Option<String>, GatewayConfigError> {diff --git a/ai-gateway/src/http.rs b/ai-gateway/src/http.rsindex 55b53610bb..648ed9de98 100644--- a/ai-gateway/src/http.rs+++ b/ai-gateway/src/http.rs@@ -1,27 +1,27 @@-use std::{error::Error, ops::Range, sync::Arc, time::Duration};+use std::{ops::Range, sync::Arc};use axum::{Json, Router,- body::{Body, Bytes, to_bytes},+ body::{Body, Bytes},extract::{Request, State},http::{HeaderMap, StatusCode, header},response::{IntoResponse, Response},routing::{get, post},};+use http_body_util::BodyExt;use serde::{Deserialize, Serialize};use serde_json::value::RawValue;+use tower_http::timeout::{RequestBodyDeadlineLayer, TimeoutError};use tracing::Instrument;use crate::{correlation::{RequestCorrelation, assign_request_id},- inference::{InferenceService, RequestPreparationError},- providers::{ProviderError, Route},+ inference::{InferenceService, RequestBodyPermit, RequestPreparationError},+ providers::{ProviderError, RequestBody, Route},resolution::{ApiFormat, ResolutionError},server::GatewayLifecycleState,};-const MAX_REQUEST_BYTES: usize = 10 * 1024 * 1024;-const REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(30);const MAX_GATEWAY_KEY_BYTES: usize = 8192;#[derive(Clone)]@@ -32,7 +32,10 @@ struct InferenceRouteState {/// Build inference routes. An unconfigured or draining service returns 503.pub fn router(inference: Option<InferenceService>, lifecycle: GatewayLifecycleState) -> Router {- Router::new()+ let body_deadline = inference+ .as_ref()+ .map(InferenceService::request_read_timeout);+ let router = Router::new().route("/openai/v1/responses", post(handle_responses)).route("/openai/v1/responses/compact",@@ -49,8 +52,12 @@ pub fn router(inference: Option<InferenceService>, lifecycle: GatewayLifecycleSt.with_state(InferenceRouteState {inference: inference.map(Arc::new),lifecycle,- })- .layer(axum::middleware::from_fn(assign_request_id))+ });+ let router = match body_deadline {+ Some(deadline) => router.layer(RequestBodyDeadlineLayer::new(deadline)),+ None => router,+ };+ router.layer(axum::middleware::from_fn(assign_request_id))}async fn handle_responses(State(state): State<InferenceRouteState>, request: Request) -> Response {@@ -185,11 +192,19 @@ async fn relay(if !crate::transport::identity_encoding(request.headers()) {return Err(InferenceHttpError::UnsupportedEncoding);}- let _body_permit = inference+ // Held with the execution and released once the response ends.+ let body_permit = inference.try_reserve_request_body().map_err(InferenceHttpError::from)?;let (parts, body) = request.into_parts();- let request_body = request_body_and_model(body).await?;+ let (bytes, body_permit) = read_request_body(+ body,+ &parts.headers,+ inference.max_request_bytes(),+ body_permit,+ )+ .await?;+ let request_body = request_body_and_model(bytes)?;let (permit, context) = inference.resolve_and_admit(gateway_key,@@ -204,8 +219,12 @@ async fn relay(permit,context,&parts.headers,- request_body.bytes,- request_body.model_range,+ RequestBody {+ bytes: request_body.bytes,+ model_range: request_body.model_range,+ streaming: request_body.streaming,+ reservation: Some(Box::new(body_permit)),+ },route,correlation,)@@ -217,16 +236,19 @@ struct InferenceRequestBody {bytes: Bytes,model: String,model_range: Range<usize>,+ streaming: bool,}#[derive(Deserialize)]struct InferenceRequest<'a> {#[serde(borrow)]model: &'a RawValue,+ /// Kept raw so a non-boolean value is left for the provider to reject.+ #[serde(borrow, default)]+ stream: Option<&'a RawValue>,}-async fn request_body_and_model(body: Body) -> Result<InferenceRequestBody, InferenceHttpError> {- let bytes = read_request_body(body).await?;+fn request_body_and_model(bytes: Bytes) -> Result<InferenceRequestBody, InferenceHttpError> {let request = serde_json::from_slice::<InferenceRequest<'_>>(&bytes).map_err(|_| InferenceHttpError::InvalidBody)?;let raw_model = request.model.get();@@ -244,43 +266,81 @@ async fn request_body_and_model(body: Body) -> Result<InferenceRequestBody, Infe.checked_add(raw_model.len()).filter(|end| *end <= bytes.len()).ok_or(InferenceHttpError::InvalidBody)?;+ let streaming = request.stream.is_some_and(|stream| stream.get() == "true");Ok(InferenceRequestBody {bytes,model,model_range: start..end,+ streaming,})}-/// Buffer the client body. The wait is mostly the caller's upload, so it gets its-/// own span and the measured size lands on the server span for aggregation.-async fn read_request_body(body: Body) -> Result<Bytes, InferenceHttpError> {+/// Buffer the client body, reserving its allocation from the shared budget as bytes+/// arrive. A declared length is checked before reading and caps buffer growth, but is+/// not reserved up front, so a client that declares a large body and stalls holds only+/// what it actually sent. The router's body deadline bounds the read. The wait is mostly+/// the caller's upload, so it gets its own span and the measured size lands on the server+/// span for aggregation.+async fn read_request_body(+ body: Body,+ headers: &HeaderMap,+ max_bytes: usize,+ mut permit: RequestBodyPermit,+) -> Result<(Bytes, RequestBodyPermit), InferenceHttpError> {let server = tracing::Span::current();let span = tracing::info_span!("request.body",otel.kind = "internal",http.request.body.size = tracing::field::Empty);+ let declared = headers+ .get(header::CONTENT_LENGTH)+ .and_then(|value| value.to_str().ok()?.parse::<u64>().ok())+ .map(|length| usize::try_from(length).unwrap_or(usize::MAX));let bytes = async {- let bytes = tokio::time::timeout(REQUEST_READ_TIMEOUT, to_bytes(body, MAX_REQUEST_BYTES))- .await- .map_err(|_| InferenceHttpError::RequestTimeout)?- .map_err(|error| {- if error- .source()- .is_some_and(<dyn Error + 'static>::is::<http_body_util::LengthLimitError>)- {- InferenceHttpError::TooLarge- } else {- InferenceHttpError::InvalidBody- }- })?;+ if declared.is_some_and(|declared| declared > max_bytes) {+ return Err(InferenceHttpError::TooLarge);+ }+ let capacity_limit = declared.unwrap_or(max_bytes);+ let mut body = body;+ let mut buffer = Vec::new();+ while let Some(frame) = body.frame().await {+ let Ok(data) = frame.map_err(body_error)?.into_data() else {+ continue;+ };+ let total = buffer.len() + data.len();+ if total > max_bytes {+ return Err(InferenceHttpError::TooLarge);+ }+ if total > buffer.capacity() {+ let capacity = buffer+ .capacity()+ .saturating_mul(2)+ .clamp(total, capacity_limit.max(total));+ permit+ .try_reserve_bytes(capacity)+ .map_err(InferenceHttpError::Resolution)?;+ buffer.reserve_exact(capacity - buffer.len());+ }+ buffer.extend_from_slice(&data);+ }+ let bytes = Bytes::from(buffer);tracing::Span::current().record("http.request.body.size", byte_count(bytes.len()));Ok::<_, InferenceHttpError>(bytes)}.instrument(span).await?;server.record("http.request.body.size", byte_count(bytes.len()));- Ok(bytes)+ Ok((bytes, permit))+}++/// The router's body deadline surfaces as a body error.+fn body_error(error: axum::Error) -> InferenceHttpError {+ if error.into_inner().is::<TimeoutError>() {+ InferenceHttpError::RequestTimeout+ } else {+ InferenceHttpError::InvalidBody+ }}fn byte_count(len: usize) -> i64 {diff --git a/ai-gateway/src/http/tests.rs b/ai-gateway/src/http/tests.rsindex 1c25ce91e5..7c993826a8 100644--- a/ai-gateway/src/http/tests.rs+++ b/ai-gateway/src/http/tests.rs@@ -1,14 +1,17 @@use super::*;use crate::{+ config::{DEFAULT_MAX_REQUEST_BYTES as MAX_REQUEST_BYTES, ExecutionLimits, RequestLimits},providers::{ProviderLimits, ProviderTransport},resolution::ApiFormat,test_support::{FakeServer, resolution_response, resolution_response_for},};+use axum::body::to_bytes;use futures_util::{StreamExt, stream};use std::{convert::Infallible,io::Write,sync::atomic::{AtomicBool, Ordering},+ time::Duration,};use tokio::{net::TcpListener, sync::oneshot, task::JoinHandle};use tower::ServiceExt;@@ -1002,12 +1005,29 @@ async fn slow_resolution_has_a_bounded_wait() {#[tokio::test(start_paused = true)]async fn a_stalled_request_body_times_out_at_the_read_deadline() {+ let web = FakeServer::start(|_| async { resolution_response("provider") }).await;+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;+ let service = inference(&web, &provider);+ // The 16 MiB default keeps the 10 MiB per 30 seconds minimum upload rate.+ assert_eq!(service.request_read_timeout(), Duration::from_secs(49));+ let gateway = Gateway::start(Some(service)).await;let stalled = Body::from_stream(stream::pending::<Result<&'static str, Infallible>>());let started = tokio::time::Instant::now();- let result = read_request_body(stalled).await;- assert!(matches!(result, Err(InferenceHttpError::RequestTimeout)));- assert_eq!(started.elapsed(), REQUEST_READ_TIMEOUT);- assert_eq!(REQUEST_READ_TIMEOUT, Duration::from_secs(30));+ let request = gateway.app.clone().oneshot(+ Request::post("/openai/v1/responses")+ .header(header::AUTHORIZATION, "Bearer stalled-upload")+ .body(stalled)+ .unwrap(),+ );+ let response = tokio::time::timeout(Duration::from_secs(120), request)+ .await+ .expect("the router must bound the body read")+ .unwrap();+ assert_eq!(response.status(), StatusCode::REQUEST_TIMEOUT);+ assert_eq!(started.elapsed(), Duration::from_secs(49));+ let body = to_bytes(response.into_body(), 4096).await.unwrap();+ assert!(String::from_utf8_lossy(&body).contains("request_timeout"));+ assert_eq!(web.calls(), 0);}#[tokio::test]@@ -1042,7 +1062,10 @@ async fn pre_header_provider_failures_export_the_status_returned_to_the_caller()let provider = ProviderTransport::for_test(upstream_url,ProviderLimits {- headers_timeout: Duration::from_millis(100),+ execution: ExecutionLimits {+ stall_timeout: Duration::from_millis(100),+ ..ExecutionLimits::default()+ },..ProviderLimits::default()},)@@ -1056,7 +1079,7 @@ async fn pre_header_provider_failures_export_the_status_returned_to_the_caller()let response = gateway.post().bearer_auth("gateway-key")- .body(r#"{"model":"requested","input":"hello"}"#)+ .body(r#"{"model":"requested","input":"hello","stream":true}"#).send().await.unwrap();@@ -1302,7 +1325,7 @@ async fn active_admission_is_held_through_stream_and_shutdown_drains_it() {.status(),StatusCode::SERVICE_UNAVAILABLE);- assert_eq!(web.calls(), 2);+ assert_eq!(web.calls(), 1);assert_eq!(provider.calls(), 1);gateway.shutdown.take().unwrap().send(()).unwrap();assert!(!gateway.task.is_finished());@@ -1747,3 +1770,204 @@ async fn anthropic_routes_share_the_request_body_limit() {}assert_eq!(provider.calls(), 2);}++#[tokio::test]+async fn buffered_request_bytes_are_released_once_the_response_ends() {+ let (release, released) = tokio::sync::watch::channel(false);+ let web = FakeServer::start(|_| async { resolution_response("provider") }).await;+ let provider = FakeServer::start(move |request: Request| {+ let mut released = released.clone();+ async move {+ to_bytes(request.into_body(), usize::MAX).await.unwrap();+ released.wait_for(|done| *done).await.unwrap();+ Response::new(Body::empty())+ }+ })+ .await;+ let service = inference(&web, &provider).with_request_limits(RequestLimits {+ max_request_bytes: 64,+ shared_buffer_bytes: 100,+ });+ let gateway = Gateway::start(Some(service)).await;+ let body = |size: usize| {+ let prefix = r#"{"model":"Production key:test","padding":""#;+ format!("{prefix}{}\"}}", "x".repeat(size - prefix.len() - 2))+ };+ let oversized = gateway.post().bearer_auth("key").body(body(65)).send();+ assert_eq!(+ oversized.await.unwrap().status(),+ StatusCode::PAYLOAD_TOO_LARGE+ );+ // Two 60-byte calls do not fit the 100-byte budget at once.+ let first = tokio::spawn(gateway.post().bearer_auth("key").body(body(60)).send());+ while provider.calls() == 0 {+ tokio::task::yield_now().await;+ }+ let second = gateway.post().bearer_auth("key").body(body(60)).send();+ let second = tokio::time::timeout(Duration::from_secs(5), second)+ .await+ .expect("the first call still holds its bytes while it waits for headers");+ assert_eq!(second.unwrap().status(), StatusCode::SERVICE_UNAVAILABLE);+ release.send(true).unwrap();+ let first = first.await.unwrap().unwrap();+ assert_eq!(first.status(), StatusCode::OK);+ first.bytes().await.unwrap();+ tokio::time::timeout(Duration::from_secs(5), async {+ loop {+ let next = gateway.post().bearer_auth("key").body(body(60)).send();+ if next.await.unwrap().status() == StatusCode::OK {+ break;+ }+ tokio::task::yield_now().await;+ }+ })+ .await+ .expect("the first call's bytes are released once its response ends");+}++async fn budgeted_service() -> (FakeServer, FakeServer, InferenceService) {+ let web = FakeServer::start(|_| async { resolution_response("provider") }).await;+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;+ let service = inference(&web, &provider).with_request_limits(RequestLimits {+ max_request_bytes: 64,+ shared_buffer_bytes: 100,+ });+ (web, provider, service)+}++fn streamed_body(chunks: Vec<&'static [u8]>, stall: bool) -> Body {+ let chunks = stream::iter(chunks.into_iter().map(Ok::<_, Infallible>));+ if stall {+ Body::from_stream(chunks.chain(stream::pending()))+ } else {+ Body::from_stream(chunks)+ }+}++#[tokio::test]+async fn a_stalled_upload_holds_only_the_bytes_it_sent() {+ let (_web, _provider, service) = budgeted_service().await;+ let mut headers = HeaderMap::new();+ headers.insert(header::CONTENT_LENGTH, "60".parse().unwrap());+ let stalled = service.try_reserve_request_body().ok().unwrap();+ let read = read_request_body(+ streamed_body(vec![&[b'x'; 10]], true),+ &headers,+ 64,+ stalled,+ );+ tokio::pin!(read);+ assert!(+ tokio::time::timeout(Duration::from_millis(50), &mut read)+ .await+ .is_err()+ );+ let mut next = service.try_reserve_request_body().ok().unwrap();+ assert!(next.try_reserve_bytes(91).is_err());+ assert!(next.try_reserve_bytes(90).is_ok());+}++#[tokio::test]+async fn buffered_bytes_reserve_the_allocation_not_just_the_length() {+ let (_web, _provider, service) = budgeted_service().await;+ let held = service.try_reserve_request_body().ok().unwrap();+ let (bytes, _held) = read_request_body(+ streamed_body(vec![&[b'x'; 32], b"x"], false),+ &HeaderMap::new(),+ 64,+ held,+ )+ .await+ .ok()+ .unwrap();+ assert_eq!(bytes.len(), 33);+ let mut next = service.try_reserve_request_body().ok().unwrap();+ assert!(next.try_reserve_bytes(37).is_err());+ assert!(next.try_reserve_bytes(36).is_ok());+}++#[test]+fn only_a_literal_true_stream_member_marks_a_streamed_call() {+ for (body, streaming) in [+ (r#"{"model":"m","stream":true}"#, true),+ (r#"{"model":"m","stream":false}"#, false),+ (r#"{"model":"m","stream":"true"}"#, false),+ (r#"{"model":"m","stream":null}"#, false),+ (r#"{"model":"m"}"#, false),+ ] {+ let request = request_body_and_model(Bytes::from_static(body.as_bytes()))+ .ok()+ .unwrap();+ assert_eq!(request.streaming, streaming, "{body}");+ assert_eq!(&body.as_bytes()[request.model_range], br#""m""#);+ }+ for body in [+ r#"{"model":"m","model":"n"}"#,+ r#"{"model":"m","stream":true,"stream":true}"#,+ r#"{"stream":true}"#,+ ] {+ assert!(+ request_body_and_model(Bytes::from_static(body.as_bytes())).is_err(),+ "{body}"+ );+ }+}++#[tokio::test]+async fn only_non_streamed_generations_wait_past_the_stall_for_provider_headers() {+ let web = FakeServer::start(|_| async {+ resolution_response_for(ApiFormat::AnthropicMessages, "provider-anthropic")+ })+ .await;+ let provider = FakeServer::start(|_| async {+ tokio::time::sleep(Duration::from_millis(300)).await;+ Response::new(Body::from("{}"))+ })+ .await;+ let service = InferenceService::for_test(+ web.control_plane(),+ ProviderTransport::for_test(+ format!("{}/v1", provider.url),+ ProviderLimits {+ execution: ExecutionLimits {+ stall_timeout: Duration::from_millis(100),+ ..ExecutionLimits::default()+ },+ ..ProviderLimits::default()+ },+ ),+ 128,+ );+ let gateway = Gateway::start(Some(service)).await;+ let client = reqwest::Client::new();+ for (path, body, status) in [+ (+ "messages",+ r#"{"model":"Production key:test"}"#,+ StatusCode::OK,+ ),+ (+ "messages",+ r#"{"model":"Production key:test","stream":false}"#,+ StatusCode::OK,+ ),+ (+ "messages",+ r#"{"model":"Production key:test","stream":true}"#,+ StatusCode::GATEWAY_TIMEOUT,+ ),+ (+ "messages/count_tokens",+ r#"{"model":"Production key:test"}"#,+ StatusCode::GATEWAY_TIMEOUT,+ ),+ ] {+ let response = client+ .post(format!("{}/anthropic/v1/{path}", gateway.url))+ .header("x-api-key", "gateway-anthropic")+ .body(body)+ .send()+ .await;+ assert_eq!(response.unwrap().status(), status, "{path} {body}");+ }+}diff --git a/ai-gateway/src/inference.rs b/ai-gateway/src/inference.rsindex 23282f2dda..eb951a689c 100644--- a/ai-gateway/src/inference.rs+++ b/ai-gateway/src/inference.rs@@ -1,16 +1,17 @@//! Resolve the caller's credential, then execute using only the trusted context.-use std::ops::Range;+use std::{sync::Arc, time::Duration};use axum::{- body::{Body, Bytes},+ body::Body,http::{HeaderMap, Response},};-use tokio::sync::{Semaphore, SemaphorePermit};+use tokio::sync::{OwnedSemaphorePermit, Semaphore, SemaphorePermit};use tracing::Instrument;use crate::{+ config::{GatewayConfig, RequestLimits},correlation::RequestCorrelation,- providers::{ProviderError, ProviderTransport, RequestPermit, Route},+ providers::{ProviderError, ProviderTransport, RequestBody, RequestPermit, Route},resolution::{ApiFormat, ControlPlaneClient, ControlPlaneConfig, ResolutionError, ResolvedRequestContext,},@@ -19,7 +20,9 @@ use crate::{pub struct InferenceService {control_plane: ControlPlaneClient,provider: ProviderTransport,- request_body_capacity: Semaphore,+ request_body_capacity: Arc<Semaphore>,+ request_limits: RequestLimits,+ request_bytes_capacity: Arc<Semaphore>,model_catalog_capacity: Semaphore,resolution_capacity: Semaphore,telemetry: Option<crate::telemetry::Telemetry>,@@ -30,31 +33,71 @@ pub(crate) enum RequestPreparationError {Provider(ProviderError),}-pub(crate) struct RequestBodyPermit<'a> {- _permit: SemaphorePermit<'a>,+/// One request-body slot plus the body bytes reserved so far from the shared budget.+/// It is held with the execution and released once the response ends.+pub(crate) struct RequestBodyPermit {+ _slot: OwnedSemaphorePermit,+ bytes: OwnedSemaphorePermit,+ budget: Arc<Semaphore>,}+impl RequestBodyPermit {+ /// Grow the reservation to `total` body bytes without waiting.+ ///+ /// # Errors+ /// Returns [`ResolutionError::Unavailable`] when the shared budget cannot cover `total`.+ pub(crate) fn try_reserve_bytes(&mut self, total: usize) -> Result<(), ResolutionError> {+ let Some(missing) = total+ .checked_sub(self.bytes.num_permits())+ .filter(|missing| *missing > 0)+ else {+ return Ok(());+ };+ let reserved = u32::try_from(missing)+ .ok()+ .and_then(|missing| {+ Arc::clone(&self.budget)+ .try_acquire_many_owned(missing)+ .ok()+ })+ .ok_or_else(|| {+ crate::observability::rejected("request_bytes");+ ResolutionError::Unavailable+ })?;+ self.bytes.merge(reserved);+ Ok(())+ }+}++/// Uploads slower than 10 MiB per 30 seconds on average miss the read deadline.+const MIN_REQUEST_UPLOAD_BYTES_PER_SECOND: usize = 10 * 1024 * 1024 / 30;+const MIN_REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(30);+impl InferenceService {/// Initialize pooled clients for resolution and provider requests.////// # Errors/// Returns a sanitized configuration error for invalid capacities or client setup.pub fn new(- config: ControlPlaneConfig,- max_active_requests: usize,- max_concurrent_resolutions: usize,- telemetry_buffer_bytes: usize,+ control_plane: ControlPlaneConfig,+ config: &GatewayConfig,) -> Result<Self, ResolutionError> {+ let max_concurrent_resolutions = config.max_concurrent_resolutions;if !(1..=Semaphore::MAX_PERMITS).contains(&max_concurrent_resolutions) {return Err(ResolutionError::Configuration);}- let telemetry = crate::telemetry::Telemetry::new(&config, telemetry_buffer_bytes)?;+ let telemetry =+ crate::telemetry::Telemetry::new(&control_plane, config.telemetry_buffer_bytes)?;Ok(Self {- control_plane: ControlPlaneClient::new(config)?,- provider: ProviderTransport::new(max_active_requests)+ control_plane: ControlPlaneClient::new(control_plane)?,+ provider: ProviderTransport::new(config.max_active_requests, config.execution_limits).map_err(|_| ResolutionError::Configuration)?.with_telemetry(telemetry.clone()),- request_body_capacity: Semaphore::new(max_active_requests),+ request_body_capacity: Arc::new(Semaphore::new(config.max_active_requests)),+ request_limits: config.request_limits,+ request_bytes_capacity: Arc::new(Semaphore::new(+ config.request_limits.shared_buffer_bytes,+ )),model_catalog_capacity: Semaphore::new(max_concurrent_resolutions),resolution_capacity: Semaphore::new(max_concurrent_resolutions),telemetry: Some(telemetry),@@ -74,12 +117,36 @@ impl InferenceService {pub(crate) fn try_reserve_request_body(&self,- ) -> Result<RequestBodyPermit<'_>, RequestPreparationError> {- let permit = self.request_body_capacity.try_acquire().map_err(|_| {- crate::observability::rejected("request_body");- RequestPreparationError::Resolution(ResolutionError::Unavailable)- })?;- Ok(RequestBodyPermit { _permit: permit })+ ) -> Result<RequestBodyPermit, RequestPreparationError> {+ let slot = Arc::clone(&self.request_body_capacity)+ .try_acquire_owned()+ .map_err(|_| {+ crate::observability::rejected("request_body");+ RequestPreparationError::Resolution(ResolutionError::Unavailable)+ })?;+ let bytes = Arc::clone(&self.request_bytes_capacity)+ .try_acquire_many_owned(0)+ .map_err(|_| RequestPreparationError::Resolution(ResolutionError::Unavailable))?;+ Ok(RequestBodyPermit {+ _slot: slot,+ bytes,+ budget: Arc::clone(&self.request_bytes_capacity),+ })+ }++ pub(crate) fn max_request_bytes(&self) -> usize {+ self.request_limits.max_request_bytes+ }++ /// The body read deadline scales with the size limit so a maximal body has the same+ /// minimum upload rate as a small one.+ pub(crate) fn request_read_timeout(&self) -> Duration {+ let seconds = self+ .request_limits+ .max_request_bytes+ .div_ceil(MIN_REQUEST_UPLOAD_BYTES_PER_SECOND);+ Duration::from_secs(u64::try_from(seconds).unwrap_or(u64::MAX))+ .max(MIN_REQUEST_READ_TIMEOUT)}pub(crate) async fn fetch_model_catalog(@@ -159,30 +226,17 @@ impl InferenceService {Ok((permit, context))}- #[expect(- clippy::too_many_arguments,- reason = "each argument is a distinct per-request input passed straight through"- )]pub(crate) async fn forward(&self,permit: RequestPermit,context: ResolvedRequestContext,headers: &HeaderMap,- body: Bytes,- model_range: Range<usize>,+ request: RequestBody,route: Route,correlation: &mut RequestCorrelation,) -> Result<Response<Body>, ProviderError> {self.provider- .forward_route(- permit,- context,- headers,- body,- model_range,- route,- correlation,- )+ .forward_route(permit, context, headers, request, route, correlation).await}@@ -195,10 +249,21 @@ impl InferenceService {Self {control_plane,provider,- request_body_capacity: Semaphore::new(resolutions),+ request_body_capacity: Arc::new(Semaphore::new(resolutions)),+ request_limits: RequestLimits::default(),+ request_bytes_capacity: Arc::new(Semaphore::new(+ RequestLimits::default().shared_buffer_bytes,+ )),model_catalog_capacity: Semaphore::new(resolutions),resolution_capacity: Semaphore::new(resolutions),telemetry: None,}}++ #[cfg(test)]+ pub(crate) fn with_request_limits(mut self, request_limits: RequestLimits) -> Self {+ self.request_bytes_capacity = Arc::new(Semaphore::new(request_limits.shared_buffer_bytes));+ self.request_limits = request_limits;+ self+ }}diff --git a/ai-gateway/src/main.rs b/ai-gateway/src/main.rsindex bc52e0e640..b32b21f87d 100644--- a/ai-gateway/src/main.rs+++ b/ai-gateway/src/main.rs@@ -25,18 +25,12 @@ async fn main() -> ExitCode {}async fn run() -> Result<(), Box<dyn Error>> {- let config = GatewayConfig::from_env()?;+ let mut config = GatewayConfig::from_env()?;let observability = observability::init(&config)?;let inference = config.control_plane- .map(|control_plane| {- InferenceService::new(- control_plane,- config.max_active_requests,- config.max_concurrent_resolutions,- config.telemetry_buffer_bytes,- )- })+ .take()+ .map(|control_plane| InferenceService::new(control_plane, &config)).transpose()?;let inference_enabled = inference.is_some();let telemetry = inference.as_ref().and_then(InferenceService::telemetry);@@ -51,7 +45,7 @@ async fn run() -> Result<(), Box<dyn Error>> {)));let shutdown = shutdown_signal()?;let listener = bind_listener(config.listen_address, config.auto_increment_listen_port).await?;- tracing::info!(address = %listener.local_addr()?, inference_enabled, max_active_requests = config.max_active_requests, max_concurrent_resolutions = config.max_concurrent_resolutions, shutdown_timeout_seconds = config.shutdown_timeout.as_secs(), "gateway listening");+ tracing::info!(address = %listener.local_addr()?, inference_enabled, max_active_requests = config.max_active_requests, max_concurrent_resolutions = config.max_concurrent_resolutions, max_request_bytes = config.request_limits.max_request_bytes, shared_request_buffer_bytes = config.request_limits.shared_buffer_bytes, max_execution_seconds = config.execution_limits.max_execution_time.as_secs(), stall_timeout_seconds = config.execution_limits.stall_timeout.as_secs(), shutdown_timeout_seconds = config.shutdown_timeout.as_secs(), "gateway listening");let result = server::serve(listener,app,diff --git a/ai-gateway/src/providers/mod.rs b/ai-gateway/src/providers/mod.rsindex cda1a77e27..4b1aa145c8 100644--- a/ai-gateway/src/providers/mod.rs+++ b/ai-gateway/src/providers/mod.rs@@ -18,6 +18,7 @@ use tokio::{pub use crate::transport::ProviderError;use crate::{capture::{ExecutionCapture, RelayOutcome},+ config::ExecutionLimits,correlation::{RequestCorrelation, UpstreamTarget},resolution::{ApiFormat, Provider, ProviderCredential, ResolvedRequestContext},transport,@@ -75,18 +76,14 @@ impl Route {pub(crate) struct ProviderLimits {pub active: usize,- pub execution_timeout: Duration,- pub headers_timeout: Duration,- pub read_timeout: Duration,+ pub execution: ExecutionLimits,}impl Default for ProviderLimits {fn default() -> Self {Self {active: 128,- execution_timeout: Duration::from_secs(600),- headers_timeout: Duration::from_secs(120),- read_timeout: Duration::from_secs(120),+ execution: ExecutionLimits::default(),}}}@@ -97,6 +94,18 @@ pub struct RequestPermit {deadline: Instant,}+/// The validated caller body and the facts the relay needs about it.+pub(crate) struct RequestBody {+ pub bytes: Bytes,+ pub model_range: Range<usize>,+ /// The caller asked for a streamed response (`"stream": true`).+ pub streaming: bool,+ /// Capacity held for the body, released with the execution once the response ends.+ pub reservation: Option<BodyReservation>,+}++pub(crate) type BodyReservation = Box<dyn Send + Sync>;+pub struct ProviderTransport {client: ClientWithMiddleware,capacity: Arc<Semaphore>,@@ -108,10 +117,13 @@ pub struct ProviderTransport {impl ProviderTransport {/// # Errors/// Returns [`ProviderError::Configuration`] when the HTTPS client cannot be initialized.- pub fn new(max_active_requests: usize) -> Result<Self, ProviderError> {+ pub fn new(+ max_active_requests: usize,+ execution: ExecutionLimits,+ ) -> Result<Self, ProviderError> {Self::with_limits(ProviderLimits {active: max_active_requests,- ..ProviderLimits::default()+ execution,})}@@ -127,8 +139,9 @@ impl ProviderTransport {.no_brotli().no_deflate().no_zstd()+ // No client read timeout: it would also cut a non-streaming call while the+ // provider generates before sending headers. The relay bounds stalls instead..connect_timeout(Duration::from_secs(5))- .read_timeout(limits.read_timeout).pool_max_idle_per_host(16).build().map_err(|_| ProviderError::Configuration)?;@@ -156,10 +169,12 @@ impl ProviderTransport {Ok(RequestPermit {_permit: permit,_active: crate::observability::Active::new("execution"),- deadline: Instant::now() + self.limits.execution_timeout,+ deadline: Instant::now() + self.limits.execution.max_execution_time,})}+ /// Forward as a streamed `OpenAI` Responses call.+ ////// # Errors/// See [`Self::forward_route`].#[cfg(test)]@@ -175,28 +190,33 @@ impl ProviderTransport {permit,context,headers,- body,- model_range,+ RequestBody {+ bytes: body,+ model_range,+ streaming: true,+ reservation: None,+ },Route::OpenAiResponses,&mut RequestCorrelation::from_headers(headers),).await}- #[expect(- clippy::too_many_arguments,- reason = "each argument is a distinct per-request input of the relay"- )]pub(crate) async fn forward_route(&self,permit: RequestPermit,context: ResolvedRequestContext,headers: &HeaderMap,- body: Bytes,- model_range: Range<usize>,+ request: RequestBody,route: Route,correlation: &mut RequestCorrelation,) -> Result<Response<Body>, ProviderError> {+ let RequestBody {+ bytes: body,+ model_range,+ streaming,+ reservation,+ } = request;let api_format = route.api_format();let (credential_name, mut credential) = provider_credential(&context)?;credential.set_sensitive(true);@@ -236,23 +256,28 @@ impl ProviderTransport {.header(header::ACCEPT_ENCODING, "identity").header(credential_name, credential);upstream = upstream.body(body);- let response = tokio::time::timeout_at(+ // A stream, or a call that does not generate, that goes quiet before its+ // headers is stalled. A non-streamed generation sends nothing until the whole+ // response is generated.+ let headers_deadline = if streaming || !route.captures_generation() {permit.deadline- .min(Instant::now() + self.limits.headers_timeout),- upstream.send(),- )- .await- .map_err(|_| ProviderError::Timeout)- .and_then(|result| {- result.map_err(|error| {- if error.is_timeout() {- ProviderError::Timeout- } else {- ProviderError::Transport- }- })- });+ .min(Instant::now() + self.limits.execution.stall_timeout)+ } else {+ permit.deadline+ };+ let response = tokio::time::timeout_at(headers_deadline, upstream.send())+ .await+ .map_err(|_| ProviderError::Timeout)+ .and_then(|result| {+ result.map_err(|error| {+ if error.is_timeout() {+ ProviderError::Timeout+ } else {+ ProviderError::Transport+ }+ })+ });let response = match response {Ok(response) => response,Err(error) => {@@ -271,8 +296,13 @@ impl ProviderTransport {let mut downstream = Response::new(Body::empty());*downstream.status_mut() = response.status();*downstream.headers_mut() = transport::response_headers(response.headers(), api_format);- *downstream.body_mut() =- transport::relay(response, permit.deadline, (permit, context), capture);+ *downstream.body_mut() = transport::relay(+ response,+ permit.deadline,+ self.limits.execution.stall_timeout,+ (permit, context, reservation),+ capture,+ );Ok(downstream)}diff --git a/ai-gateway/src/providers/tests.rs b/ai-gateway/src/providers/tests.rsindex cce2497bd7..8997824fef 100644--- a/ai-gateway/src/providers/tests.rs+++ b/ai-gateway/src/providers/tests.rs@@ -1,4 +1,5 @@use super::*;+use crate::config::ExecutionLimits;use crate::test_support::{FakeServer, ingestion_token, resolved_request_context, resolved_request_context_for,};@@ -69,8 +70,12 @@ async fn invalid_model_rewrite_never_reaches_the_provider() {relay.try_admit().unwrap(),resolved_request_context("provider-secret").await,&HeaderMap::new(),- Bytes::from_static(br#"{"model":"requested"}"#),- 100..110,+ RequestBody {+ bytes: Bytes::from_static(br#"{"model":"requested"}"#),+ model_range: 100..110,+ streaming: true,+ reservation: None,+ },Route::OpenAiResponses,&mut RequestCorrelation::default(),)@@ -412,7 +417,10 @@ async fn deadlines_bound_headers_and_stalled_bodies_without_exposing_transport_dlet relay = ProviderTransport::for_test(upstream.url.clone(),ProviderLimits {- headers_timeout: Duration::from_millis(30),+ execution: ExecutionLimits {+ stall_timeout: Duration::from_millis(30),+ ..ExecutionLimits::default()+ },..ProviderLimits::default()},);@@ -432,7 +440,10 @@ async fn deadlines_bound_headers_and_stalled_bodies_without_exposing_transport_dlet relay = ProviderTransport::for_test(upstream.url.clone(),ProviderLimits {- execution_timeout: Duration::from_millis(100),+ execution: ExecutionLimits {+ max_execution_time: Duration::from_millis(100),+ ..ExecutionLimits::default()+ },..ProviderLimits::default()},);@@ -591,8 +602,10 @@ async fn cancelled_and_timed_out_executions_upload_after_provider_context_is_relupstream.url.clone(),ProviderLimits {active: 1,- execution_timeout: Duration::from_millis(150),- ..ProviderLimits::default()+ execution: ExecutionLimits {+ max_execution_time: Duration::from_millis(150),+ ..ExecutionLimits::default()+ },},).with_telemetry(telemetry.clone());@@ -698,8 +711,12 @@ async fn compact_posts_compact_path_and_captures_ingestion() {relay.try_admit().unwrap(),resolved_request_context("provider-secret").await,&HeaderMap::new(),- Bytes::from_static(COMPACT),- model_range(COMPACT).unwrap(),+ RequestBody {+ bytes: Bytes::from_static(COMPACT),+ model_range: model_range(COMPACT).unwrap(),+ streaming: true,+ reservation: None,+ },Route::OpenAiResponsesCompact,&mut RequestCorrelation::default(),)@@ -903,8 +920,12 @@ async fn sends_the_x_api_key_credential_and_forwards_the_open_anthropic_header_f].concat(),),- Bytes::from_static(REQUEST),- model_range(REQUEST).unwrap(),+ RequestBody {+ bytes: Bytes::from_static(REQUEST),+ model_range: model_range(REQUEST).unwrap(),+ streaming: true,+ reservation: None,+ },Route::AnthropicMessages,&mut RequestCorrelation::default(),)@@ -957,8 +978,13 @@ async fn count_tokens_relays_the_body() {relay.try_admit().unwrap(),context,&headers(&[("content-type", "application/json")]),- Bytes::from_static(br#"{"model":"claude-sonnet-4-5","messages":[]}"#),- model_range(br#"{"model":"claude-sonnet-4-5","messages":[]}"#).unwrap(),+ RequestBody {+ bytes: Bytes::from_static(br#"{"model":"claude-sonnet-4-5","messages":[]}"#),+ model_range: model_range(br#"{"model":"claude-sonnet-4-5","messages":[]}"#)+ .unwrap(),+ streaming: true,+ reservation: None,+ },Route::AnthropicCountTokens,&mut RequestCorrelation::default(),)@@ -1075,8 +1101,12 @@ async fn chat_completions_streams_request_usage_and_relay_the_usage_chunk() {].concat(),),- Bytes::from_static(request.as_bytes()),- model_range(request.as_bytes()).unwrap(),+ RequestBody {+ bytes: Bytes::from_static(request.as_bytes()),+ model_range: model_range(request.as_bytes()).unwrap(),+ streaming: true,+ reservation: None,+ },Route::OpenAiChatCompletions,&mut RequestCorrelation::default(),)@@ -1123,8 +1153,12 @@ async fn chat_completions_rewrite_non_streamed_models() {).await,&headers(&[("content-type", "application/json")]),- Bytes::from_static(JSON_REQUEST.as_bytes()),- model_range(JSON_REQUEST.as_bytes()).unwrap(),+ RequestBody {+ bytes: Bytes::from_static(JSON_REQUEST.as_bytes()),+ model_range: model_range(JSON_REQUEST.as_bytes()).unwrap(),+ streaming: true,+ reservation: None,+ },Route::OpenAiChatCompletions,&mut RequestCorrelation::default(),)diff --git a/ai-gateway/src/transport/mod.rs b/ai-gateway/src/transport/mod.rsindex 373ac3a0f4..754d3167f6 100644--- a/ai-gateway/src/transport/mod.rs+++ b/ai-gateway/src/transport/mod.rs@@ -7,6 +7,7 @@ use std::{atomic::{AtomicBool, AtomicUsize, Ordering},},task::{Context, Poll},+ time::Duration,};use crate::{@@ -151,6 +152,7 @@ pub(crate) fn identity_encoding(headers: &HeaderMap) -> bool {pub(crate) fn relay<T: Send + 'static>(upstream: reqwest::Response,deadline: Instant,+ stall_timeout: Duration,owner: T,capture: ExecutionCapture,) -> Body {@@ -165,14 +167,18 @@ pub(crate) fn relay<T: Send + 'static>(})}),deadline,+ stall_timeout,owner,Some(capture),)}+/// Relay until EOF, the overall `deadline`, or `stall_timeout` without progress+/// in either direction: no upstream chunk, or a downstream that stopped reading.fn relay_stream<S, T>(upstream: S,deadline: Instant,+ stall_timeout: Duration,owner: T,capture: Option<ExecutionCapture>,) -> Body@@ -206,7 +212,10 @@ wherelet task = tokio::spawn(async move {let pump = async {tokio::pin!(upstream);- while let Some(chunk) = upstream.next().await {+ while let Some(chunk) = tokio::time::timeout(stall_timeout, upstream.next())+ .await+ .map_err(|_| ProviderError::Timeout)?+ {let chunk = chunk?;pump_resources.bytes@@ -215,7 +224,13 @@ wherepump_resources.observe(|capture| capture.push_bytes(&chunk));// One queued chunk plus one pending send; no per-chunk tasks.for bytes in chunk.chunks(64 * 1024) {- if sender.send(Bytes::copy_from_slice(bytes)).await.is_err() {+ let sent = tokio::time::timeout(+ stall_timeout,+ sender.send(Bytes::copy_from_slice(bytes)),+ )+ .await+ .map_err(|_| ProviderError::Timeout)?;+ if sent.is_err() {return Ok(());}}@@ -236,10 +251,13 @@ where}drop(sender);// Upstream EOF alone does not release admission: the downstream may still- // have queued bytes. The deadline also covers an unpolled downstream body.- if tokio::time::timeout_at(deadline, pump_resources.released.notified())- .await- .is_err()+ // have queued bytes. The stall timeout also covers an unpolled downstream body.+ if tokio::time::timeout_at(+ deadline.min(Instant::now() + stall_timeout),+ pump_resources.released.notified(),+ )+ .await+ .is_err(){pump_resources.release(RelayOutcome::Timeout);}@@ -464,6 +482,7 @@ mod tests {Err(ProviderError::Transport),]),Instant::now() + Duration::from_secs(1),+ Duration::from_secs(10),Owner(released.clone()),None,);@@ -504,6 +523,7 @@ mod tests {let body = relay_stream(upstream,Instant::now() + Duration::from_secs(10),+ Duration::from_secs(10),Owner(released.clone()),None,);@@ -522,6 +542,7 @@ mod tests {let body = relay_stream(stream::iter([Ok(bytes.clone()), Err(ProviderError::Transport)]),Instant::now() + Duration::from_secs(1),+ Duration::from_secs(10),Owner(released.clone()),None,);@@ -541,6 +562,7 @@ mod tests {let body = relay_stream(stream::once(async { Ok(Bytes::from_static(b"final bytes")) }),Instant::now() + Duration::from_millis(30),+ Duration::from_secs(10),Owner(released.clone()),None,);@@ -549,4 +571,43 @@ mod tests {.unwrap();assert!(to_bytes(body, 1024).await.is_err());}++ #[tokio::test(start_paused = true)]+ async fn a_silent_upstream_stalls_out_long_before_the_deadline() {+ let released = Arc::new(Notify::new());+ let started = Instant::now();+ let body = relay_stream(+ stream::once(async { Ok(Bytes::from_static(b"first")) }).chain(stream::pending()),+ started + Duration::from_secs(3600),+ Duration::from_secs(120),+ Owner(released.clone()),+ None,+ );+ let mut downstream = body.into_data_stream();+ assert_eq!(downstream.next().await.unwrap().unwrap(), "first");+ released.notified().await;+ assert_eq!(started.elapsed(), Duration::from_secs(120));+ assert!(downstream.next().await.unwrap().is_err());+ }++ #[tokio::test(start_paused = true)]+ async fn a_downstream_that_stops_reading_stalls_out() {+ let released = Arc::new(Notify::new());+ let started = Instant::now();+ // Steady upstream progress must not keep a non-reading client's slot alive.+ let upstream =+ stream::repeat_with(|| Ok(Bytes::from_static(b"chunk"))).then(|chunk| async {+ tokio::time::sleep(Duration::from_secs(1)).await;+ chunk+ });+ let _body = relay_stream(+ upstream,+ started + Duration::from_secs(3600),+ Duration::from_secs(120),+ Owner(released.clone()),+ None,+ );+ released.notified().await;+ assert!(started.elapsed() < Duration::from_secs(125));+ }}diff --git a/ai-gateway/tests/startup.rs b/ai-gateway/tests/startup.rsindex ba9ff20057..813e0429cd 100644--- a/ai-gateway/tests/startup.rs+++ b/ai-gateway/tests/startup.rs@@ -12,6 +12,10 @@ fn invalid_configuration_exits_without_disclosing_the_value() {"LANGFUSE_AI_GATEWAY_WEB_URL","LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS","LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS",+ "LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES",+ "LANGFUSE_AI_GATEWAY_SHARED_REQUEST_BUFFER_BYTES",+ "LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS",+ "LANGFUSE_AI_GATEWAY_STALL_TIMEOUT_SECONDS",] {let output = Command::new(env!("CARGO_BIN_EXE_ai-gateway")).env_clear()diff --git a/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.ts b/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.tsindex 87078c54f5..89df403959 100644--- a/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.ts+++ b/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.ts@@ -12,6 +12,7 @@ import {import { requireGatewayEnabledForOrganization } from "../availability";import { verifyGatewayRequestSignature } from "./gatewayRequestSignatureVerifier";+/** Outlives the gateway's default ten-minute execution ceiling plus upload batching. */export const GATEWAY_INGESTION_TOKEN_TTL_SECONDS = 15 * 60;const GatewayIngestionClaimsSchema = z.strictObject({