Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): raise request limit to 16 MiB and make execution limits configurable (#18820)

Tobias Wochinger6b6b103c98d0
diff --git a/.env.dev.example b/.env.dev.example
index 255a3bc42a..789e2fc44a 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -211,6 +211,12 @@ ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000
# Independent per-instance concurrency limits; provisional defaults, tune with load tests.
# LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS=128
# LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS=128
+# Largest request body, and the body bytes all in-flight requests may buffer together.
+# LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES=16777216
+# LANGFUSE_AI_GATEWAY_SHARED_REQUEST_BUFFER_BYTES=1073741824
+# Execution ceiling and the longest wait without stream progress.
+# LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS=600
+# LANGFUSE_AI_GATEWAY_STALL_TIMEOUT_SECONDS=120
# AI gateway control-plane key rotation. Generate service keys with
# `openssl rand -hex 32` and ES256 PEM keys with
diff --git a/.env.prod.example b/.env.prod.example
index dbfc328f45..af35519fe5 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -585,6 +585,12 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
# Independent per-instance concurrency limits; provisional defaults, tune with load tests.
# LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS=128
# LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS=128
+# Largest request body, and the body bytes all in-flight requests may buffer together.
+# LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES=16777216
+# LANGFUSE_AI_GATEWAY_SHARED_REQUEST_BUFFER_BYTES=1073741824
+# Execution ceiling and the longest wait without stream progress.
+# LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS=600
+# LANGFUSE_AI_GATEWAY_STALL_TIMEOUT_SECONDS=120
# Internal cloud trace-batch reader controls; these do not enable processing.
# ClickHouse socket idle timeout during streaming, including summary backpressure.
diff --git a/ai-gateway/Cargo.lock b/ai-gateway/Cargo.lock
index ea0e37c9b9..72a0fe70e6 100644
--- a/ai-gateway/Cargo.lock
+++ b/ai-gateway/Cargo.lock
@@ -1779,6 +1779,7 @@ dependencies = [
"http-body",
"percent-encoding",
"pin-project-lite",
+ "tokio",
"tower-layer",
"tower-service",
"tracing",
diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.toml
index c48fa1fc3f..6e3a8bfb15 100644
--- a/ai-gateway/Cargo.toml
+++ b/ai-gateway/Cargo.toml
@@ -24,7 +24,7 @@ serde = { version = "1", features = ["derive"] }
serde_json = { version = "1", features = ["raw_value"] }
sha2 = "0.11"
tokio = { version = "1.53.1", features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] }
-tower-http = { version = "0.7", features = ["trace"] }
+tower-http = { version = "0.7", features = ["timeout", "trace"] }
tracing = "0.1.44"
tracing-opentelemetry = { version = "0.34", default-features = false }
tracing-subscriber = { version = "0.3.23", default-features = false, features = ["env-filter", "fmt", "json"] }
diff --git a/ai-gateway/README.md b/ai-gateway/README.md
index e083b797a3..6040c4a1b9 100644
--- a/ai-gateway/README.md
+++ b/ai-gateway/README.md
@@ -81,6 +81,10 @@ do not load dotenv files:
| `LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS` | `128` | Positive integer up to Tokio's semaphore capacity; buffered and active requests per instance |
| `LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS` | `128` | Positive integer up to Tokio's semaphore capacity; separate model-catalog and Web-resolution budgets per instance |
| `LANGFUSE_AI_GATEWAY_TELEMETRY_BUFFER_BYTES` | `67108864` (64 MiB) | Integer from 4194304 (4 MiB) to 4294967296 (4 GiB); span and credential bytes an instance holds for upload before dropping new records |
+| `LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES` | `16777216` (16 MiB) | Integer from 1048576 (1 MiB) to 4294967295; largest accepted request body |
+| `LANGFUSE_AI_GATEWAY_SHARED_REQUEST_BUFFER_BYTES` | `1073741824` (1 GiB) | Integer from the request limit up to Tokio's semaphore capacity; request body bytes all in-flight requests of an instance may buffer together |
+| `LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS` | `600` | Integer of at least 60; ceiling from execution admission until the response completes. Web ingestion tokens last 15 minutes, so traces of longer executions may be dropped |
+| `LANGFUSE_AI_GATEWAY_STALL_TIMEOUT_SECONDS` | `120` | Integer from 1 to the execution ceiling; longest wait without progress |
The gateway shares `LANGFUSE_LOG_LEVEL` with Web and worker. Values are lowercase;
`fatal` maps to Rust's `error` level and therefore includes ordinary error logs.
@@ -372,28 +376,35 @@ error type follows the status (`authentication_error`, `permission_error`,
`api_error`).
After validating the credential header, the gateway buffers and validates the request
-under the active-request admission budget, then resolves the credential and
-model-aware route through Web under a separate budget. It holds the body slot through
-the provider request. Model catalogs use another budget so slow catalog discovery
+under the active-request admission budget and a shared request-byte budget, then resolves the credential and
+model-aware route through Web under a separate budget. A declared `Content-Length` is
+checked against the request limit before reading. The byte budget covers the buffer's
+allocation and grows as bytes arrive, so a client that declares a large body and stalls
+holds only what it sent. The body slot and its bytes stay reserved with the execution
+until the response ends. Model catalogs use another budget so slow catalog discovery
cannot block inference resolution.
Execution capacity remains reserved through response completion. An exhausted budget
returns 503 immediately without a waiting queue. Request-body and execution budgets
-use the active-request limit; model-catalog and Web-call budgets use the resolution
+use the active-request limit, and buffered bodies also share
+`LANGFUSE_AI_GATEWAY_SHARED_REQUEST_BUFFER_BYTES`; model-catalog and Web-call budgets use the resolution
limit. Their defaults of 128 are provisional
guardrails, not measured capacity: tune them using load tests for the instance
resources, request sizes and stream durations. These limits bound work; they do not
guarantee fairness between clients or tenants.
-Other limits are 10 MiB request bodies in both namespaces, 30 seconds to read a request, 5 seconds to
-connect, 120 seconds for provider response headers or an individual upstream read,
-and 600 seconds overall from execution admission in both namespaces. Response size is not capped: a single task pumps chunks
+Request bodies are limited to 16 MiB by default in both namespaces. The body read deadline
+scales with that limit: at least 30 seconds, and enough for the maximal body at 10 MiB per
+30 seconds (49 seconds at 16 MiB).
+Connecting to a provider may take 5 seconds. Executions end at a ten-minute ceiling from
+admission by default, and earlier when they stall: 120 seconds by default without an
+upstream chunk, without the downstream accepting a chunk, or, for streamed calls
+(`"stream": true`) and token counting, without provider response
+headers. A non-streamed generation waits for headers until the ceiling, because
+providers send nothing until the response is generated. Response size is not capped: a single task pumps chunks
through a one-slot channel, with chunks at most 64 KiB. It stops reading when that
-channel fills. Completion, disconnect and deadline release admission and context;
-the deadline runs even when the downstream stops polling. A failure after headers
+channel fills. Completion, disconnect, stall and ceiling release admission and context;
+both timeouts run even when the downstream stops polling. A failure after headers
terminates the response body without adding an error event or retrying.
-There is no separate downstream stall timeout; a client that stops reading can
-retain execution capacity until the overall deadline. A progress-based downstream
-stall policy is deferred to a separate change.
Only request content type/encoding and accept cross the provider boundary, plus the
resolved credential in its provider's header (`Authorization: Bearer` for OpenAI,
diff --git a/ai-gateway/src/capture/anthropic_messages/tests.rs b/ai-gateway/src/capture/anthropic_messages/tests.rs
index b29e40b1d2..0e26b4637e 100644
--- a/ai-gateway/src/capture/anthropic_messages/tests.rs
+++ b/ai-gateway/src/capture/anthropic_messages/tests.rs
@@ -557,8 +557,12 @@ async fn streamed_messages_upload_one_generation_with_native_usage() {
provider.try_admit().unwrap(),
context,
&headers,
- Bytes::from_static(REQUEST),
- crate::providers::model_range(REQUEST).unwrap(),
+ crate::providers::RequestBody {
+ bytes: Bytes::from_static(REQUEST),
+ model_range: crate::providers::model_range(REQUEST).unwrap(),
+ streaming: true,
+ reservation: None,
+ },
Route::AnthropicMessages,
&mut crate::correlation::RequestCorrelation::default(),
)
diff --git a/ai-gateway/src/capture/openai_responses/tests.rs b/ai-gateway/src/capture/openai_responses/tests.rs
index dcc45fc962..55f49121ed 100644
--- a/ai-gateway/src/capture/openai_responses/tests.rs
+++ b/ai-gateway/src/capture/openai_responses/tests.rs
@@ -3,6 +3,7 @@ use crate::capture::{
ExecutionCapture, InputOmission, InputOmissionReason, MAX_INPUT_CAPTURE_BYTES, ProtocolCapture,
RelayOutcome, sse::SseDecoder,
};
+use crate::config::ExecutionLimits;
use crate::resolution::ApiFormat;
use crate::{
providers::{ProviderLimits, ProviderTransport},
@@ -513,8 +514,10 @@ async fn upstream_read_timeout_is_logged_as_timeout() {
format!("{}/v1", upstream.url),
ProviderLimits {
active: 1,
- read_timeout: Duration::from_millis(50),
- ..ProviderLimits::default()
+ execution: ExecutionLimits {
+ stall_timeout: Duration::from_millis(50),
+ ..ExecutionLimits::default()
+ },
},
);
let writer = LogWriter::default();
@@ -568,8 +571,10 @@ async fn native_http_relay_logs_once_on_eof_drop_and_unpolled_deadline() {
format!("{}/v1", upstream.url),
ProviderLimits {
active: 1,
- execution_timeout: Duration::from_secs(1),
- ..ProviderLimits::default()
+ execution: ExecutionLimits {
+ max_execution_time: Duration::from_secs(1),
+ ..ExecutionLimits::default()
+ },
},
);
let writer = LogWriter::default();
@@ -632,8 +637,10 @@ async fn provider_future_finalizes_on_timeout_and_cancellation_before_headers()
format!("{}/v1", upstream.url),
ProviderLimits {
active: 1,
- headers_timeout: Duration::from_millis(100),
- ..ProviderLimits::default()
+ execution: ExecutionLimits {
+ stall_timeout: Duration::from_millis(100),
+ ..ExecutionLimits::default()
+ },
},
);
let writer = LogWriter::default();
diff --git a/ai-gateway/src/config.rs b/ai-gateway/src/config.rs
index 3061b9768c..5f59f685bf 100644
--- a/ai-gateway/src/config.rs
+++ b/ai-gateway/src/config.rs
@@ -1,5 +1,5 @@
use crate::resolution::ControlPlaneConfig;
-use std::{env, error::Error, fmt, net::SocketAddr, time::Duration};
+use std::{env, error::Error, fmt, net::SocketAddr, ops::RangeInclusive, time::Duration};
use tracing::level_filters::LevelFilter;
pub struct GatewayConfig {
@@ -12,6 +12,50 @@ pub struct GatewayConfig {
pub max_active_requests: usize,
pub max_concurrent_resolutions: usize,
pub telemetry_buffer_bytes: usize,
+ pub request_limits: RequestLimits,
+ pub execution_limits: ExecutionLimits,
+}
+
+const MIB: usize = 1024 * 1024;
+const DEFAULT_CONCURRENCY: usize = 128;
+const SEMAPHORE_CAPACITY: usize = tokio::sync::Semaphore::MAX_PERMITS;
+pub const DEFAULT_MAX_REQUEST_BYTES: usize = 16 * MIB;
+const DEFAULT_SHARED_REQUEST_BUFFER_BYTES: usize = 1024 * MIB;
+const DEFAULT_MAX_EXECUTION_SECONDS: u32 = 10 * 60;
+const DEFAULT_STALL_TIMEOUT_SECONDS: u32 = 120;
+/// Buffered request body bounds per instance.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct RequestLimits {
+ pub max_request_bytes: usize,
+ /// Request body bytes all in-flight requests may buffer together.
+ pub shared_buffer_bytes: usize,
+}
+
+impl Default for RequestLimits {
+ fn default() -> Self {
+ Self {
+ max_request_bytes: DEFAULT_MAX_REQUEST_BYTES,
+ shared_buffer_bytes: DEFAULT_SHARED_REQUEST_BUFFER_BYTES,
+ }
+ }
+}
+
+/// Provider execution deadlines.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct ExecutionLimits {
+ /// Ceiling from execution admission until the response completes.
+ pub max_execution_time: Duration,
+ /// Longest wait without progress: stream response headers, upstream reads, downstream writes.
+ pub stall_timeout: Duration,
+}
+
+impl Default for ExecutionLimits {
+ fn default() -> Self {
+ Self {
+ max_execution_time: Duration::from_secs(DEFAULT_MAX_EXECUTION_SECONDS.into()),
+ stall_timeout: Duration::from_secs(DEFAULT_STALL_TIMEOUT_SECONDS.into()),
+ }
+ }
}
#[derive(Debug, PartialEq, Eq)]
@@ -51,6 +95,14 @@ impl GatewayConfig {
config.telemetry_buffer_bytes = telemetry_buffer_bytes(
read_env("LANGFUSE_AI_GATEWAY_TELEMETRY_BUFFER_BYTES")?.as_deref(),
)?;
+ config.request_limits = request_limits(
+ read_env("LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES")?.as_deref(),
+ read_env("LANGFUSE_AI_GATEWAY_SHARED_REQUEST_BUFFER_BYTES")?.as_deref(),
+ )?;
+ config.execution_limits = execution_limits(
+ read_env("LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS")?.as_deref(),
+ read_env("LANGFUSE_AI_GATEWAY_STALL_TIMEOUT_SECONDS")?.as_deref(),
+ )?;
if let Some(web_url) =
read_env("LANGFUSE_AI_GATEWAY_WEB_URL")?.filter(|url| !url.is_empty())
{
@@ -98,16 +150,12 @@ impl GatewayConfig {
.ok_or(GatewayConfigError(
"LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT must be true or false",
))?;
- let seconds: u64 = shutdown_timeout.unwrap_or("10").parse().map_err(|_| {
- GatewayConfigError(
- "LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",
- )
- })?;
- if !(1..=300).contains(&seconds) {
- return Err(GatewayConfigError(
- "LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",
- ));
- }
+ let seconds = integer_setting(
+ shutdown_timeout,
+ 10,
+ 1..=300,
+ "LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",
+ )?;
let log_level = match log_level.unwrap_or("info") {
"debug" => LevelFilter::DEBUG,
"info" => LevelFilter::INFO,
@@ -136,43 +184,95 @@ impl GatewayConfig {
shutdown_timeout: Duration::from_secs(seconds),
log_level,
log_format,
- max_active_requests: concurrency_limit(
+ max_active_requests: integer_setting(
max_active_requests,
+ DEFAULT_CONCURRENCY,
+ 1..=SEMAPHORE_CAPACITY,
"LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS must be a positive integer within the semaphore capacity",
)?,
- max_concurrent_resolutions: concurrency_limit(
+ max_concurrent_resolutions: integer_setting(
max_concurrent_resolutions,
+ DEFAULT_CONCURRENCY,
+ 1..=SEMAPHORE_CAPACITY,
"LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS must be a positive integer within the semaphore capacity",
)?,
telemetry_buffer_bytes: crate::telemetry::DEFAULT_RETAINED_BYTES,
+ request_limits: RequestLimits::default(),
+ execution_limits: ExecutionLimits::default(),
})
}
}
-fn concurrency_limit(
- value: Option<&str>,
- message: &'static str,
-) -> Result<usize, GatewayConfigError> {
- value
- .unwrap_or("128")
- .parse::<usize>()
- .ok()
- .filter(|limit| (1..=tokio::sync::Semaphore::MAX_PERMITS).contains(limit))
- .ok_or(GatewayConfigError(message))
+fn request_limits(
+ max_request_bytes: Option<&str>,
+ shared_buffer_bytes: Option<&str>,
+) -> Result<RequestLimits, GatewayConfigError> {
+ let defaults = RequestLimits::default();
+ // One request reserves its body in a single semaphore acquisition, which takes a u32.
+ let max_reservation = usize::try_from(u32::MAX)
+ .unwrap_or(usize::MAX)
+ .min(SEMAPHORE_CAPACITY);
+ let max_request_bytes = integer_setting(
+ max_request_bytes,
+ defaults.max_request_bytes,
+ MIB..=max_reservation,
+ "LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES must be an integer of at least 1048576 (1 MiB) and within a single semaphore reservation",
+ )?;
+ let shared_buffer_bytes = integer_setting(
+ shared_buffer_bytes,
+ defaults.shared_buffer_bytes,
+ max_request_bytes..=SEMAPHORE_CAPACITY,
+ "LANGFUSE_AI_GATEWAY_SHARED_REQUEST_BUFFER_BYTES must be an integer of at least LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES within the semaphore capacity",
+ )?;
+ Ok(RequestLimits {
+ max_request_bytes,
+ shared_buffer_bytes,
+ })
+}
+
+fn execution_limits(
+ max_execution_time: Option<&str>,
+ stall_timeout: Option<&str>,
+) -> Result<ExecutionLimits, GatewayConfigError> {
+ // `u32` seconds keep every deadline far from `Instant` overflow.
+ let max_execution_time: u32 = integer_setting(
+ max_execution_time,
+ DEFAULT_MAX_EXECUTION_SECONDS,
+ 60..=u32::MAX,
+ "LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS must be an integer of at least 60",
+ )?;
+ let stall_timeout: u32 = integer_setting(
+ stall_timeout,
+ DEFAULT_STALL_TIMEOUT_SECONDS,
+ 1..=max_execution_time,
+ "LANGFUSE_AI_GATEWAY_STALL_TIMEOUT_SECONDS must be an integer from 1 to LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS",
+ )?;
+ Ok(ExecutionLimits {
+ max_execution_time: Duration::from_secs(max_execution_time.into()),
+ stall_timeout: Duration::from_secs(stall_timeout.into()),
+ })
}
fn telemetry_buffer_bytes(value: Option<&str>) -> Result<usize, GatewayConfigError> {
- const MIB: usize = 1024 * 1024;
- let Some(value) = value else {
- return Ok(crate::telemetry::DEFAULT_RETAINED_BYTES);
- };
+ integer_setting(
+ value,
+ crate::telemetry::DEFAULT_RETAINED_BYTES,
+ 4 * MIB..=4096 * MIB,
+ "LANGFUSE_AI_GATEWAY_TELEMETRY_BUFFER_BYTES must be an integer from 4194304 (4 MiB) to 4294967296 (4 GiB)",
+ )
+}
+
+/// An unset value takes `default`. Errors carry only `message`, never the supplied value.
+fn integer_setting<T: std::str::FromStr + PartialOrd>(
+ value: Option<&str>,
+ default: T,
+ range: RangeInclusive<T>,
+ message: &'static str,
+) -> Result<T, GatewayConfigError> {
value
- .parse::<usize>()
- .ok()
- .filter(|bytes| (4 * MIB..=4096 * MIB).contains(bytes))
- .ok_or(GatewayConfigError(
- "LANGFUSE_AI_GATEWAY_TELEMETRY_BUFFER_BYTES must be an integer from 4194304 (4 MiB) to 4294967296 (4 GiB)",
- ))
+ .map_or(Some(default), |value| value.parse().ok())
+ .filter(|value| range.contains(value))
+ .ok_or(GatewayConfigError(message))
}
fn read_env(name: &'static str) -> Result<Option<String>, GatewayConfigError> {
diff --git a/ai-gateway/src/http.rs b/ai-gateway/src/http.rs
index 55b53610bb..648ed9de98 100644
--- a/ai-gateway/src/http.rs
+++ b/ai-gateway/src/http.rs
@@ -1,27 +1,27 @@
-use std::{error::Error, ops::Range, sync::Arc, time::Duration};
+use std::{ops::Range, sync::Arc};
use axum::{
Json, Router,
- body::{Body, Bytes, to_bytes},
+ body::{Body, Bytes},
extract::{Request, State},
http::{HeaderMap, StatusCode, header},
response::{IntoResponse, Response},
routing::{get, post},
};
+use http_body_util::BodyExt;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
+use tower_http::timeout::{RequestBodyDeadlineLayer, TimeoutError};
use tracing::Instrument;
use crate::{
correlation::{RequestCorrelation, assign_request_id},
- inference::{InferenceService, RequestPreparationError},
- providers::{ProviderError, Route},
+ inference::{InferenceService, RequestBodyPermit, RequestPreparationError},
+ providers::{ProviderError, RequestBody, Route},
resolution::{ApiFormat, ResolutionError},
server::GatewayLifecycleState,
};
-const MAX_REQUEST_BYTES: usize = 10 * 1024 * 1024;
-const REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(30);
const MAX_GATEWAY_KEY_BYTES: usize = 8192;
#[derive(Clone)]
@@ -32,7 +32,10 @@ struct InferenceRouteState {
/// Build inference routes. An unconfigured or draining service returns 503.
pub fn router(inference: Option<InferenceService>, lifecycle: GatewayLifecycleState) -> Router {
- Router::new()
+ let body_deadline = inference
+ .as_ref()
+ .map(InferenceService::request_read_timeout);
+ let router = Router::new()
.route("/openai/v1/responses", post(handle_responses))
.route(
"/openai/v1/responses/compact",
@@ -49,8 +52,12 @@ pub fn router(inference: Option<InferenceService>, lifecycle: GatewayLifecycleSt
.with_state(InferenceRouteState {
inference: inference.map(Arc::new),
lifecycle,
- })
- .layer(axum::middleware::from_fn(assign_request_id))
+ });
+ let router = match body_deadline {
+ Some(deadline) => router.layer(RequestBodyDeadlineLayer::new(deadline)),
+ None => router,
+ };
+ router.layer(axum::middleware::from_fn(assign_request_id))
}
async fn handle_responses(State(state): State<InferenceRouteState>, request: Request) -> Response {
@@ -185,11 +192,19 @@ async fn relay(
if !crate::transport::identity_encoding(request.headers()) {
return Err(InferenceHttpError::UnsupportedEncoding);
}
- let _body_permit = inference
+ // Held with the execution and released once the response ends.
+ let body_permit = inference
.try_reserve_request_body()
.map_err(InferenceHttpError::from)?;
let (parts, body) = request.into_parts();
- let request_body = request_body_and_model(body).await?;
+ let (bytes, body_permit) = read_request_body(
+ body,
+ &parts.headers,
+ inference.max_request_bytes(),
+ body_permit,
+ )
+ .await?;
+ let request_body = request_body_and_model(bytes)?;
let (permit, context) = inference
.resolve_and_admit(
gateway_key,
@@ -204,8 +219,12 @@ async fn relay(
permit,
context,
&parts.headers,
- request_body.bytes,
- request_body.model_range,
+ RequestBody {
+ bytes: request_body.bytes,
+ model_range: request_body.model_range,
+ streaming: request_body.streaming,
+ reservation: Some(Box::new(body_permit)),
+ },
route,
correlation,
)
@@ -217,16 +236,19 @@ struct InferenceRequestBody {
bytes: Bytes,
model: String,
model_range: Range<usize>,
+ streaming: bool,
}
#[derive(Deserialize)]
struct InferenceRequest<'a> {
#[serde(borrow)]
model: &'a RawValue,
+ /// Kept raw so a non-boolean value is left for the provider to reject.
+ #[serde(borrow, default)]
+ stream: Option<&'a RawValue>,
}
-async fn request_body_and_model(body: Body) -> Result<InferenceRequestBody, InferenceHttpError> {
- let bytes = read_request_body(body).await?;
+fn request_body_and_model(bytes: Bytes) -> Result<InferenceRequestBody, InferenceHttpError> {
let request = serde_json::from_slice::<InferenceRequest<'_>>(&bytes)
.map_err(|_| InferenceHttpError::InvalidBody)?;
let raw_model = request.model.get();
@@ -244,43 +266,81 @@ async fn request_body_and_model(body: Body) -> Result<InferenceRequestBody, Infe
.checked_add(raw_model.len())
.filter(|end| *end <= bytes.len())
.ok_or(InferenceHttpError::InvalidBody)?;
+ let streaming = request.stream.is_some_and(|stream| stream.get() == "true");
Ok(InferenceRequestBody {
bytes,
model,
model_range: start..end,
+ streaming,
})
}
-/// Buffer the client body. The wait is mostly the caller's upload, so it gets its
-/// own span and the measured size lands on the server span for aggregation.
-async fn read_request_body(body: Body) -> Result<Bytes, InferenceHttpError> {
+/// Buffer the client body, reserving its allocation from the shared budget as bytes
+/// arrive. A declared length is checked before reading and caps buffer growth, but is
+/// not reserved up front, so a client that declares a large body and stalls holds only
+/// what it actually sent. The router's body deadline bounds the read. The wait is mostly
+/// the caller's upload, so it gets its own span and the measured size lands on the server
+/// span for aggregation.
+async fn read_request_body(
+ body: Body,
+ headers: &HeaderMap,
+ max_bytes: usize,
+ mut permit: RequestBodyPermit,
+) -> Result<(Bytes, RequestBodyPermit), InferenceHttpError> {
let server = tracing::Span::current();
let span = tracing::info_span!(
"request.body",
otel.kind = "internal",
http.request.body.size = tracing::field::Empty
);
+ let declared = headers
+ .get(header::CONTENT_LENGTH)
+ .and_then(|value| value.to_str().ok()?.parse::<u64>().ok())
+ .map(|length| usize::try_from(length).unwrap_or(usize::MAX));
let bytes = async {
- let bytes = tokio::time::timeout(REQUEST_READ_TIMEOUT, to_bytes(body, MAX_REQUEST_BYTES))
- .await
- .map_err(|_| InferenceHttpError::RequestTimeout)?
- .map_err(|error| {
- if error
- .source()
- .is_some_and(<dyn Error + 'static>::is::<http_body_util::LengthLimitError>)
- {
- InferenceHttpError::TooLarge
- } else {
- InferenceHttpError::InvalidBody
- }
- })?;
+ if declared.is_some_and(|declared| declared > max_bytes) {
+ return Err(InferenceHttpError::TooLarge);
+ }
+ let capacity_limit = declared.unwrap_or(max_bytes);
+ let mut body = body;
+ let mut buffer = Vec::new();
+ while let Some(frame) = body.frame().await {
+ let Ok(data) = frame.map_err(body_error)?.into_data() else {
+ continue;
+ };
+ let total = buffer.len() + data.len();
+ if total > max_bytes {
+ return Err(InferenceHttpError::TooLarge);
+ }
+ if total > buffer.capacity() {
+ let capacity = buffer
+ .capacity()
+ .saturating_mul(2)
+ .clamp(total, capacity_limit.max(total));
+ permit
+ .try_reserve_bytes(capacity)
+ .map_err(InferenceHttpError::Resolution)?;
+ buffer.reserve_exact(capacity - buffer.len());
+ }
+ buffer.extend_from_slice(&data);
+ }
+ let bytes = Bytes::from(buffer);
tracing::Span::current().record("http.request.body.size", byte_count(bytes.len()));
Ok::<_, InferenceHttpError>(bytes)
}
.instrument(span)
.await?;
server.record("http.request.body.size", byte_count(bytes.len()));
- Ok(bytes)
+ Ok((bytes, permit))
+}
+
+/// The router's body deadline surfaces as a body error.
+fn body_error(error: axum::Error) -> InferenceHttpError {
+ if error.into_inner().is::<TimeoutError>() {
+ InferenceHttpError::RequestTimeout
+ } else {
+ InferenceHttpError::InvalidBody
+ }
}
fn byte_count(len: usize) -> i64 {
diff --git a/ai-gateway/src/http/tests.rs b/ai-gateway/src/http/tests.rs
index 1c25ce91e5..7c993826a8 100644
--- a/ai-gateway/src/http/tests.rs
+++ b/ai-gateway/src/http/tests.rs
@@ -1,14 +1,17 @@
use super::*;
use crate::{
+ config::{DEFAULT_MAX_REQUEST_BYTES as MAX_REQUEST_BYTES, ExecutionLimits, RequestLimits},
providers::{ProviderLimits, ProviderTransport},
resolution::ApiFormat,
test_support::{FakeServer, resolution_response, resolution_response_for},
};
+use axum::body::to_bytes;
use futures_util::{StreamExt, stream};
use std::{
convert::Infallible,
io::Write,
sync::atomic::{AtomicBool, Ordering},
+ time::Duration,
};
use tokio::{net::TcpListener, sync::oneshot, task::JoinHandle};
use tower::ServiceExt;
@@ -1002,12 +1005,29 @@ async fn slow_resolution_has_a_bounded_wait() {
#[tokio::test(start_paused = true)]
async fn a_stalled_request_body_times_out_at_the_read_deadline() {
+ let web = FakeServer::start(|_| async { resolution_response("provider") }).await;
+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
+ let service = inference(&web, &provider);
+ // The 16 MiB default keeps the 10 MiB per 30 seconds minimum upload rate.
+ assert_eq!(service.request_read_timeout(), Duration::from_secs(49));
+ let gateway = Gateway::start(Some(service)).await;
let stalled = Body::from_stream(stream::pending::<Result<&'static str, Infallible>>());
let started = tokio::time::Instant::now();
- let result = read_request_body(stalled).await;
- assert!(matches!(result, Err(InferenceHttpError::RequestTimeout)));
- assert_eq!(started.elapsed(), REQUEST_READ_TIMEOUT);
- assert_eq!(REQUEST_READ_TIMEOUT, Duration::from_secs(30));
+ let request = gateway.app.clone().oneshot(
+ Request::post("/openai/v1/responses")
+ .header(header::AUTHORIZATION, "Bearer stalled-upload")
+ .body(stalled)
+ .unwrap(),
+ );
+ let response = tokio::time::timeout(Duration::from_secs(120), request)
+ .await
+ .expect("the router must bound the body read")
+ .unwrap();
+ assert_eq!(response.status(), StatusCode::REQUEST_TIMEOUT);
+ assert_eq!(started.elapsed(), Duration::from_secs(49));
+ let body = to_bytes(response.into_body(), 4096).await.unwrap();
+ assert!(String::from_utf8_lossy(&body).contains("request_timeout"));
+ assert_eq!(web.calls(), 0);
}
#[tokio::test]
@@ -1042,7 +1062,10 @@ async fn pre_header_provider_failures_export_the_status_returned_to_the_caller()
let provider = ProviderTransport::for_test(
upstream_url,
ProviderLimits {
- headers_timeout: Duration::from_millis(100),
+ execution: ExecutionLimits {
+ stall_timeout: Duration::from_millis(100),
+ ..ExecutionLimits::default()
+ },
..ProviderLimits::default()
},
)
@@ -1056,7 +1079,7 @@ async fn pre_header_provider_failures_export_the_status_returned_to_the_caller()
let response = gateway
.post()
.bearer_auth("gateway-key")
- .body(r#"{"model":"requested","input":"hello"}"#)
+ .body(r#"{"model":"requested","input":"hello","stream":true}"#)
.send()
.await
.unwrap();
@@ -1302,7 +1325,7 @@ async fn active_admission_is_held_through_stream_and_shutdown_drains_it() {
.status(),
StatusCode::SERVICE_UNAVAILABLE
);
- assert_eq!(web.calls(), 2);
+ assert_eq!(web.calls(), 1);
assert_eq!(provider.calls(), 1);
gateway.shutdown.take().unwrap().send(()).unwrap();
assert!(!gateway.task.is_finished());
@@ -1747,3 +1770,204 @@ async fn anthropic_routes_share_the_request_body_limit() {
}
assert_eq!(provider.calls(), 2);
}
+
+#[tokio::test]
+async fn buffered_request_bytes_are_released_once_the_response_ends() {
+ let (release, released) = tokio::sync::watch::channel(false);
+ let web = FakeServer::start(|_| async { resolution_response("provider") }).await;
+ let provider = FakeServer::start(move |request: Request| {
+ let mut released = released.clone();
+ async move {
+ to_bytes(request.into_body(), usize::MAX).await.unwrap();
+ released.wait_for(|done| *done).await.unwrap();
+ Response::new(Body::empty())
+ }
+ })
+ .await;
+ let service = inference(&web, &provider).with_request_limits(RequestLimits {
+ max_request_bytes: 64,
+ shared_buffer_bytes: 100,
+ });
+ let gateway = Gateway::start(Some(service)).await;
+ let body = |size: usize| {
+ let prefix = r#"{"model":"Production key:test","padding":""#;
+ format!("{prefix}{}\"}}", "x".repeat(size - prefix.len() - 2))
+ };
+ let oversized = gateway.post().bearer_auth("key").body(body(65)).send();
+ assert_eq!(
+ oversized.await.unwrap().status(),
+ StatusCode::PAYLOAD_TOO_LARGE
+ );
+ // Two 60-byte calls do not fit the 100-byte budget at once.
+ let first = tokio::spawn(gateway.post().bearer_auth("key").body(body(60)).send());
+ while provider.calls() == 0 {
+ tokio::task::yield_now().await;
+ }
+ let second = gateway.post().bearer_auth("key").body(body(60)).send();
+ let second = tokio::time::timeout(Duration::from_secs(5), second)
+ .await
+ .expect("the first call still holds its bytes while it waits for headers");
+ assert_eq!(second.unwrap().status(), StatusCode::SERVICE_UNAVAILABLE);
+ release.send(true).unwrap();
+ let first = first.await.unwrap().unwrap();
+ assert_eq!(first.status(), StatusCode::OK);
+ first.bytes().await.unwrap();
+ tokio::time::timeout(Duration::from_secs(5), async {
+ loop {
+ let next = gateway.post().bearer_auth("key").body(body(60)).send();
+ if next.await.unwrap().status() == StatusCode::OK {
+ break;
+ }
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .expect("the first call's bytes are released once its response ends");
+}
+
+async fn budgeted_service() -> (FakeServer, FakeServer, InferenceService) {
+ let web = FakeServer::start(|_| async { resolution_response("provider") }).await;
+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
+ let service = inference(&web, &provider).with_request_limits(RequestLimits {
+ max_request_bytes: 64,
+ shared_buffer_bytes: 100,
+ });
+ (web, provider, service)
+}
+
+fn streamed_body(chunks: Vec<&'static [u8]>, stall: bool) -> Body {
+ let chunks = stream::iter(chunks.into_iter().map(Ok::<_, Infallible>));
+ if stall {
+ Body::from_stream(chunks.chain(stream::pending()))
+ } else {
+ Body::from_stream(chunks)
+ }
+}
+
+#[tokio::test]
+async fn a_stalled_upload_holds_only_the_bytes_it_sent() {
+ let (_web, _provider, service) = budgeted_service().await;
+ let mut headers = HeaderMap::new();
+ headers.insert(header::CONTENT_LENGTH, "60".parse().unwrap());
+ let stalled = service.try_reserve_request_body().ok().unwrap();
+ let read = read_request_body(
+ streamed_body(vec![&[b'x'; 10]], true),
+ &headers,
+ 64,
+ stalled,
+ );
+ tokio::pin!(read);
+ assert!(
+ tokio::time::timeout(Duration::from_millis(50), &mut read)
+ .await
+ .is_err()
+ );
+ let mut next = service.try_reserve_request_body().ok().unwrap();
+ assert!(next.try_reserve_bytes(91).is_err());
+ assert!(next.try_reserve_bytes(90).is_ok());
+}
+
+#[tokio::test]
+async fn buffered_bytes_reserve_the_allocation_not_just_the_length() {
+ let (_web, _provider, service) = budgeted_service().await;
+ let held = service.try_reserve_request_body().ok().unwrap();
+ let (bytes, _held) = read_request_body(
+ streamed_body(vec![&[b'x'; 32], b"x"], false),
+ &HeaderMap::new(),
+ 64,
+ held,
+ )
+ .await
+ .ok()
+ .unwrap();
+ assert_eq!(bytes.len(), 33);
+ let mut next = service.try_reserve_request_body().ok().unwrap();
+ assert!(next.try_reserve_bytes(37).is_err());
+ assert!(next.try_reserve_bytes(36).is_ok());
+}
+
+#[test]
+fn only_a_literal_true_stream_member_marks_a_streamed_call() {
+ for (body, streaming) in [
+ (r#"{"model":"m","stream":true}"#, true),
+ (r#"{"model":"m","stream":false}"#, false),
+ (r#"{"model":"m","stream":"true"}"#, false),
+ (r#"{"model":"m","stream":null}"#, false),
+ (r#"{"model":"m"}"#, false),
+ ] {
+ let request = request_body_and_model(Bytes::from_static(body.as_bytes()))
+ .ok()
+ .unwrap();
+ assert_eq!(request.streaming, streaming, "{body}");
+ assert_eq!(&body.as_bytes()[request.model_range], br#""m""#);
+ }
+ for body in [
+ r#"{"model":"m","model":"n"}"#,
+ r#"{"model":"m","stream":true,"stream":true}"#,
+ r#"{"stream":true}"#,
+ ] {
+ assert!(
+ request_body_and_model(Bytes::from_static(body.as_bytes())).is_err(),
+ "{body}"
+ );
+ }
+}
+
+#[tokio::test]
+async fn only_non_streamed_generations_wait_past_the_stall_for_provider_headers() {
+ let web = FakeServer::start(|_| async {
+ resolution_response_for(ApiFormat::AnthropicMessages, "provider-anthropic")
+ })
+ .await;
+ let provider = FakeServer::start(|_| async {
+ tokio::time::sleep(Duration::from_millis(300)).await;
+ Response::new(Body::from("{}"))
+ })
+ .await;
+ let service = InferenceService::for_test(
+ web.control_plane(),
+ ProviderTransport::for_test(
+ format!("{}/v1", provider.url),
+ ProviderLimits {
+ execution: ExecutionLimits {
+ stall_timeout: Duration::from_millis(100),
+ ..ExecutionLimits::default()
+ },
+ ..ProviderLimits::default()
+ },
+ ),
+ 128,
+ );
+ let gateway = Gateway::start(Some(service)).await;
+ let client = reqwest::Client::new();
+ for (path, body, status) in [
+ (
+ "messages",
+ r#"{"model":"Production key:test"}"#,
+ StatusCode::OK,
+ ),
+ (
+ "messages",
+ r#"{"model":"Production key:test","stream":false}"#,
+ StatusCode::OK,
+ ),
+ (
+ "messages",
+ r#"{"model":"Production key:test","stream":true}"#,
+ StatusCode::GATEWAY_TIMEOUT,
+ ),
+ (
+ "messages/count_tokens",
+ r#"{"model":"Production key:test"}"#,
+ StatusCode::GATEWAY_TIMEOUT,
+ ),
+ ] {
+ let response = client
+ .post(format!("{}/anthropic/v1/{path}", gateway.url))
+ .header("x-api-key", "gateway-anthropic")
+ .body(body)
+ .send()
+ .await;
+ assert_eq!(response.unwrap().status(), status, "{path} {body}");
+ }
+}
diff --git a/ai-gateway/src/inference.rs b/ai-gateway/src/inference.rs
index 23282f2dda..eb951a689c 100644
--- a/ai-gateway/src/inference.rs
+++ b/ai-gateway/src/inference.rs
@@ -1,16 +1,17 @@
//! Resolve the caller's credential, then execute using only the trusted context.
-use std::ops::Range;
+use std::{sync::Arc, time::Duration};
use axum::{
- body::{Body, Bytes},
+ body::Body,
http::{HeaderMap, Response},
};
-use tokio::sync::{Semaphore, SemaphorePermit};
+use tokio::sync::{OwnedSemaphorePermit, Semaphore, SemaphorePermit};
use tracing::Instrument;
use crate::{
+ config::{GatewayConfig, RequestLimits},
correlation::RequestCorrelation,
- providers::{ProviderError, ProviderTransport, RequestPermit, Route},
+ providers::{ProviderError, ProviderTransport, RequestBody, RequestPermit, Route},
resolution::{
ApiFormat, ControlPlaneClient, ControlPlaneConfig, ResolutionError, ResolvedRequestContext,
},
@@ -19,7 +20,9 @@ use crate::{
pub struct InferenceService {
control_plane: ControlPlaneClient,
provider: ProviderTransport,
- request_body_capacity: Semaphore,
+ request_body_capacity: Arc<Semaphore>,
+ request_limits: RequestLimits,
+ request_bytes_capacity: Arc<Semaphore>,
model_catalog_capacity: Semaphore,
resolution_capacity: Semaphore,
telemetry: Option<crate::telemetry::Telemetry>,
@@ -30,31 +33,71 @@ pub(crate) enum RequestPreparationError {
Provider(ProviderError),
}
-pub(crate) struct RequestBodyPermit<'a> {
- _permit: SemaphorePermit<'a>,
+/// One request-body slot plus the body bytes reserved so far from the shared budget.
+/// It is held with the execution and released once the response ends.
+pub(crate) struct RequestBodyPermit {
+ _slot: OwnedSemaphorePermit,
+ bytes: OwnedSemaphorePermit,
+ budget: Arc<Semaphore>,
}
+impl RequestBodyPermit {
+ /// Grow the reservation to `total` body bytes without waiting.
+ ///
+ /// # Errors
+ /// Returns [`ResolutionError::Unavailable`] when the shared budget cannot cover `total`.
+ pub(crate) fn try_reserve_bytes(&mut self, total: usize) -> Result<(), ResolutionError> {
+ let Some(missing) = total
+ .checked_sub(self.bytes.num_permits())
+ .filter(|missing| *missing > 0)
+ else {
+ return Ok(());
+ };
+ let reserved = u32::try_from(missing)
+ .ok()
+ .and_then(|missing| {
+ Arc::clone(&self.budget)
+ .try_acquire_many_owned(missing)
+ .ok()
+ })
+ .ok_or_else(|| {
+ crate::observability::rejected("request_bytes");
+ ResolutionError::Unavailable
+ })?;
+ self.bytes.merge(reserved);
+ Ok(())
+ }
+}
+
+/// Uploads slower than 10 MiB per 30 seconds on average miss the read deadline.
+const MIN_REQUEST_UPLOAD_BYTES_PER_SECOND: usize = 10 * 1024 * 1024 / 30;
+const MIN_REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(30);
+
impl InferenceService {
/// Initialize pooled clients for resolution and provider requests.
///
/// # Errors
/// Returns a sanitized configuration error for invalid capacities or client setup.
pub fn new(
- config: ControlPlaneConfig,
- max_active_requests: usize,
- max_concurrent_resolutions: usize,
- telemetry_buffer_bytes: usize,
+ control_plane: ControlPlaneConfig,
+ config: &GatewayConfig,
) -> Result<Self, ResolutionError> {
+ let max_concurrent_resolutions = config.max_concurrent_resolutions;
if !(1..=Semaphore::MAX_PERMITS).contains(&max_concurrent_resolutions) {
return Err(ResolutionError::Configuration);
}
- let telemetry = crate::telemetry::Telemetry::new(&config, telemetry_buffer_bytes)?;
+ let telemetry =
+ crate::telemetry::Telemetry::new(&control_plane, config.telemetry_buffer_bytes)?;
Ok(Self {
- control_plane: ControlPlaneClient::new(config)?,
- provider: ProviderTransport::new(max_active_requests)
+ control_plane: ControlPlaneClient::new(control_plane)?,
+ provider: ProviderTransport::new(config.max_active_requests, config.execution_limits)
.map_err(|_| ResolutionError::Configuration)?
.with_telemetry(telemetry.clone()),
- request_body_capacity: Semaphore::new(max_active_requests),
+ request_body_capacity: Arc::new(Semaphore::new(config.max_active_requests)),
+ request_limits: config.request_limits,
+ request_bytes_capacity: Arc::new(Semaphore::new(
+ config.request_limits.shared_buffer_bytes,
+ )),
model_catalog_capacity: Semaphore::new(max_concurrent_resolutions),
resolution_capacity: Semaphore::new(max_concurrent_resolutions),
telemetry: Some(telemetry),
@@ -74,12 +117,36 @@ impl InferenceService {
pub(crate) fn try_reserve_request_body(
&self,
- ) -> Result<RequestBodyPermit<'_>, RequestPreparationError> {
- let permit = self.request_body_capacity.try_acquire().map_err(|_| {
- crate::observability::rejected("request_body");
- RequestPreparationError::Resolution(ResolutionError::Unavailable)
- })?;
- Ok(RequestBodyPermit { _permit: permit })
+ ) -> Result<RequestBodyPermit, RequestPreparationError> {
+ let slot = Arc::clone(&self.request_body_capacity)
+ .try_acquire_owned()
+ .map_err(|_| {
+ crate::observability::rejected("request_body");
+ RequestPreparationError::Resolution(ResolutionError::Unavailable)
+ })?;
+ let bytes = Arc::clone(&self.request_bytes_capacity)
+ .try_acquire_many_owned(0)
+ .map_err(|_| RequestPreparationError::Resolution(ResolutionError::Unavailable))?;
+ Ok(RequestBodyPermit {
+ _slot: slot,
+ bytes,
+ budget: Arc::clone(&self.request_bytes_capacity),
+ })
+ }
+
+ pub(crate) fn max_request_bytes(&self) -> usize {
+ self.request_limits.max_request_bytes
+ }
+
+ /// The body read deadline scales with the size limit so a maximal body has the same
+ /// minimum upload rate as a small one.
+ pub(crate) fn request_read_timeout(&self) -> Duration {
+ let seconds = self
+ .request_limits
+ .max_request_bytes
+ .div_ceil(MIN_REQUEST_UPLOAD_BYTES_PER_SECOND);
+ Duration::from_secs(u64::try_from(seconds).unwrap_or(u64::MAX))
+ .max(MIN_REQUEST_READ_TIMEOUT)
}
pub(crate) async fn fetch_model_catalog(
@@ -159,30 +226,17 @@ impl InferenceService {
Ok((permit, context))
}
- #[expect(
- clippy::too_many_arguments,
- reason = "each argument is a distinct per-request input passed straight through"
- )]
pub(crate) async fn forward(
&self,
permit: RequestPermit,
context: ResolvedRequestContext,
headers: &HeaderMap,
- body: Bytes,
- model_range: Range<usize>,
+ request: RequestBody,
route: Route,
correlation: &mut RequestCorrelation,
) -> Result<Response<Body>, ProviderError> {
self.provider
- .forward_route(
- permit,
- context,
- headers,
- body,
- model_range,
- route,
- correlation,
- )
+ .forward_route(permit, context, headers, request, route, correlation)
.await
}
@@ -195,10 +249,21 @@ impl InferenceService {
Self {
control_plane,
provider,
- request_body_capacity: Semaphore::new(resolutions),
+ request_body_capacity: Arc::new(Semaphore::new(resolutions)),
+ request_limits: RequestLimits::default(),
+ request_bytes_capacity: Arc::new(Semaphore::new(
+ RequestLimits::default().shared_buffer_bytes,
+ )),
model_catalog_capacity: Semaphore::new(resolutions),
resolution_capacity: Semaphore::new(resolutions),
telemetry: None,
}
}
+
+ #[cfg(test)]
+ pub(crate) fn with_request_limits(mut self, request_limits: RequestLimits) -> Self {
+ self.request_bytes_capacity = Arc::new(Semaphore::new(request_limits.shared_buffer_bytes));
+ self.request_limits = request_limits;
+ self
+ }
}
diff --git a/ai-gateway/src/main.rs b/ai-gateway/src/main.rs
index bc52e0e640..b32b21f87d 100644
--- a/ai-gateway/src/main.rs
+++ b/ai-gateway/src/main.rs
@@ -25,18 +25,12 @@ async fn main() -> ExitCode {
}
async fn run() -> Result<(), Box<dyn Error>> {
- let config = GatewayConfig::from_env()?;
+ let mut config = GatewayConfig::from_env()?;
let observability = observability::init(&config)?;
let inference = config
.control_plane
- .map(|control_plane| {
- InferenceService::new(
- control_plane,
- config.max_active_requests,
- config.max_concurrent_resolutions,
- config.telemetry_buffer_bytes,
- )
- })
+ .take()
+ .map(|control_plane| InferenceService::new(control_plane, &config))
.transpose()?;
let inference_enabled = inference.is_some();
let telemetry = inference.as_ref().and_then(InferenceService::telemetry);
@@ -51,7 +45,7 @@ async fn run() -> Result<(), Box<dyn Error>> {
)));
let shutdown = shutdown_signal()?;
let listener = bind_listener(config.listen_address, config.auto_increment_listen_port).await?;
- tracing::info!(address = %listener.local_addr()?, inference_enabled, max_active_requests = config.max_active_requests, max_concurrent_resolutions = config.max_concurrent_resolutions, shutdown_timeout_seconds = config.shutdown_timeout.as_secs(), "gateway listening");
+ tracing::info!(address = %listener.local_addr()?, inference_enabled, max_active_requests = config.max_active_requests, max_concurrent_resolutions = config.max_concurrent_resolutions, max_request_bytes = config.request_limits.max_request_bytes, shared_request_buffer_bytes = config.request_limits.shared_buffer_bytes, max_execution_seconds = config.execution_limits.max_execution_time.as_secs(), stall_timeout_seconds = config.execution_limits.stall_timeout.as_secs(), shutdown_timeout_seconds = config.shutdown_timeout.as_secs(), "gateway listening");
let result = server::serve(
listener,
app,
diff --git a/ai-gateway/src/providers/mod.rs b/ai-gateway/src/providers/mod.rs
index cda1a77e27..4b1aa145c8 100644
--- a/ai-gateway/src/providers/mod.rs
+++ b/ai-gateway/src/providers/mod.rs
@@ -18,6 +18,7 @@ use tokio::{
pub use crate::transport::ProviderError;
use crate::{
capture::{ExecutionCapture, RelayOutcome},
+ config::ExecutionLimits,
correlation::{RequestCorrelation, UpstreamTarget},
resolution::{ApiFormat, Provider, ProviderCredential, ResolvedRequestContext},
transport,
@@ -75,18 +76,14 @@ impl Route {
pub(crate) struct ProviderLimits {
pub active: usize,
- pub execution_timeout: Duration,
- pub headers_timeout: Duration,
- pub read_timeout: Duration,
+ pub execution: ExecutionLimits,
}
impl Default for ProviderLimits {
fn default() -> Self {
Self {
active: 128,
- execution_timeout: Duration::from_secs(600),
- headers_timeout: Duration::from_secs(120),
- read_timeout: Duration::from_secs(120),
+ execution: ExecutionLimits::default(),
}
}
}
@@ -97,6 +94,18 @@ pub struct RequestPermit {
deadline: Instant,
}
+/// The validated caller body and the facts the relay needs about it.
+pub(crate) struct RequestBody {
+ pub bytes: Bytes,
+ pub model_range: Range<usize>,
+ /// The caller asked for a streamed response (`"stream": true`).
+ pub streaming: bool,
+ /// Capacity held for the body, released with the execution once the response ends.
+ pub reservation: Option<BodyReservation>,
+}
+
+pub(crate) type BodyReservation = Box<dyn Send + Sync>;
+
pub struct ProviderTransport {
client: ClientWithMiddleware,
capacity: Arc<Semaphore>,
@@ -108,10 +117,13 @@ pub struct ProviderTransport {
impl ProviderTransport {
/// # Errors
/// Returns [`ProviderError::Configuration`] when the HTTPS client cannot be initialized.
- pub fn new(max_active_requests: usize) -> Result<Self, ProviderError> {
+ pub fn new(
+ max_active_requests: usize,
+ execution: ExecutionLimits,
+ ) -> Result<Self, ProviderError> {
Self::with_limits(ProviderLimits {
active: max_active_requests,
- ..ProviderLimits::default()
+ execution,
})
}
@@ -127,8 +139,9 @@ impl ProviderTransport {
.no_brotli()
.no_deflate()
.no_zstd()
+ // No client read timeout: it would also cut a non-streaming call while the
+ // provider generates before sending headers. The relay bounds stalls instead.
.connect_timeout(Duration::from_secs(5))
- .read_timeout(limits.read_timeout)
.pool_max_idle_per_host(16)
.build()
.map_err(|_| ProviderError::Configuration)?;
@@ -156,10 +169,12 @@ impl ProviderTransport {
Ok(RequestPermit {
_permit: permit,
_active: crate::observability::Active::new("execution"),
- deadline: Instant::now() + self.limits.execution_timeout,
+ deadline: Instant::now() + self.limits.execution.max_execution_time,
})
}
+ /// Forward as a streamed `OpenAI` Responses call.
+ ///
/// # Errors
/// See [`Self::forward_route`].
#[cfg(test)]
@@ -175,28 +190,33 @@ impl ProviderTransport {
permit,
context,
headers,
- body,
- model_range,
+ RequestBody {
+ bytes: body,
+ model_range,
+ streaming: true,
+ reservation: None,
+ },
Route::OpenAiResponses,
&mut RequestCorrelation::from_headers(headers),
)
.await
}
- #[expect(
- clippy::too_many_arguments,
- reason = "each argument is a distinct per-request input of the relay"
- )]
pub(crate) async fn forward_route(
&self,
permit: RequestPermit,
context: ResolvedRequestContext,
headers: &HeaderMap,
- body: Bytes,
- model_range: Range<usize>,
+ request: RequestBody,
route: Route,
correlation: &mut RequestCorrelation,
) -> Result<Response<Body>, ProviderError> {
+ let RequestBody {
+ bytes: body,
+ model_range,
+ streaming,
+ reservation,
+ } = request;
let api_format = route.api_format();
let (credential_name, mut credential) = provider_credential(&context)?;
credential.set_sensitive(true);
@@ -236,23 +256,28 @@ impl ProviderTransport {
.header(header::ACCEPT_ENCODING, "identity")
.header(credential_name, credential);
upstream = upstream.body(body);
- let response = tokio::time::timeout_at(
+ // A stream, or a call that does not generate, that goes quiet before its
+ // headers is stalled. A non-streamed generation sends nothing until the whole
+ // response is generated.
+ let headers_deadline = if streaming || !route.captures_generation() {
permit
.deadline
- .min(Instant::now() + self.limits.headers_timeout),
- upstream.send(),
- )
- .await
- .map_err(|_| ProviderError::Timeout)
- .and_then(|result| {
- result.map_err(|error| {
- if error.is_timeout() {
- ProviderError::Timeout
- } else {
- ProviderError::Transport
- }
- })
- });
+ .min(Instant::now() + self.limits.execution.stall_timeout)
+ } else {
+ permit.deadline
+ };
+ let response = tokio::time::timeout_at(headers_deadline, upstream.send())
+ .await
+ .map_err(|_| ProviderError::Timeout)
+ .and_then(|result| {
+ result.map_err(|error| {
+ if error.is_timeout() {
+ ProviderError::Timeout
+ } else {
+ ProviderError::Transport
+ }
+ })
+ });
let response = match response {
Ok(response) => response,
Err(error) => {
@@ -271,8 +296,13 @@ impl ProviderTransport {
let mut downstream = Response::new(Body::empty());
*downstream.status_mut() = response.status();
*downstream.headers_mut() = transport::response_headers(response.headers(), api_format);
- *downstream.body_mut() =
- transport::relay(response, permit.deadline, (permit, context), capture);
+ *downstream.body_mut() = transport::relay(
+ response,
+ permit.deadline,
+ self.limits.execution.stall_timeout,
+ (permit, context, reservation),
+ capture,
+ );
Ok(downstream)
}
diff --git a/ai-gateway/src/providers/tests.rs b/ai-gateway/src/providers/tests.rs
index cce2497bd7..8997824fef 100644
--- a/ai-gateway/src/providers/tests.rs
+++ b/ai-gateway/src/providers/tests.rs
@@ -1,4 +1,5 @@
use super::*;
+use crate::config::ExecutionLimits;
use crate::test_support::{
FakeServer, ingestion_token, resolved_request_context, resolved_request_context_for,
};
@@ -69,8 +70,12 @@ async fn invalid_model_rewrite_never_reaches_the_provider() {
relay.try_admit().unwrap(),
resolved_request_context("provider-secret").await,
&HeaderMap::new(),
- Bytes::from_static(br#"{"model":"requested"}"#),
- 100..110,
+ RequestBody {
+ bytes: Bytes::from_static(br#"{"model":"requested"}"#),
+ model_range: 100..110,
+ streaming: true,
+ reservation: None,
+ },
Route::OpenAiResponses,
&mut RequestCorrelation::default(),
)
@@ -412,7 +417,10 @@ async fn deadlines_bound_headers_and_stalled_bodies_without_exposing_transport_d
let relay = ProviderTransport::for_test(
upstream.url.clone(),
ProviderLimits {
- headers_timeout: Duration::from_millis(30),
+ execution: ExecutionLimits {
+ stall_timeout: Duration::from_millis(30),
+ ..ExecutionLimits::default()
+ },
..ProviderLimits::default()
},
);
@@ -432,7 +440,10 @@ async fn deadlines_bound_headers_and_stalled_bodies_without_exposing_transport_d
let relay = ProviderTransport::for_test(
upstream.url.clone(),
ProviderLimits {
- execution_timeout: Duration::from_millis(100),
+ execution: ExecutionLimits {
+ max_execution_time: Duration::from_millis(100),
+ ..ExecutionLimits::default()
+ },
..ProviderLimits::default()
},
);
@@ -591,8 +602,10 @@ async fn cancelled_and_timed_out_executions_upload_after_provider_context_is_rel
upstream.url.clone(),
ProviderLimits {
active: 1,
- execution_timeout: Duration::from_millis(150),
- ..ProviderLimits::default()
+ execution: ExecutionLimits {
+ max_execution_time: Duration::from_millis(150),
+ ..ExecutionLimits::default()
+ },
},
)
.with_telemetry(telemetry.clone());
@@ -698,8 +711,12 @@ async fn compact_posts_compact_path_and_captures_ingestion() {
relay.try_admit().unwrap(),
resolved_request_context("provider-secret").await,
&HeaderMap::new(),
- Bytes::from_static(COMPACT),
- model_range(COMPACT).unwrap(),
+ RequestBody {
+ bytes: Bytes::from_static(COMPACT),
+ model_range: model_range(COMPACT).unwrap(),
+ streaming: true,
+ reservation: None,
+ },
Route::OpenAiResponsesCompact,
&mut RequestCorrelation::default(),
)
@@ -903,8 +920,12 @@ async fn sends_the_x_api_key_credential_and_forwards_the_open_anthropic_header_f
]
.concat(),
),
- Bytes::from_static(REQUEST),
- model_range(REQUEST).unwrap(),
+ RequestBody {
+ bytes: Bytes::from_static(REQUEST),
+ model_range: model_range(REQUEST).unwrap(),
+ streaming: true,
+ reservation: None,
+ },
Route::AnthropicMessages,
&mut RequestCorrelation::default(),
)
@@ -957,8 +978,13 @@ async fn count_tokens_relays_the_body() {
relay.try_admit().unwrap(),
context,
&headers(&[("content-type", "application/json")]),
- Bytes::from_static(br#"{"model":"claude-sonnet-4-5","messages":[]}"#),
- model_range(br#"{"model":"claude-sonnet-4-5","messages":[]}"#).unwrap(),
+ RequestBody {
+ bytes: Bytes::from_static(br#"{"model":"claude-sonnet-4-5","messages":[]}"#),
+ model_range: model_range(br#"{"model":"claude-sonnet-4-5","messages":[]}"#)
+ .unwrap(),
+ streaming: true,
+ reservation: None,
+ },
Route::AnthropicCountTokens,
&mut RequestCorrelation::default(),
)
@@ -1075,8 +1101,12 @@ async fn chat_completions_streams_request_usage_and_relay_the_usage_chunk() {
]
.concat(),
),
- Bytes::from_static(request.as_bytes()),
- model_range(request.as_bytes()).unwrap(),
+ RequestBody {
+ bytes: Bytes::from_static(request.as_bytes()),
+ model_range: model_range(request.as_bytes()).unwrap(),
+ streaming: true,
+ reservation: None,
+ },
Route::OpenAiChatCompletions,
&mut RequestCorrelation::default(),
)
@@ -1123,8 +1153,12 @@ async fn chat_completions_rewrite_non_streamed_models() {
)
.await,
&headers(&[("content-type", "application/json")]),
- Bytes::from_static(JSON_REQUEST.as_bytes()),
- model_range(JSON_REQUEST.as_bytes()).unwrap(),
+ RequestBody {
+ bytes: Bytes::from_static(JSON_REQUEST.as_bytes()),
+ model_range: model_range(JSON_REQUEST.as_bytes()).unwrap(),
+ streaming: true,
+ reservation: None,
+ },
Route::OpenAiChatCompletions,
&mut RequestCorrelation::default(),
)
diff --git a/ai-gateway/src/transport/mod.rs b/ai-gateway/src/transport/mod.rs
index 373ac3a0f4..754d3167f6 100644
--- a/ai-gateway/src/transport/mod.rs
+++ b/ai-gateway/src/transport/mod.rs
@@ -7,6 +7,7 @@ use std::{
atomic::{AtomicBool, AtomicUsize, Ordering},
},
task::{Context, Poll},
+ time::Duration,
};
use crate::{
@@ -151,6 +152,7 @@ pub(crate) fn identity_encoding(headers: &HeaderMap) -> bool {
pub(crate) fn relay<T: Send + 'static>(
upstream: reqwest::Response,
deadline: Instant,
+ stall_timeout: Duration,
owner: T,
capture: ExecutionCapture,
) -> Body {
@@ -165,14 +167,18 @@ pub(crate) fn relay<T: Send + 'static>(
})
}),
deadline,
+ stall_timeout,
owner,
Some(capture),
)
}
+/// Relay until EOF, the overall `deadline`, or `stall_timeout` without progress
+/// in either direction: no upstream chunk, or a downstream that stopped reading.
fn relay_stream<S, T>(
upstream: S,
deadline: Instant,
+ stall_timeout: Duration,
owner: T,
capture: Option<ExecutionCapture>,
) -> Body
@@ -206,7 +212,10 @@ where
let task = tokio::spawn(async move {
let pump = async {
tokio::pin!(upstream);
- while let Some(chunk) = upstream.next().await {
+ while let Some(chunk) = tokio::time::timeout(stall_timeout, upstream.next())
+ .await
+ .map_err(|_| ProviderError::Timeout)?
+ {
let chunk = chunk?;
pump_resources
.bytes
@@ -215,7 +224,13 @@ where
pump_resources.observe(|capture| capture.push_bytes(&chunk));
// One queued chunk plus one pending send; no per-chunk tasks.
for bytes in chunk.chunks(64 * 1024) {
- if sender.send(Bytes::copy_from_slice(bytes)).await.is_err() {
+ let sent = tokio::time::timeout(
+ stall_timeout,
+ sender.send(Bytes::copy_from_slice(bytes)),
+ )
+ .await
+ .map_err(|_| ProviderError::Timeout)?;
+ if sent.is_err() {
return Ok(());
}
}
@@ -236,10 +251,13 @@ where
}
drop(sender);
// Upstream EOF alone does not release admission: the downstream may still
- // have queued bytes. The deadline also covers an unpolled downstream body.
- if tokio::time::timeout_at(deadline, pump_resources.released.notified())
- .await
- .is_err()
+ // have queued bytes. The stall timeout also covers an unpolled downstream body.
+ if tokio::time::timeout_at(
+ deadline.min(Instant::now() + stall_timeout),
+ pump_resources.released.notified(),
+ )
+ .await
+ .is_err()
{
pump_resources.release(RelayOutcome::Timeout);
}
@@ -464,6 +482,7 @@ mod tests {
Err(ProviderError::Transport),
]),
Instant::now() + Duration::from_secs(1),
+ Duration::from_secs(10),
Owner(released.clone()),
None,
);
@@ -504,6 +523,7 @@ mod tests {
let body = relay_stream(
upstream,
Instant::now() + Duration::from_secs(10),
+ Duration::from_secs(10),
Owner(released.clone()),
None,
);
@@ -522,6 +542,7 @@ mod tests {
let body = relay_stream(
stream::iter([Ok(bytes.clone()), Err(ProviderError::Transport)]),
Instant::now() + Duration::from_secs(1),
+ Duration::from_secs(10),
Owner(released.clone()),
None,
);
@@ -541,6 +562,7 @@ mod tests {
let body = relay_stream(
stream::once(async { Ok(Bytes::from_static(b"final bytes")) }),
Instant::now() + Duration::from_millis(30),
+ Duration::from_secs(10),
Owner(released.clone()),
None,
);
@@ -549,4 +571,43 @@ mod tests {
.unwrap();
assert!(to_bytes(body, 1024).await.is_err());
}
+
+ #[tokio::test(start_paused = true)]
+ async fn a_silent_upstream_stalls_out_long_before_the_deadline() {
+ let released = Arc::new(Notify::new());
+ let started = Instant::now();
+ let body = relay_stream(
+ stream::once(async { Ok(Bytes::from_static(b"first")) }).chain(stream::pending()),
+ started + Duration::from_secs(3600),
+ Duration::from_secs(120),
+ Owner(released.clone()),
+ None,
+ );
+ let mut downstream = body.into_data_stream();
+ assert_eq!(downstream.next().await.unwrap().unwrap(), "first");
+ released.notified().await;
+ assert_eq!(started.elapsed(), Duration::from_secs(120));
+ assert!(downstream.next().await.unwrap().is_err());
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn a_downstream_that_stops_reading_stalls_out() {
+ let released = Arc::new(Notify::new());
+ let started = Instant::now();
+ // Steady upstream progress must not keep a non-reading client's slot alive.
+ let upstream =
+ stream::repeat_with(|| Ok(Bytes::from_static(b"chunk"))).then(|chunk| async {
+ tokio::time::sleep(Duration::from_secs(1)).await;
+ chunk
+ });
+ let _body = relay_stream(
+ upstream,
+ started + Duration::from_secs(3600),
+ Duration::from_secs(120),
+ Owner(released.clone()),
+ None,
+ );
+ released.notified().await;
+ assert!(started.elapsed() < Duration::from_secs(125));
+ }
}
diff --git a/ai-gateway/tests/startup.rs b/ai-gateway/tests/startup.rs
index ba9ff20057..813e0429cd 100644
--- a/ai-gateway/tests/startup.rs
+++ b/ai-gateway/tests/startup.rs
@@ -12,6 +12,10 @@ fn invalid_configuration_exits_without_disclosing_the_value() {
"LANGFUSE_AI_GATEWAY_WEB_URL",
"LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS",
"LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS",
+ "LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES",
+ "LANGFUSE_AI_GATEWAY_SHARED_REQUEST_BUFFER_BYTES",
+ "LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS",
+ "LANGFUSE_AI_GATEWAY_STALL_TIMEOUT_SECONDS",
] {
let output = Command::new(env!("CARGO_BIN_EXE_ai-gateway"))
.env_clear()
diff --git a/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.ts b/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.ts
index 87078c54f5..89df403959 100644
--- a/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.ts
+++ b/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.ts
@@ -12,6 +12,7 @@ import {
import { requireGatewayEnabledForOrganization } from "../availability";
import { verifyGatewayRequestSignature } from "./gatewayRequestSignatureVerifier";
+/** Outlives the gateway's default ten-minute execution ceiling plus upload batching. */
export const GATEWAY_INGESTION_TOKEN_TTL_SECONDS = 15 * 60;
const GatewayIngestionClaimsSchema = z.strictObject({