🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(ai-gateway): relay authenticated OpenAI Responses requests (#17352)
Hassieb Pakzad45f520827a2d
diff --git a/.env.dev.example b/.env.dev.exampleindex 101ee270b1..1087c02f1f 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -159,6 +159,13 @@ REDIS_AUTH="myredissecret"# openssl rand -hex 32 used only hereENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000+# Rust gateway: set the Web base URL to enable inference; include any Web base path.+# The service key below must match Web. Without this URL, readiness/inference return 503.+# LANGFUSE_AI_GATEWAY_WEB_URL=http://localhost:3000+# Independent per-instance concurrency limits; provisional defaults, tune with load tests.+# LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS=128+# LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS=128+# AI gateway control-plane key rotation. Generate service keys with# `openssl rand -hex 32` and ES256 PEM keys with# `openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256`.diff --git a/.env.prod.example b/.env.prod.exampleindex 826b246598..9400f5d93d 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -496,3 +496,12 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET=# CHB's resource-server identifier; defaults to billing-api.# CLICKHOUSE_BILLING_AUTH0_AUDIENCE=++# Rust AI gateway. Use the trusted HTTPS Web URL including any NEXT_PUBLIC_BASE_PATH.+# Without a Web URL the gateway starts, but readiness and inference return 503.+# LANGFUSE_AI_GATEWAY_WEB_URL=https://langfuse.example.com/app+# Same shared service key configured in Web. Provider keys are resolved through Web.+# LANGFUSE_AI_GATEWAY_SERVICE_KEY=+# Independent per-instance concurrency limits; provisional defaults, tune with load tests.+# LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS=128+# LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS=128diff --git a/ai-gateway/Cargo.lock b/ai-gateway/Cargo.lockindex 99225e5df4..de2918a800 100644--- a/ai-gateway/Cargo.lock+++ b/ai-gateway/Cargo.lock@@ -9,6 +9,7 @@ dependencies = ["axum","futures-util","hmac",+ "http-body-util","reqwest","serde","serde_json",@@ -257,6 +258,12 @@ version = "0.3.34"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"+[[package]]+name = "futures-io"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed"+[[package]]name = "futures-macro"version = "0.3.34"@@ -268,6 +275,12 @@ dependencies = ["syn 3.0.5",]+[[package]]+name = "futures-sink"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d"+[[package]]name = "futures-task"version = "0.3.34"@@ -281,8 +294,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"dependencies = ["futures-core",+ "futures-io","futures-macro",+ "futures-sink","futures-task",+ "memchr","pin-project-lite","slab",]@@ -771,6 +787,7 @@ dependencies = ["base64","bytes","futures-core",+ "futures-util","http","http-body","http-body-util",@@ -791,12 +808,14 @@ dependencies = ["sync_wrapper","tokio","tokio-rustls",+ "tokio-util","tower","tower-http","tower-service","url","wasm-bindgen","wasm-bindgen-futures",+ "wasm-streams","web-sys",]@@ -1180,6 +1199,19 @@ dependencies = ["tokio",]+[[package]]+name = "tokio-util"+version = "0.7.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52"+dependencies = [+ "bytes",+ "futures-core",+ "futures-sink",+ "pin-project-lite",+ "tokio",+]+[[package]]name = "tower"version = "0.5.3"@@ -1399,6 +1431,19 @@ dependencies = ["unicode-ident",]+[[package]]+name = "wasm-streams"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"+dependencies = [+ "futures-util",+ "js-sys",+ "wasm-bindgen",+ "wasm-bindgen-futures",+ "web-sys",+]+[[package]]name = "web-sys"version = "0.3.105"diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.tomlindex f737e62162..316aa4f509 100644--- a/ai-gateway/Cargo.toml+++ b/ai-gateway/Cargo.toml@@ -8,7 +8,9 @@ license = "MIT"[dependencies]axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "tokio"] }hmac = "0.13"-reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls-native-roots"] }+http-body-util = "0.1"+futures-util = "0.3"+reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls-native-roots", "stream"] }serde = { version = "1", features = ["derive"] }serde_json = "1"sha2 = "0.11"@@ -17,7 +19,6 @@ tracing = "0.1.44"tracing-subscriber = { version = "0.3.23", default-features = false, features = ["fmt", "json"] }[dev-dependencies]-futures-util = "0.3"tokio = { version = "1.53.1", features = ["io-util"] }tower = { version = "0.5", features = ["util"] }diff --git a/ai-gateway/README.md b/ai-gateway/README.mdindex 6deb790587..f847ed3dab 100644--- a/ai-gateway/README.md+++ b/ai-gateway/README.md@@ -1,9 +1,9 @@-# AI Gateway service foundation+# AI Gateway service-A standalone Rust HTTP process with health probes, bounded shutdown, structured-logs and a container, with a tested Web resolution client library. Inference and-customer telemetry are not implemented yet; inference paths return 404. No Web, database or provider-credentials are needed to build, start or test this package.+A standalone Rust gateway for `POST /openai/v1/responses`. Web resolves the gateway+key to a trusted provider connection; Rust relays native JSON or SSE without+parsing the request or rewriting provider bytes. Customer telemetry is not exported+yet. Building and testing need no real Web, database or provider credentials.## Run locally@@ -27,7 +27,8 @@ keep their built-in watchers. The gateway loads the root `.env` and listens onthe first available port starting at 8080. This lets multiple worktrees runtheir development stacks at the same time. Add overrides from the table belowto your root `.env`; exported shell variables take precedence.-No gateway-specific credentials are needed for this foundation slice.+Without a Web URL, the process starts with liveness available; readiness and+inference return 503. To enable inference, configure the Web URL and shared service key.To work on only the gateway, use `pnpm dev --filter=ai-gateway`.Both commands watch Rust source changes; do not run both on the same port.@@ -48,21 +49,25 @@ In another terminal:```shcurl --fail http://localhost:8080/health# {"status":"ok"}-curl --fail http://localhost:8080/ready-# {"status":"ready"}+curl http://localhost:8080/ready+# 503 without configuration; 200 {"status":"ready"} when configured```The binary reads configuration from the process environment. The pnpm developmentscript loads the root `.env` before starting it; production and direct Cargo runs-do not load dotenv files. All settings have defaults:--| Variable | Default | Validation |-| ------------------------------------------------ | -------------- | -------------------------------------------------- |-| `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS` | `0.0.0.0:8080` | IP address and port; IPv6 uses `[::]:8080` |-| `LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT` | `false` | `true`, `false`; the pnpm dev task sets `true` |-| `LANGFUSE_LOG_FORMAT` | `text` | `text`, `json` |-| `LANGFUSE_LOG_LEVEL` | `info` | `trace`, `debug`, `info`, `warn`, `error`, `fatal` |-| `LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS` | `10` | Integer from 1 to 300 |+do not load dotenv files:++| Variable | Default | Validation |+| ---------------------------------------------- | -------------- | -------------------------------------------------- |+| `LANGFUSE_AI_GATEWAY_WEB_URL` | unset; inference disabled | Trusted Web base URL, including any deployment prefix |+| `LANGFUSE_AI_GATEWAY_SERVICE_KEY` | unset | Required with Web URL; same key configured in Web |+| `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS` | `0.0.0.0:8080` | IP address and port; IPv6 uses `[::]:8080` |+| `LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT` | `false` | `true`, `false`; the pnpm dev task sets `true` |+| `LANGFUSE_LOG_FORMAT` | `text` | `text`, `json` |+| `LANGFUSE_LOG_LEVEL` | `info` | `trace`, `debug`, `info`, `warn`, `error`, `fatal` |+| `LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS` | `10` | Integer from 1 to 300 |+| `LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS` | `128` | Positive integer up to Tokio's semaphore capacity; authenticated requests per instance |+| `LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS` | `128` | Positive integer up to Tokio's semaphore capacity; concurrent Web resolutions per instance |The gateway shares `LANGFUSE_LOG_LEVEL` with Web and worker. Values are lowercase;`fatal` maps to Rust's `error` level and therefore includes ordinary error logs.@@ -79,17 +84,86 @@ Invalid values fail startup without echoing their contents. Logs contain servicelifecycle events, not request bodies or headers. For directhost-only development, set `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS=127.0.0.1:8080`.+## Call the gateway++Set these in the repository root `.env` for `pnpm dev` (or export them for Cargo):++```dotenv+LANGFUSE_AI_GATEWAY_WEB_URL=http://localhost:3000+LANGFUSE_AI_GATEWAY_SERVICE_KEY=<same-service-key-as-web>+```++For production use the HTTPS Web URL; include `/app` when Web was built with that+base path. Web also needs its existing ingestion JWT signer configured, an enabled+gateway key, and an eligible OpenAI connection. Rust needs neither the JWT signing+key nor an operator-supplied OpenAI key: provider credentials come from resolution.+A service key alone does not enable inference without the Web URL.++```sh+curl http://localhost:8080/openai/v1/responses \+ -H "Authorization: Bearer $LANGFUSE_GATEWAY_KEY" \+ -H 'Content-Type: application/json' \+ -d '{"model":"gpt-4.1-mini","input":"Say hello"}'+# Add "stream":true to the JSON and use curl -N for SSE.+```++```python+from openai import OpenAI+import os++client = OpenAI(+ base_url="http://localhost:8080/openai/v1",+ api_key=os.environ["LANGFUSE_GATEWAY_KEY"],+ max_retries=0,+)+response = client.responses.create(model="gpt-4.1-mini", input="Say hello")+print(response.output_text)+```++The gateway forwards to the official OpenAI Responses endpoint only. It does not+retry, follow redirects, parse `model`/`stream`, or accept client routing overrides.+Provider errors retain their status and body. Gateway errors use an OpenAI-style+`{"error":{"message":"...","type":"...","param":null,"code":"..."}}` envelope.++The gateway authenticates through Web before reserving execution capacity or+reading the request body. Web resolution uses a separate concurrency budget,+released after resolution finishes; authenticated requests then reserve execution+capacity through body reading and response completion. Either budget returns 503+immediately when full, without a waiting queue. Both defaults of 128 are provisional+guardrails, not measured capacity: tune them independently using load tests for the+instance resources, request sizes and stream durations. These limits bound work;+they do not guarantee fairness between clients or tenants.++Other limits are 4 MiB request bodies, 10 seconds to read a request, 5 seconds to+connect, 120 seconds for provider response headers or an individual upstream read,+and 600 seconds overall from execution admission. Response size is not capped: a single task pumps chunks+through a one-slot channel, with chunks at most 64 KiB. It stops reading when that+channel fills. Completion, disconnect and deadline release admission and context;+the deadline runs even when the downstream stops polling. A failure after headers+terminates the response body without adding an error event or retrying.+There is no separate downstream stall timeout; a client that stops reading can+retain execution capacity until the overall deadline. A progress-based downstream+stall policy is deferred to a separate change.++Only request content type/encoding and accept/accept-encoding cross the provider+boundary, plus the resolved Bearer token. Response content type/encoding, cache+control, retry-after, request ID and selected OpenAI timing/version/rate-limit+headers are retained. Cookies, routing overrides, gateway/ingestion credentials,+hop-by-hop headers and upstream framing are excluded.+## Process lifecycle- `/health` returns 200 while HTTP is serving. It never probes dependencies.-- `/ready` returns 200 after initialization. On shutdown, readiness changes to+- `/ready` returns 200 after initialization with inference configured. It does not+ probe Web or OpenAI; dependency failures are reported per request. Without a Web+ URL it returns 503. On shutdown, readiness changes to503 with `{"status":"draining"}` and the listener stops accepting connections.An external probe may see a closed connection instead of the brief 503 state.- SIGTERM or Ctrl-C starts graceful shutdown. In-flight requests may finishwithin the configured timeout. Deadline expiry exits nonzero and terminatesremaining runtime tasks; ordinary shutdown exits zero.-- The deadline starts on the shutdown signal, not at process startup. Future- stream/export slices must integrate their own work into this lifecycle.+- The shutdown deadline starts on the shutdown signal. New inference requests+ are rejected while draining; existing response streams share this budget.## Container@@ -108,29 +182,29 @@ The runtime image runs as UID/GID 10001, includes CA certificates, and executesthe binary directly so it receives signals. Set the container/orchestrator stopgrace period longer than the gateway shutdown timeout. The smoke test uses anisolated container and an ephemeral host port, checks read-only/non-root startup,-probes, missing inference routes and a clean SIGTERM exit, then removes it.+probes, rejection of unauthenticated inference and a clean SIGTERM exit, then removes it.## Tests and module boundaries### Web resolution client-`resolution::Resolver` resolves a gateway key through an operator-configured Web-base URL. It is a library boundary; the binary does not initialize it yet. The+`resolution::ControlPlaneClient` resolves a gateway key through an operator-configured Web+base URL. The binary initializes it when a Web URL is configured. Thecaller supplies the existing `LANGFUSE_AI_GATEWAY_SERVICE_KEY` and a trusted Web-base URL to `ResolverConfig::new`. HTTPS is required except on loopback for local+base URL to `ControlPlaneConfig::new`. HTTPS is required except on loopback for localdevelopment. URLs cannot contain credentials, a query or fragment. Include theWeb deployment's `NEXT_PUBLIC_BASE_PATH`, if set: both `https://host/app` and`https://host/app/` resolve through `/app/api/internal/ai-gateway/v1/resolve`.```rust,no_run-use ai_gateway::resolution::{ApiFormat, ResolveError, Resolver, ResolverConfig};+use ai_gateway::resolution::{ApiFormat, ResolutionError, ControlPlaneClient, ControlPlaneConfig};async fn example(web_base_url: &str, service_key: &str, gateway_key: &str)- -> Result<(), ResolveError>+ -> Result<(), ResolutionError>{- let resolver = Resolver::new(ResolverConfig::new(web_base_url, service_key)?)?;- let execution = resolver.resolve(gateway_key, ApiFormat::OpenAiResponses).await?;- assert_eq!(execution.connection().base_url(), "https://api.openai.com/v1");+ let control_plane = ControlPlaneClient::new(ControlPlaneConfig::new(web_base_url, service_key)?)?;+ let context = control_plane.resolve(gateway_key, ApiFormat::OpenAiResponses).await?;+ assert_eq!(context.connection().base_url(), "https://api.openai.com/v1");Ok(())}```@@ -181,6 +255,10 @@ cargo test --locked- `server.rs`: router, probe state and bounded graceful shutdown. A listener andshutdown future are injected, so tests do not depend on fixed ports or signals.- `main.rs`: configuration, logging, signal registration and process exit.+- `http.rs`: credential extraction, bounded body reads and gateway error envelopes.+- `inference.rs`: `InferenceService` coordinates resolution, admission and forwarding.+- `providers/openai.rs`: fixed destination, provider credentials and admission.+- `transport/mod.rs`: header allowlists, bounded byte relay and response lifetime.- `resolution/mod.rs`: trusted base URL configuration and bounded Web HTTP client.- `resolution/contracts.rs`: strict Web response validation and immutable execution context.- `resolution/signing.rs`: Web v1 HMAC; a literal shared fixture pins byte compatibility.diff --git a/ai-gateway/scripts/smoke-image.sh b/ai-gateway/scripts/smoke-image.shindex 57c10d31ba..70c740b016 100644--- a/ai-gateway/scripts/smoke-image.sh+++ b/ai-gateway/scripts/smoke-image.sh@@ -3,7 +3,9 @@ set -euo pipefailimage="${1:?Usage: bash scripts/smoke-image.sh IMAGE}"container="$(docker run -d --read-only --cap-drop ALL --security-opt no-new-privileges \- -e LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS=2 -p 127.0.0.1::8080 "$image")"+ -e LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS=2 \+ -e LANGFUSE_AI_GATEWAY_WEB_URL=https://web.example \+ -e LANGFUSE_AI_GATEWAY_SERVICE_KEY=smoke-test-service-key -p 127.0.0.1::8080 "$image")"cleanup() {docker logs "$container" || truedocker rm -f "$container" >/dev/null || true@@ -20,9 +22,9 @@ for route in health ready; dodone[[ "$(docker exec "$container" id -u)" != 0 ]][[ "$(curl --silent --output /dev/null --write-out '%{http_code}' \- --max-time 2 -X POST "http://$port/openai/v1/responses")" == 404 ]]+ --max-time 2 -X POST "http://$port/openai/v1/responses")" == 401 ]]docker stop --time 5 "$container" >/dev/null[[ "$(docker inspect --format '{{.State.ExitCode}}' "$container")" == 0 ]]docker logs "$container" 2>&1 | grep -q 'gateway draining'docker logs "$container" 2>&1 | grep -q 'gateway stopped'-printf '%s\n' 'Image smoke: non-root, probes, unimplemented inference, SIGTERM passed'+printf '%s\n' 'Image smoke: non-root, probes, unauthenticated inference, SIGTERM passed'diff --git a/ai-gateway/src/config.rs b/ai-gateway/src/config.rsindex e9b7c5a092..9deb1dd077 100644--- a/ai-gateway/src/config.rs+++ b/ai-gateway/src/config.rs@@ -1,12 +1,16 @@+use crate::resolution::ControlPlaneConfig;use std::{env, error::Error, fmt, net::SocketAddr, time::Duration};use tracing::level_filters::LevelFilter;-pub struct Config {+pub struct GatewayConfig {+ pub control_plane: Option<ControlPlaneConfig>,pub listen_address: SocketAddr,pub auto_increment_listen_port: bool,pub shutdown_timeout: Duration,pub log_level: LevelFilter,pub log_format: LogFormat,+ pub max_active_requests: usize,+ pub max_concurrent_resolutions: usize,}#[derive(Debug, PartialEq, Eq)]@@ -16,61 +20,87 @@ pub enum LogFormat {}#[derive(Debug, PartialEq, Eq)]-pub struct ConfigError(&'static str);+pub struct GatewayConfigError(&'static str);-impl fmt::Display for ConfigError {+impl fmt::Display for GatewayConfigError {fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {f.write_str(self.0)}}-impl Error for ConfigError {}+impl Error for GatewayConfigError {}-impl Config {+impl GatewayConfig {/// Read gateway configuration from the process environment, using defaults for absent values.////// # Errors/// Returns an error if a configured value is not Unicode or fails validation- /// in [`Self::from_values`]. Error messages never include the supplied values.- pub fn from_env() -> Result<Self, ConfigError> {- Self::from_values(+ /// in [`Self::from_values`], or if a Web URL is configured with an invalid URL+ /// or missing/blank service key. Error messages never include supplied values.+ pub fn from_env() -> Result<Self, GatewayConfigError> {+ let mut config = Self::from_values(read_env("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS")?.as_deref(),read_env("LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT")?.as_deref(),read_env("LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS")?.as_deref(),read_env("LANGFUSE_LOG_LEVEL")?.as_deref(),read_env("LANGFUSE_LOG_FORMAT")?.as_deref(),- )+ read_env("LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS")?.as_deref(),+ read_env("LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS")?.as_deref(),+ )?;+ if let Some(web_url) =+ read_env("LANGFUSE_AI_GATEWAY_WEB_URL")?.filter(|url| !url.is_empty())+ {+ let service_key = read_env("LANGFUSE_AI_GATEWAY_SERVICE_KEY")?.ok_or(GatewayConfigError(+ "LANGFUSE_AI_GATEWAY_SERVICE_KEY is required when LANGFUSE_AI_GATEWAY_WEB_URL is set",+ ))?;+ config.control_plane = Some(ControlPlaneConfig::new(&web_url, &service_key).map_err(+ |_| {+ GatewayConfigError(+ "invalid LANGFUSE_AI_GATEWAY_WEB_URL or LANGFUSE_AI_GATEWAY_SERVICE_KEY",+ )+ },+ )?);+ }+ Ok(config)}/// Parse gateway configuration, using defaults for absent values.////// # Errors- /// Returns an error for an invalid IP address and port, a shutdown timeout outside- /// 1–300 integer seconds, or an unsupported log level or format.+ /// Returns an error for an invalid IP address and port, a non-boolean port+ /// auto-increment setting, a shutdown timeout outside 1–300 integer seconds,+ /// an unsupported log level or format, or concurrency+ /// limits outside 1 through [`tokio::sync::Semaphore::MAX_PERMITS`].pub fn from_values(listen_address: Option<&str>,auto_increment_listen_port: Option<&str>,shutdown_timeout: Option<&str>,log_level: Option<&str>,log_format: Option<&str>,- ) -> Result<Self, ConfigError> {+ max_active_requests: Option<&str>,+ max_concurrent_resolutions: Option<&str>,+ ) -> Result<Self, GatewayConfigError> {let listen_address = listen_address.unwrap_or("0.0.0.0:8080").parse().map_err(|_| {- ConfigError("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS must be an IP address and port")+ GatewayConfigError(+ "LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS must be an IP address and port",+ )})?;- let auto_increment_listen_port =- parse_boolean(auto_increment_listen_port.unwrap_or("false")).ok_or(ConfigError(- "LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT must be true or false",- ))?;+ let auto_increment_listen_port = parse_boolean(+ auto_increment_listen_port.unwrap_or("false"),+ )+ .ok_or(GatewayConfigError(+ "LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT must be true or false",+ ))?;let seconds: u64 = shutdown_timeout.unwrap_or("10").parse().map_err(|_| {- ConfigError(+ GatewayConfigError("LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",)})?;if !(1..=300).contains(&seconds) {- return Err(ConfigError(+ return Err(GatewayConfigError("LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",));}@@ -82,7 +112,7 @@ impl Config {// Tracing has no separate fatal severity."error" | "fatal" => LevelFilter::ERROR,_ => {- return Err(ConfigError(+ return Err(GatewayConfigError("LANGFUSE_LOG_LEVEL must be trace, debug, info, warn, error or fatal",));}@@ -90,23 +120,48 @@ impl Config {let log_format = match log_format.unwrap_or("text") {"text" => LogFormat::Text,"json" => LogFormat::Json,- _ => return Err(ConfigError("LANGFUSE_LOG_FORMAT must be text or json")),+ _ => {+ return Err(GatewayConfigError(+ "LANGFUSE_LOG_FORMAT must be text or json",+ ));+ }};Ok(Self {+ control_plane: None,listen_address,auto_increment_listen_port,shutdown_timeout: Duration::from_secs(seconds),log_level,log_format,+ max_active_requests: concurrency_limit(+ max_active_requests,+ "LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS must be a positive integer within the semaphore capacity",+ )?,+ max_concurrent_resolutions: concurrency_limit(+ max_concurrent_resolutions,+ "LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS must be a positive integer within the semaphore capacity",+ )?,})}}-fn read_env(name: &'static str) -> Result<Option<String>, ConfigError> {+fn concurrency_limit(+ value: Option<&str>,+ message: &'static str,+) -> Result<usize, GatewayConfigError> {+ value+ .unwrap_or("128")+ .parse::<usize>()+ .ok()+ .filter(|limit| (1..=tokio::sync::Semaphore::MAX_PERMITS).contains(limit))+ .ok_or(GatewayConfigError(message))+}++fn read_env(name: &'static str) -> Result<Option<String>, GatewayConfigError> {match env::var(name) {Ok(value) => Ok(Some(value)),Err(env::VarError::NotPresent) => Ok(None),- Err(env::VarError::NotUnicode(_)) => Err(ConfigError(+ Err(env::VarError::NotUnicode(_)) => Err(GatewayConfigError("gateway configuration contains a non-Unicode environment value",)),}@@ -126,18 +181,22 @@ mod tests {#[test]fn configuration_parses_defaults_and_overrides() {- let default = Config::from_values(None, None, None, None, None).unwrap();+ let default = GatewayConfig::from_values(None, None, None, None, None, None, None).unwrap();assert_eq!(default.listen_address, "0.0.0.0:8080".parse().unwrap());assert!(!default.auto_increment_listen_port);assert_eq!(default.shutdown_timeout, Duration::from_secs(10));assert_eq!(default.log_level, LevelFilter::INFO);assert_eq!(default.log_format, LogFormat::Text);- let custom = Config::from_values(+ assert_eq!(default.max_active_requests, 128);+ assert_eq!(default.max_concurrent_resolutions, 128);+ let custom = GatewayConfig::from_values(Some("[::1]:9000"),Some("true"),Some("30"),Some("debug"),Some("json"),+ Some("256"),+ Some("32"),).unwrap();assert_eq!(custom.listen_address, "[::1]:9000".parse().unwrap());@@ -145,20 +204,22 @@ mod tests {assert_eq!(custom.shutdown_timeout, Duration::from_secs(30));assert_eq!(custom.log_level, LevelFilter::DEBUG);assert_eq!(custom.log_format, LogFormat::Json);+ assert_eq!(custom.max_active_requests, 256);+ assert_eq!(custom.max_concurrent_resolutions, 32);}#[test]fn validates_shared_log_format() {for (value, expected) in [("text", LogFormat::Text), ("json", LogFormat::Json)] {assert_eq!(- Config::from_values(None, None, None, None, Some(value))+ GatewayConfig::from_values(None, None, None, None, Some(value), None, None).unwrap().log_format,expected);}for value in ["", "TEXT", "pretty", "0", "secret-that-must-not-appear"] {- let error = Config::from_values(None, None, None, None, Some(value))+ let error = GatewayConfig::from_values(None, None, None, None, Some(value), None, None).err().unwrap();assert_eq!(@@ -178,7 +239,9 @@ mod tests {("error", LevelFilter::ERROR),("fatal", LevelFilter::ERROR),] {- let config = Config::from_values(None, None, None, Some(value), None).unwrap();+ let config =+ GatewayConfig::from_values(None, None, None, Some(value), None, None, None)+ .unwrap();assert_eq!(config.log_level, expected, "{value}");}}@@ -199,18 +262,58 @@ mod tests {(None, Some("-1"), None),(Some("localhost:8080"), None, None),] {- let error = Config::from_values(address, None, timeout, level, None)+ let error = GatewayConfig::from_values(address, None, timeout, level, None, None, None).err().unwrap();assert!(!error.to_string().contains(sensitive_input));assert!(!format!("{error:?}").contains(sensitive_input));}for value in ["", "TRUE", "1", sensitive_input] {- let error = Config::from_values(None, Some(value), None, None, None)+ let error = GatewayConfig::from_values(None, Some(value), None, None, None, None, None).err().unwrap();assert!(!error.to_string().contains(sensitive_input));assert!(!format!("{error:?}").contains(sensitive_input));}}++ #[test]+ fn concurrency_limits_reject_invalid_values_and_accept_boundaries() {+ let maximum = tokio::sync::Semaphore::MAX_PERMITS.to_string();+ let overflow = (tokio::sync::Semaphore::MAX_PERMITS + 1).to_string();+ for value in ["1", maximum.as_str()] {+ let config =+ GatewayConfig::from_values(None, None, None, None, None, Some(value), Some(value))+ .unwrap();+ assert_eq!(config.max_active_requests, value.parse::<usize>().unwrap());+ assert_eq!(+ config.max_concurrent_resolutions,+ value.parse::<usize>().unwrap()+ );+ }+ for value in [+ "0",+ "-1",+ "1.5",+ "",+ "secret-that-must-not-appear",+ overflow.as_str(),+ ] {+ for (active, resolutions, name) in [+ (Some(value), None, "LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS"),+ (+ None,+ Some(value),+ "LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS",+ ),+ ] {+ let error =+ GatewayConfig::from_values(None, None, None, None, None, active, resolutions)+ .err()+ .unwrap();+ assert!(error.to_string().contains(name));+ assert!(!error.to_string().contains("secret-that-must-not-appear"));+ }+ }+ }}diff --git a/ai-gateway/src/http.rs b/ai-gateway/src/http.rsnew file mode 100644index 0000000000..f56f4138cd--- /dev/null+++ b/ai-gateway/src/http.rs@@ -0,0 +1,202 @@+//! Public OpenAI-compatible envelope around the opaque resolve/execute flow.+use std::{error::Error, sync::Arc, time::Duration};++use axum::{+ Json, Router,+ body::{Body, to_bytes},+ extract::{Request, State},+ http::{HeaderMap, StatusCode, header},+ response::{IntoResponse, Response},+ routing::post,+};+use serde::Serialize;++use crate::{+ inference::{InferenceService, RequestPreparationError},+ providers::openai::ProviderError,+ resolution::ResolutionError,+ server::GatewayLifecycleState,+};++const MAX_REQUEST_BYTES: usize = 4 * 1024 * 1024;+const REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(10);++#[derive(Clone)]+struct InferenceRouteState {+ inference: Option<Arc<InferenceService>>,+ lifecycle: GatewayLifecycleState,+}++/// Build inference routes. An unconfigured or draining service returns 503.+pub fn router(inference: Option<InferenceService>, lifecycle: GatewayLifecycleState) -> Router {+ Router::new()+ .route("/openai/v1/responses", post(handle_responses))+ .with_state(InferenceRouteState {+ inference: inference.map(Arc::new),+ lifecycle,+ })+}++async fn handle_responses(+ State(state): State<InferenceRouteState>,+ request: Request,+) -> Result<Response, InferenceHttpError> {+ let inference = state+ .inference+ .as_ref()+ .filter(|_| state.lifecycle.is_ready())+ .ok_or(InferenceHttpError::Unavailable)?;+ let gateway_key = gateway_key(request.headers())?.to_owned();+ let (permit, context) = inference+ .resolve_and_admit(&gateway_key)+ .await+ .map_err(|error| match error {+ RequestPreparationError::Resolution(error) => InferenceHttpError::Resolution(error),+ RequestPreparationError::Provider(error) => InferenceHttpError::Provider(error),+ })?;+ let (parts, body) = request.into_parts();+ let bytes = tokio::time::timeout(REQUEST_READ_TIMEOUT, to_bytes(body, MAX_REQUEST_BYTES))+ .await+ .map_err(|_| InferenceHttpError::RequestTimeout)?+ .map_err(|error| {+ if error+ .source()+ .is_some_and(<dyn Error + 'static>::is::<http_body_util::LengthLimitError>)+ {+ InferenceHttpError::TooLarge+ } else {+ InferenceHttpError::InvalidBody+ }+ })?;+ inference+ .forward(permit, context, &parts.headers, bytes)+ .await+ .map_err(InferenceHttpError::Provider)+}++fn gateway_key(headers: &HeaderMap) -> Result<&str, InferenceHttpError> {+ let mut values = headers.get_all(header::AUTHORIZATION).iter();+ let value = values+ .next()+ .and_then(|value| value.to_str().ok())+ .ok_or(InferenceHttpError::Credential)?;+ if values.next().is_some() {+ return Err(InferenceHttpError::Credential);+ }+ let (scheme, token) = value+ .split_once(' ')+ .ok_or(InferenceHttpError::Credential)?;+ if !scheme.eq_ignore_ascii_case("Bearer")+ || token.is_empty()+ || token.len() > 8192+ || !token+ .bytes()+ .all(|byte| byte.is_ascii_graphic() && byte != b',')+ {+ return Err(InferenceHttpError::Credential);+ }+ Ok(token)+}++enum InferenceHttpError {+ Unavailable,+ Credential,+ TooLarge,+ RequestTimeout,+ InvalidBody,+ Resolution(ResolutionError),+ Provider(ProviderError),+}++#[derive(Serialize)]+struct OpenAiErrorResponse {+ error: OpenAiErrorDetail,+}+#[derive(Serialize)]+struct OpenAiErrorDetail {+ message: &'static str,+ #[serde(rename = "type")]+ kind: &'static str,+ param: Option<&'static str>,+ code: &'static str,+}++impl IntoResponse for InferenceHttpError {+ fn into_response(self) -> Response<Body> {+ use ResolutionError as R;+ let (status, message, kind, code) = match self {+ Self::Credential | Self::Resolution(R::InvalidCredential | R::Authentication) => (+ StatusCode::UNAUTHORIZED,+ "Invalid gateway credential",+ "authentication_error",+ "invalid_api_key",+ ),+ Self::Resolution(R::Forbidden) => (+ StatusCode::FORBIDDEN,+ "Gateway access forbidden",+ "permission_error",+ "permission_denied",+ ),+ Self::Resolution(R::NoRoute) => (+ StatusCode::NOT_FOUND,+ "No eligible provider connection",+ "invalid_request_error",+ "no_route",+ ),+ Self::TooLarge => (+ StatusCode::PAYLOAD_TOO_LARGE,+ "Request body exceeds gateway limit",+ "invalid_request_error",+ "request_too_large",+ ),+ Self::RequestTimeout => (+ StatusCode::REQUEST_TIMEOUT,+ "Request body read deadline exceeded",+ "invalid_request_error",+ "request_timeout",+ ),+ Self::InvalidBody => (+ StatusCode::BAD_REQUEST,+ "Invalid request body",+ "invalid_request_error",+ "invalid_body",+ ),+ Self::Unavailable+ | Self::Resolution(R::Unavailable | R::Configuration)+ | Self::Provider(ProviderError::Configuration | ProviderError::Busy) => (+ StatusCode::SERVICE_UNAVAILABLE,+ "Gateway temporarily unavailable",+ "server_error",+ "gateway_unavailable",+ ),+ Self::Resolution(R::Timeout) | Self::Provider(ProviderError::Timeout) => (+ StatusCode::GATEWAY_TIMEOUT,+ "Upstream deadline exceeded",+ "server_error",+ "upstream_timeout",+ ),+ Self::Resolution(R::Transport | R::InvalidResponse | R::ResponseTooLarge)+ | Self::Provider(ProviderError::Transport) => (+ StatusCode::BAD_GATEWAY,+ "Upstream request failed",+ "server_error",+ "upstream_error",+ ),+ };+ (+ status,+ Json(OpenAiErrorResponse {+ error: OpenAiErrorDetail {+ message,+ kind,+ param: None,+ code,+ },+ }),+ )+ .into_response()+ }+}++#[cfg(test)]+mod tests;diff --git a/ai-gateway/src/http/tests.rs b/ai-gateway/src/http/tests.rsnew file mode 100644index 0000000000..ad6aa91780--- /dev/null+++ b/ai-gateway/src/http/tests.rs@@ -0,0 +1,578 @@+use super::*;+use crate::{+ providers::openai::{OpenAiProvider, ProviderLimits},+ test_support::{FakeServer, resolution_response},+};+use futures_util::{StreamExt, stream};+use std::{convert::Infallible, task::Poll};+use tokio::{net::TcpListener, sync::oneshot, task::JoinHandle};+use tower::ServiceExt;++#[tokio::test]+async fn invalid_credentials_are_rejected_before_admission_or_body_reads() {+ let web = FakeServer::start(|_| async {+ Response::builder()+ .status(StatusCode::UNAUTHORIZED)+ .body(Body::empty())+ .unwrap()+ })+ .await;+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;+ let provider_client = OpenAiProvider::for_test(+ provider.url.clone(),+ ProviderLimits {+ active: 1,+ ..ProviderLimits::default()+ },+ );+ let permit = provider_client.try_admit().unwrap();+ let gateway = Gateway::start(Some(InferenceService::for_test(+ web.control_plane(),+ provider_client,+ 1,+ )))+ .await;+ // Exercise both an exhausted and an available execution pool.+ for occupied in [Some(permit), None] {+ let body = Body::from_stream(stream::poll_fn(+ |_| -> Poll<Option<Result<&'static str, Infallible>>> {+ panic!("an invalid credential must be rejected before polling the body");+ },+ ));+ let response = tokio::time::timeout(+ Duration::from_secs(2),+ gateway.app.clone().oneshot(+ Request::post("/openai/v1/responses")+ .header(header::AUTHORIZATION, "Bearer invalid-key")+ .body(body)+ .unwrap(),+ ),+ )+ .await+ .unwrap()+ .unwrap();+ assert_eq!(response.status(), StatusCode::UNAUTHORIZED);+ drop(occupied);+ }+ assert_eq!(web.calls(), 2);+ assert_eq!(provider.calls(), 0);+}++struct Gateway {+ url: String,+ shutdown: Option<oneshot::Sender<()>>,+ task: JoinHandle<std::io::Result<()>>,+ app: Router,+}++#[tokio::test]+async fn resolution_capacity_is_bounded_and_released_on_cancellation_and_failure() {+ let entered = Arc::new(tokio::sync::Notify::new());+ let started = entered.clone();+ let web = FakeServer::start(move |request| {+ let started = started.clone();+ async move {+ match request.headers()[header::AUTHORIZATION].to_str().unwrap() {+ "Bearer stalled" => {+ started.notify_one();+ std::future::pending().await+ }+ "Bearer invalid" => Response::builder()+ .status(StatusCode::UNAUTHORIZED)+ .body(Body::empty())+ .unwrap(),+ _ => resolution_response("provider"),+ }+ }+ })+ .await;+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;+ let service = InferenceService::for_test(+ web.control_plane(),+ OpenAiProvider::for_test(provider.url.clone(), ProviderLimits::default()),+ 1,+ );+ let gateway = Gateway::start(Some(service)).await;+ let request = |key| {+ Request::post("/openai/v1/responses")+ .header(header::AUTHORIZATION, key)+ .body(Body::from("{}"))+ .unwrap()+ };+ let mut stalled = Box::pin(gateway.app.clone().oneshot(request("Bearer stalled")));+ tokio::time::timeout(Duration::from_secs(2), async {+ tokio::select! {+ () = entered.notified() => {},+ _ = &mut stalled => panic!("resolution should be pending"),+ }+ })+ .await+ .unwrap();+ let busy = tokio::time::timeout(+ Duration::from_secs(2),+ gateway.app.clone().oneshot(request("Bearer valid")),+ )+ .await+ .unwrap()+ .unwrap();+ assert_eq!(busy.status(), StatusCode::SERVICE_UNAVAILABLE);+ assert_eq!(web.calls(), 1);+ assert_eq!(provider.calls(), 0);++ drop(stalled);+ for (key, status) in [+ ("Bearer invalid", StatusCode::UNAUTHORIZED),+ ("Bearer valid", StatusCode::OK),+ ] {+ let response = tokio::time::timeout(+ Duration::from_secs(2),+ gateway.app.clone().oneshot(request(key)),+ )+ .await+ .unwrap()+ .unwrap();+ assert_eq!(response.status(), status);+ }+ assert_eq!(web.calls(), 3);+ assert_eq!(provider.calls(), 1);+}++impl Gateway {+ async fn start(inference: Option<InferenceService>) -> Self {+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();+ let url = format!("http://{}", listener.local_addr().unwrap());+ let state = if inference.is_some() {+ GatewayLifecycleState::default()+ } else {+ GatewayLifecycleState::unconfigured()+ };+ let app = crate::server::router(state.clone()).merge(router(inference, state.clone()));+ let (shutdown, signal) = oneshot::channel();+ let task = tokio::spawn(crate::server::serve(+ listener,+ app.clone(),+ state,+ async {+ let _ = signal.await;+ },+ Duration::from_secs(1),+ ));+ let client = reqwest::Client::new();+ for _ in 0..100 {+ if client.get(format!("{url}/health")).send().await.is_ok() {+ return Self {+ url,+ shutdown: Some(shutdown),+ task,+ app,+ };+ }+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ panic!("gateway did not start");+ }++ fn post(&self) -> reqwest::RequestBuilder {+ reqwest::Client::new().post(format!("{}/openai/v1/responses", self.url))+ }+}++impl Drop for Gateway {+ fn drop(&mut self) {+ self.task.abort();+ }+}++fn inference(web: &FakeServer, provider: &FakeServer) -> InferenceService {+ InferenceService::for_test(+ web.control_plane(),+ OpenAiProvider::for_test(+ format!("{}/v1/responses", provider.url),+ ProviderLimits::default(),+ ),+ 128,+ )+}++#[tokio::test]+async fn native_json_and_sse_traverse_resolution_and_relay() {+ let web = FakeServer::start(|request| async move {+ assert_eq!(request.uri(), "/api/internal/ai-gateway/v1/resolve");+ assert!(+ request+ .headers()+ .contains_key("langfuse-gateway-authorization")+ );+ let key = request.headers()[header::AUTHORIZATION]+ .to_str()+ .unwrap()+ .to_owned();+ assert_eq!(+ to_bytes(request.into_body(), 1024).await.unwrap(),+ r#"{"apiFormat":"openai.responses"}"#+ );+ resolution_response(if key == "Bearer gateway-json" {+ "provider-json"+ } else {+ "provider-sse"+ })+ })+ .await;+ let provider = FakeServer::start(|request| async move {+ assert_eq!(request.uri(), "/v1/responses");+ assert!(!request.headers().contains_key("x-langfuse-project-id"));+ assert!(+ !request+ .headers()+ .contains_key("langfuse-gateway-authorization")+ );+ let key = request.headers()[header::AUTHORIZATION]+ .to_str()+ .unwrap()+ .to_owned();+ let body = to_bytes(request.into_body(), 4096).await.unwrap();+ if key == "Bearer provider-json" {+ assert_eq!(body, "{ \"model\":\"unchanged\", \"unknown\":true }");+ Response::builder()+ .header("content-type", "application/json")+ .body(Body::from("{\"output\":[]}"))+ .unwrap()+ } else {+ assert_eq!(key, "Bearer provider-sse");+ assert_eq!(body, r#"{"model":"anything","stream":true}"#);+ Response::builder()+ .header("content-type", "text/event-stream")+ .body(Body::from_stream(stream::iter([+ Ok::<_, Infallible>("event: arbitrary\ndata: "),+ Ok("hello\n\n"),+ ])))+ .unwrap()+ }+ })+ .await;+ let gateway = Gateway::start(Some(inference(&web, &provider))).await;+ let json = gateway+ .post()+ .bearer_auth("gateway-json")+ .header("x-langfuse-project-id", "attacker-project")+ .body("{ \"model\":\"unchanged\", \"unknown\":true }")+ .send();+ let sse = gateway+ .post()+ .bearer_auth("gateway-sse")+ .body(r#"{"model":"anything","stream":true}"#)+ .send();+ let (json, sse) = tokio::join!(json, sse);+ let json = json.unwrap();+ let sse = sse.unwrap();+ assert_eq!(json.status(), StatusCode::OK);+ assert_eq!(json.text().await.unwrap(), r#"{"output":[]}"#);+ assert_eq!(sse.headers()[header::CONTENT_TYPE], "text/event-stream");+ assert_eq!(+ sse.text().await.unwrap(),+ "event: arbitrary\ndata: hello\n\n"+ );+ assert_eq!(web.calls(), 2);+ assert_eq!(provider.calls(), 2);+}++#[tokio::test]+async fn malformed_credentials_skip_resolution_and_oversized_body_releases_capacity() {+ let web = FakeServer::start(|_| async { resolution_response("provider-secret") }).await;+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;+ let service = InferenceService::for_test(+ web.control_plane(),+ OpenAiProvider::for_test(+ provider.url.clone(),+ ProviderLimits {+ active: 1,+ ..ProviderLimits::default()+ },+ ),+ 1,+ );+ let gateway = Gateway::start(Some(service)).await;+ for authorization in [+ None,+ Some("Basic abc"),+ Some("Bearer "),+ Some("Bearer key,other"),+ Some("Bearer key extra"),+ ] {+ let mut request = gateway.post();+ if let Some(value) = authorization {+ request = request.header(header::AUTHORIZATION, value);+ }+ let response = request.send().await.unwrap();+ assert_eq!(response.status(), StatusCode::UNAUTHORIZED);+ let body: serde_json::Value =+ serde_json::from_slice(&response.bytes().await.unwrap()).unwrap();+ assert_eq!(body["error"]["code"], "invalid_api_key");+ }+ let mut headers = HeaderMap::new();+ headers.append(header::AUTHORIZATION, "Bearer one".parse().unwrap());+ headers.append(header::AUTHORIZATION, "Bearer two".parse().unwrap());+ assert_eq!(+ gateway+ .post()+ .headers(headers)+ .send()+ .await+ .unwrap()+ .status(),+ StatusCode::UNAUTHORIZED+ );+ assert_eq!(web.calls(), 0);+ let response = gateway+ .post()+ .bearer_auth("valid-key")+ .body(vec![b'x'; MAX_REQUEST_BYTES + 1])+ .send()+ .await+ .unwrap();+ assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE);+ assert_eq!(web.calls(), 1);+ assert_eq!(provider.calls(), 0);+ let next = gateway+ .post()+ .bearer_auth("valid-key")+ .body("{}")+ .send()+ .await+ .unwrap();+ assert_eq!(next.status(), StatusCode::OK);+ assert_eq!(provider.calls(), 1);+}++#[tokio::test]+async fn resolution_failures_are_sanitized_and_never_call_provider() {+ let web = FakeServer::start(|request| async move {+ let key = request.headers()[header::AUTHORIZATION].to_str().unwrap();+ match key {+ "Bearer malformed" => Response::new(Body::from("secret malformed response")),+ "Bearer invalid-origin" => {+ let body = to_bytes(resolution_response("provider-secret").into_body(), 4096)+ .await+ .unwrap();+ let mut body: serde_json::Value = serde_json::from_slice(&body).unwrap();+ body["connection"]["base_url"] = "https://attacker.example/v1".into();+ Response::new(Body::from(body.to_string()))+ }+ _ => {+ let status: u16 = key.strip_prefix("Bearer ").unwrap().parse().unwrap();+ Response::builder()+ .status(status)+ .body(Body::from("secret upstream error"))+ .unwrap()+ }+ }+ })+ .await;+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;+ let gateway = Gateway::start(Some(inference(&web, &provider))).await;+ for (key, status) in [+ ("401", 401),+ ("403", 403),+ ("404", 404),+ ("503", 503),+ ("malformed", 502),+ ("invalid-origin", 502),+ ] {+ let response = gateway+ .post()+ .bearer_auth(key)+ .body("{}")+ .send()+ .await+ .unwrap();+ assert_eq!(response.status().as_u16(), status);+ let body = response.text().await.unwrap();+ assert!(!body.contains("secret"));+ assert!(serde_json::from_str::<serde_json::Value>(&body).unwrap()["error"].is_object());+ }+ assert_eq!(provider.calls(), 0);+}++#[tokio::test]+async fn slow_request_body_and_resolution_have_bounded_waits() {+ let web = FakeServer::start(|request| async move {+ if request.headers()[header::AUTHORIZATION] != "Bearer slow-resolution" {+ return resolution_response("provider");+ }+ Response::new(Body::from_stream(stream::pending::<+ Result<&'static str, Infallible>,+ >()))+ })+ .await;+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;+ let gateway = Gateway::start(Some(inference(&web, &provider))).await;+ let slow_body = gateway+ .post()+ .bearer_auth("key")+ .body(reqwest::Body::wrap_stream(stream::pending::<+ Result<&'static str, Infallible>,+ >()))+ .send();+ let slow_resolution = gateway+ .post()+ .bearer_auth("slow-resolution")+ .body("{}")+ .send();+ let (body, resolution) = tokio::time::timeout(Duration::from_secs(12), async {+ tokio::join!(slow_body, slow_resolution)+ })+ .await+ .unwrap();+ assert_eq!(body.unwrap().status(), StatusCode::REQUEST_TIMEOUT);+ assert_eq!(resolution.unwrap().status(), StatusCode::GATEWAY_TIMEOUT);+ assert_eq!(web.calls(), 2);+ assert_eq!(provider.calls(), 0);+}++#[tokio::test]+async fn unconfigured_gateway_is_live_but_not_ready_and_inference_is_unavailable() {+ let gateway = Gateway::start(None).await;+ let client = reqwest::Client::new();+ assert_eq!(+ client+ .get(format!("{}/health", gateway.url))+ .send()+ .await+ .unwrap()+ .status(),+ StatusCode::OK+ );+ assert_eq!(+ client+ .get(format!("{}/ready", gateway.url))+ .send()+ .await+ .unwrap()+ .status(),+ StatusCode::SERVICE_UNAVAILABLE+ );+ assert_eq!(+ gateway.post().send().await.unwrap().status(),+ StatusCode::SERVICE_UNAVAILABLE+ );+}++#[tokio::test]+async fn active_admission_is_held_through_stream_and_shutdown_drains_it() {+ let (release, released) = tokio::sync::watch::channel(false);+ let web = FakeServer::start(|_| async { resolution_response("provider") }).await;+ let provider = FakeServer::start(move |_| {+ let mut released = released.clone();+ async move {+ let stream = stream::once(async { Ok::<_, Infallible>("data: first\n\n") }).chain(+ stream::once(async move {+ released.wait_for(|done| *done).await.unwrap();+ Ok("data: last\n\n")+ }),+ );+ Response::builder()+ .header("content-type", "text/event-stream")+ .body(Body::from_stream(stream))+ .unwrap()+ }+ })+ .await;+ let service = InferenceService::for_test(+ web.control_plane(),+ OpenAiProvider::for_test(+ provider.url.clone(),+ ProviderLimits {+ active: 1,+ ..ProviderLimits::default()+ },+ ),+ 1,+ );+ let mut gateway = Gateway::start(Some(service)).await;+ let mut response = gateway+ .post()+ .bearer_auth("key")+ .body("{}")+ .send()+ .await+ .unwrap();+ assert_eq!(response.chunk().await.unwrap().unwrap(), "data: first\n\n");+ assert_eq!(+ gateway+ .post()+ .bearer_auth("key")+ .body("{}")+ .send()+ .await+ .unwrap()+ .status(),+ StatusCode::SERVICE_UNAVAILABLE+ );+ assert_eq!(web.calls(), 2);+ assert_eq!(provider.calls(), 1);+ gateway.shutdown.take().unwrap().send(()).unwrap();+ assert!(!gateway.task.is_finished());+ release.send(true).unwrap();+ assert_eq!(response.text().await.unwrap(), "data: last\n\n");+ tokio::time::timeout(Duration::from_secs(2), &mut gateway.task)+ .await+ .unwrap()+ .unwrap()+ .unwrap();+}++#[tokio::test]+async fn client_disconnect_releases_capacity_for_the_next_request() {+ let web = FakeServer::start(|_| async { resolution_response("provider") }).await;+ let provider = FakeServer::start(|_| async {+ let stream =+ stream::once(async { Ok::<_, Infallible>("data: first\n\n") }).chain(stream::pending());+ Response::builder()+ .header("content-type", "text/event-stream")+ .body(Body::from_stream(stream))+ .unwrap()+ })+ .await;+ let service = InferenceService::for_test(+ web.control_plane(),+ OpenAiProvider::for_test(+ provider.url.clone(),+ ProviderLimits {+ active: 1,+ ..ProviderLimits::default()+ },+ ),+ 1,+ );+ let gateway = Gateway::start(Some(service)).await;+ let mut first = gateway+ .post()+ .bearer_auth("key")+ .body("{}")+ .send()+ .await+ .unwrap();+ assert_eq!(first.chunk().await.unwrap().unwrap(), "data: first\n\n");+ drop(first);+ let mut next = tokio::time::timeout(Duration::from_secs(2), async {+ loop {+ let response = gateway+ .post()+ .bearer_auth("key")+ .body("{}")+ .send()+ .await+ .unwrap();+ if response.status() == StatusCode::OK {+ break response;+ }+ assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ })+ .await+ .unwrap();+ assert_eq!(next.chunk().await.unwrap().unwrap(), "data: first\n\n");+ assert!(web.calls() >= 2);+ assert_eq!(provider.calls(), 2);+}diff --git a/ai-gateway/src/inference.rs b/ai-gateway/src/inference.rsnew file mode 100644index 0000000000..fce29bf6bc--- /dev/null+++ b/ai-gateway/src/inference.rs@@ -0,0 +1,91 @@+//! Resolve the caller's credential, then execute using only the trusted context.+use axum::{+ body::{Body, Bytes},+ http::{HeaderMap, Response},+};+use tokio::sync::Semaphore;++use crate::{+ providers::openai::{OpenAiProvider, ProviderError, RequestPermit},+ resolution::{+ ApiFormat, ControlPlaneClient, ControlPlaneConfig, ResolutionError, ResolvedRequestContext,+ },+};++pub struct InferenceService {+ control_plane: ControlPlaneClient,+ provider: OpenAiProvider,+ resolution_capacity: Semaphore,+}++pub(crate) enum RequestPreparationError {+ Resolution(ResolutionError),+ Provider(ProviderError),+}++impl InferenceService {+ /// Initialize pooled clients for resolution and provider requests.+ ///+ /// # Errors+ /// Returns a sanitized configuration error for invalid capacities or client setup.+ pub fn new(+ config: ControlPlaneConfig,+ max_active_requests: usize,+ max_concurrent_resolutions: usize,+ ) -> Result<Self, ResolutionError> {+ if !(1..=Semaphore::MAX_PERMITS).contains(&max_concurrent_resolutions) {+ return Err(ResolutionError::Configuration);+ }+ Ok(Self {+ control_plane: ControlPlaneClient::new(config)?,+ provider: OpenAiProvider::new(max_active_requests)+ .map_err(|_| ResolutionError::Configuration)?,+ resolution_capacity: Semaphore::new(max_concurrent_resolutions),+ })+ }++ /// Authenticate with a separate bounded budget before reserving execution capacity.+ pub(crate) async fn resolve_and_admit(+ &self,+ gateway_key: &str,+ ) -> Result<(RequestPermit, ResolvedRequestContext), RequestPreparationError> {+ let context = {+ let _permit = self+ .resolution_capacity+ .try_acquire()+ .map_err(|_| RequestPreparationError::Resolution(ResolutionError::Unavailable))?;+ self.control_plane+ .resolve(gateway_key, ApiFormat::OpenAiResponses)+ .await+ .map_err(RequestPreparationError::Resolution)?+ };+ let permit = self+ .provider+ .try_admit()+ .map_err(RequestPreparationError::Provider)?;+ Ok((permit, context))+ }++ pub(crate) async fn forward(+ &self,+ permit: RequestPermit,+ context: ResolvedRequestContext,+ headers: &HeaderMap,+ body: Bytes,+ ) -> Result<Response<Body>, ProviderError> {+ self.provider.forward(permit, context, headers, body).await+ }++ #[cfg(test)]+ pub(crate) fn for_test(+ control_plane: ControlPlaneClient,+ provider: OpenAiProvider,+ resolutions: usize,+ ) -> Self {+ Self {+ control_plane,+ provider,+ resolution_capacity: Semaphore::new(resolutions),+ }+ }+}diff --git a/ai-gateway/src/lib.rs b/ai-gateway/src/lib.rsindex ff115b8a31..60740a9181 100644--- a/ai-gateway/src/lib.rs+++ b/ai-gateway/src/lib.rs@@ -1,3 +1,10 @@pub mod config;+pub mod http;+pub mod inference;+pub mod providers;pub mod resolution;pub mod server;+mod transport;++#[cfg(test)]+mod test_support;diff --git a/ai-gateway/src/main.rs b/ai-gateway/src/main.rsindex fa0511ea9a..3d4873dabd 100644--- a/ai-gateway/src/main.rs+++ b/ai-gateway/src/main.rs@@ -1,8 +1,10 @@use std::{error::Error, io, net::SocketAddr, process::ExitCode};use ai_gateway::{- config::{Config, LogFormat},- server::{self, AppState},+ config::{GatewayConfig, LogFormat},+ http,+ inference::InferenceService,+ server::{self, GatewayLifecycleState},};use tokio::net::TcpListener;@@ -18,7 +20,7 @@ async fn main() -> ExitCode {}async fn run() -> Result<(), Box<dyn Error>> {- let config = Config::from_env()?;+ let config = GatewayConfig::from_env()?;let logging = tracing_subscriber::fmt().with_max_level(config.log_level).with_target(false);@@ -26,18 +28,27 @@ async fn run() -> Result<(), Box<dyn Error>> {LogFormat::Text => logging.compact().init(),LogFormat::Json => logging.json().init(),}+ let inference = config+ .control_plane+ .map(|control_plane| {+ InferenceService::new(+ control_plane,+ config.max_active_requests,+ config.max_concurrent_resolutions,+ )+ })+ .transpose()?;+ let inference_enabled = inference.is_some();+ let state = if inference_enabled {+ GatewayLifecycleState::default()+ } else {+ GatewayLifecycleState::unconfigured()+ };+ let app = server::router(state.clone()).merge(http::router(inference, state.clone()));let shutdown = shutdown_signal()?;let listener = bind_listener(config.listen_address, config.auto_increment_listen_port).await?;- tracing::info!(address = %listener.local_addr()?, "gateway listening");- let state = AppState::default();- server::serve(- listener,- server::router(state.clone()),- state,- shutdown,- config.shutdown_timeout,- )- .await?;+ tracing::info!(address = %listener.local_addr()?, inference_enabled, "gateway listening");+ server::serve(listener, app, state, shutdown, config.shutdown_timeout).await?;tracing::info!("gateway stopped");Ok(())}diff --git a/ai-gateway/src/providers/mod.rs b/ai-gateway/src/providers/mod.rsnew file mode 100644index 0000000000..d8c308735b--- /dev/null+++ b/ai-gateway/src/providers/mod.rs@@ -0,0 +1 @@+pub mod openai;diff --git a/ai-gateway/src/providers/openai.rs b/ai-gateway/src/providers/openai.rsnew file mode 100644index 0000000000..e8882905b4--- /dev/null+++ b/ai-gateway/src/providers/openai.rs@@ -0,0 +1,163 @@+//! Native Responses transport. Provider requests use only resolved credentials.+use std::{sync::Arc, time::Duration};++use axum::{+ body::{Body, Bytes},+ http::{HeaderMap, HeaderValue, Response, header},+};+use reqwest::Client;+use tokio::{+ sync::{OwnedSemaphorePermit, Semaphore},+ time::Instant,+};++use crate::{resolution::ResolvedRequestContext, transport};++pub use crate::transport::ProviderError;++pub(crate) struct ProviderLimits {+ pub active: usize,+ pub execution_timeout: Duration,+ pub headers_timeout: Duration,+ pub read_timeout: Duration,+}++impl Default for ProviderLimits {+ fn default() -> Self {+ Self {+ active: 128,+ execution_timeout: Duration::from_secs(600),+ headers_timeout: Duration::from_secs(120),+ read_timeout: Duration::from_secs(120),+ }+ }+}++/// An admitted execution. Dropping it releases capacity; there is no waiting queue.+pub struct RequestPermit {+ _permit: OwnedSemaphorePermit,+ deadline: Instant,+}++/// A pooled client for the official `OpenAI` Responses endpoint.+pub struct OpenAiProvider {+ client: Client,+ capacity: Arc<Semaphore>,+ limits: ProviderLimits,+ #[cfg(test)]+ endpoint: String,+}++impl OpenAiProvider {+ /// Construct the provider transport with bounded admission and transport waits.+ ///+ /// # Errors+ /// Returns [`ProviderError::Configuration`] when the HTTPS client cannot be initialized.+ pub fn new(max_active_requests: usize) -> Result<Self, ProviderError> {+ Self::with_limits(ProviderLimits {+ active: max_active_requests,+ ..ProviderLimits::default()+ })+ }++ fn with_limits(limits: ProviderLimits) -> Result<Self, ProviderError> {+ if !(1..=Semaphore::MAX_PERMITS).contains(&limits.active) {+ return Err(ProviderError::Configuration);+ }+ let client = Client::builder()+ .redirect(reqwest::redirect::Policy::none())+ .retry(reqwest::retry::never())+ .no_proxy()+ .no_gzip()+ .no_brotli()+ .no_deflate()+ .no_zstd()+ .connect_timeout(Duration::from_secs(5))+ .read_timeout(limits.read_timeout)+ .pool_max_idle_per_host(16)+ .build()+ .map_err(|_| ProviderError::Configuration)?;+ Ok(Self {+ client,+ capacity: Arc::new(Semaphore::new(limits.active)),+ limits,+ #[cfg(test)]+ endpoint: "https://api.openai.com/v1/responses".to_owned(),+ })+ }++ /// Reserve capacity for an authenticated request before reading its body.+ ///+ /// # Errors+ /// Returns [`ProviderError::Busy`] immediately when all execution slots are occupied.+ pub fn try_admit(&self) -> Result<RequestPermit, ProviderError> {+ let permit = self+ .capacity+ .clone()+ .try_acquire_owned()+ .map_err(|_| ProviderError::Busy)?;+ Ok(RequestPermit {+ _permit: permit,+ deadline: Instant::now() + self.limits.execution_timeout,+ })+ }++ /// Execute once and stream native status, safe headers and entity bytes.+ /// The response body owns admission and the trusted context until completion/drop.+ ///+ /// # Errors+ /// Returns a sanitized error for invalid credentials, transport failures or a+ /// deadline before response headers arrive. Later failures terminate the body.+ pub async fn forward(+ &self,+ permit: RequestPermit,+ context: ResolvedRequestContext,+ headers: &HeaderMap,+ body: Bytes,+ ) -> Result<Response<Body>, ProviderError> {+ #[cfg(not(test))]+ let endpoint = "https://api.openai.com/v1/responses";+ #[cfg(test)]+ let endpoint = &self.endpoint;++ let mut authorization =+ HeaderValue::from_str(&format!("Bearer {}", context.connection().provider_token()))+ .map_err(|_| ProviderError::Configuration)?;+ authorization.set_sensitive(true);+ let response = tokio::time::timeout_at(+ permit+ .deadline+ .min(Instant::now() + self.limits.headers_timeout),+ self.client+ .post(endpoint)+ .headers(transport::request_headers(headers))+ .header(header::AUTHORIZATION, authorization)+ .body(body)+ .send(),+ )+ .await+ .map_err(|_| ProviderError::Timeout)?+ .map_err(|error| {+ if error.is_timeout() {+ ProviderError::Timeout+ } else {+ ProviderError::Transport+ }+ })?;+ let mut downstream = Response::new(Body::empty());+ *downstream.status_mut() = response.status();+ *downstream.headers_mut() = transport::response_headers(response.headers());+ *downstream.body_mut() = transport::relay(response, permit.deadline, (permit, context));+ Ok(downstream)+ }++ #[cfg(test)]+ pub(crate) fn for_test(endpoint: String, limits: ProviderLimits) -> Self {+ let mut provider = Self::with_limits(limits).unwrap();+ provider.endpoint = endpoint;+ provider+ }+}++#[cfg(test)]+mod tests;diff --git a/ai-gateway/src/providers/openai/tests.rs b/ai-gateway/src/providers/openai/tests.rsnew file mode 100644index 0000000000..4c1bde86b8--- /dev/null+++ b/ai-gateway/src/providers/openai/tests.rs@@ -0,0 +1,333 @@+use super::*;+use crate::test_support::{FakeServer, resolved_request_context};+use axum::{body::to_bytes, http::StatusCode};+use futures_util::{StreamExt, stream};+use std::{convert::Infallible, future::pending};+use tokio::sync::Notify;++fn provider(server: &FakeServer, active: usize) -> OpenAiProvider {+ OpenAiProvider::for_test(+ format!("{}/v1/responses", server.url),+ ProviderLimits {+ active,+ ..ProviderLimits::default()+ },+ )+}++#[tokio::test]+async fn preserves_opaque_bytes_and_isolates_request_and_response_headers() {+ const REQUEST: &[u8] = b"{ \"model\": \"opaque\", \"future_field\": [true, 3] }\n";+ const RESPONSE: &[u8] = b"\x1f\x8b\x08\x00opaque-encoded-response";+ let upstream = FakeServer::start(|request| async move {+ assert_eq!(request.uri(), "/v1/responses");+ assert_eq!(request.method(), "POST");+ assert_eq!(+ request.headers()[header::AUTHORIZATION],+ "Bearer provider-secret"+ );+ assert_eq!(request.headers()[header::CONTENT_TYPE], "application/json");+ for name in [+ "cookie",+ "langfuse-gateway-authorization",+ "openai-organization",+ "openai-project",+ "x-api-key",+ "x-forwarded-host",+ "accept-encoding",+ ] {+ assert!(!request.headers().contains_key(name), "forwarded {name}");+ }+ assert_eq!(to_bytes(request.into_body(), 1024).await.unwrap(), REQUEST);+ Response::builder()+ .status(201)+ .header("content-type", "application/json")+ .header("content-encoding", "gzip")+ .header("x-request-id", "request-123")+ .header("set-cookie", "private-cookie")+ .header("connection", "openai-version")+ .header("openai-version", "private-hop-value")+ .body(Body::from(RESPONSE))+ .unwrap()+ })+ .await;+ let relay = provider(&upstream, 1);+ let mut headers = HeaderMap::new();+ for (name, value) in [+ ("authorization", "Bearer gateway-secret"),+ ("cookie", "session=secret"),+ ("langfuse-gateway-authorization", "service-signature"),+ ("openai-organization", "wrong-org"),+ ("openai-project", "wrong-project"),+ ("x-api-key", "wrong-key"),+ ("x-forwarded-host", "attacker.example"),+ ("content-type", "application/json"),+ ("accept-encoding", "gzip"),+ ("connection", "accept-encoding"),+ ] {+ headers.insert(+ axum::http::HeaderName::from_static(name),+ HeaderValue::from_static(value),+ );+ }+ let response = relay+ .forward(+ relay.try_admit().unwrap(),+ resolved_request_context("provider-secret").await,+ &headers,+ Bytes::from_static(REQUEST),+ )+ .await+ .unwrap();+ assert_eq!(response.status(), StatusCode::CREATED);+ assert_eq!(response.headers()["content-encoding"], "gzip");+ assert_eq!(response.headers()["x-request-id"], "request-123");+ for name in [+ "set-cookie",+ "connection",+ "openai-version",+ "content-length",+ "transfer-encoding",+ ] {+ assert!(!response.headers().contains_key(name));+ }+ assert_eq!(+ to_bytes(response.into_body(), 1024).await.unwrap(),+ RESPONSE+ );+ assert!(relay.try_admit().is_ok());+ assert_eq!(upstream.calls(), 1);+}++#[tokio::test]+async fn preserves_provider_failures_without_retries_or_redirects() {+ let target = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;+ for status in [307, 429, 500, 503] {+ let location = target.url.clone();+ let upstream = FakeServer::start(move |_| {+ let location = location.clone();+ async move {+ Response::builder()+ .status(status)+ .header("location", location)+ .header("retry-after", "7")+ .body(Body::from("native-provider-body"))+ .unwrap()+ }+ })+ .await;+ let relay = provider(&upstream, 1);+ let response = relay+ .forward(+ relay.try_admit().unwrap(),+ resolved_request_context("provider-secret").await,+ &HeaderMap::new(),+ Bytes::new(),+ )+ .await+ .unwrap();+ assert_eq!(response.status().as_u16(), status);+ assert_eq!(response.headers()["retry-after"], "7");+ assert!(!response.headers().contains_key("location"));+ assert_eq!(+ to_bytes(response.into_body(), 1024).await.unwrap(),+ "native-provider-body"+ );+ assert_eq!(upstream.calls(), 1);+ }+ assert_eq!(target.calls(), 0);+}++#[tokio::test]+async fn streams_sse_incrementally_and_holds_admission_until_eof() {+ let release = Arc::new(Notify::new());+ let upstream_release = release.clone();+ let upstream = FakeServer::start(move |_| {+ let release = upstream_release.clone();+ async move {+ let chunks = stream::once(async { Ok::<_, Infallible>("data: first\n\n") }).chain(+ stream::once(async move {+ release.notified().await;+ Ok("data: [DONE]\n\n")+ }),+ );+ Response::builder()+ .header("content-type", "text/event-stream")+ .body(Body::from_stream(chunks))+ .unwrap()+ }+ })+ .await;+ let relay = provider(&upstream, 1);+ let response = relay+ .forward(+ relay.try_admit().unwrap(),+ resolved_request_context("provider-secret").await,+ &HeaderMap::new(),+ Bytes::new(),+ )+ .await+ .unwrap();+ assert_eq!(response.headers()["content-type"], "text/event-stream");+ let mut body = response.into_body().into_data_stream();+ assert_eq!(+ tokio::time::timeout(Duration::from_secs(1), body.next())+ .await+ .unwrap()+ .unwrap()+ .unwrap(),+ "data: first\n\n"+ );+ assert!(matches!(relay.try_admit(), Err(ProviderError::Busy)));+ release.notify_one();+ let mut remaining = Vec::new();+ while let Some(chunk) = body.next().await {+ remaining.extend_from_slice(&chunk.unwrap());+ }+ assert_eq!(remaining, b"data: [DONE]\n\n");+ assert!(relay.try_admit().is_ok());+}++#[tokio::test]+async fn simultaneous_requests_keep_provider_credentials_and_bodies_isolated() {+ let upstream = FakeServer::start(|request| async move {+ let authorization = request.headers()[header::AUTHORIZATION]+ .to_str()+ .unwrap()+ .to_owned();+ let body = to_bytes(request.into_body(), 1024).await.unwrap();+ Response::new(Body::from(format!(+ "{authorization}:{}",+ String::from_utf8(body.to_vec()).unwrap()+ )))+ })+ .await;+ let relay = provider(&upstream, 2);+ let alice = resolved_request_context("alice-secret").await;+ let bob = resolved_request_context("bob-secret").await;+ let headers = HeaderMap::new();+ let (alice, bob) = tokio::join!(+ relay.forward(+ relay.try_admit().unwrap(),+ alice,+ &headers,+ Bytes::from_static(b"alice-body")+ ),+ relay.forward(+ relay.try_admit().unwrap(),+ bob,+ &headers,+ Bytes::from_static(b"bob-body")+ ),+ );+ assert_eq!(+ to_bytes(alice.unwrap().into_body(), 1024).await.unwrap(),+ "Bearer alice-secret:alice-body"+ );+ assert_eq!(+ to_bytes(bob.unwrap().into_body(), 1024).await.unwrap(),+ "Bearer bob-secret:bob-body"+ );+ assert_eq!(upstream.calls(), 2);+}++struct Dropped(Arc<Notify>);+impl Drop for Dropped {+ fn drop(&mut self) {+ self.0.notify_one();+ }+}++async fn stalled_provider(dropped: Arc<Notify>) -> FakeServer {+ FakeServer::start(move |_| {+ let guard = Dropped(dropped.clone());+ async move {+ let tail = stream::once(async move {+ let _guard = guard;+ pending::<Result<&str, Infallible>>().await+ });+ Response::new(Body::from_stream(+ stream::once(async { Ok::<_, Infallible>("first") }).chain(tail),+ ))+ }+ })+ .await+}++#[tokio::test]+async fn dropping_downstream_cancels_upstream_and_releases_admission() {+ let dropped = Arc::new(Notify::new());+ let upstream = stalled_provider(dropped.clone()).await;+ let relay = provider(&upstream, 1);+ let response = relay+ .forward(+ relay.try_admit().unwrap(),+ resolved_request_context("provider-secret").await,+ &HeaderMap::new(),+ Bytes::new(),+ )+ .await+ .unwrap();+ let mut body = response.into_body().into_data_stream();+ assert_eq!(body.next().await.unwrap().unwrap(), "first");+ assert!(matches!(relay.try_admit(), Err(ProviderError::Busy)));+ drop(body);+ assert!(relay.try_admit().is_ok());+ tokio::time::timeout(Duration::from_secs(1), dropped.notified())+ .await+ .expect("upstream body should be dropped on cancellation");+}++#[tokio::test]+async fn deadlines_bound_headers_and_stalled_bodies_without_exposing_transport_details() {+ let upstream = FakeServer::start(|_| async {+ pending::<()>().await;+ Response::new(Body::empty())+ })+ .await;+ let relay = OpenAiProvider::for_test(+ upstream.url.clone(),+ ProviderLimits {+ headers_timeout: Duration::from_millis(30),+ ..ProviderLimits::default()+ },+ );+ let response = relay+ .forward(+ relay.try_admit().unwrap(),+ resolved_request_context("provider-secret").await,+ &HeaderMap::new(),+ Bytes::new(),+ )+ .await;+ assert!(matches!(response, Err(ProviderError::Timeout)));+ assert!(relay.try_admit().is_ok());++ let dropped = Arc::new(Notify::new());+ let upstream = stalled_provider(dropped.clone()).await;+ let relay = OpenAiProvider::for_test(+ upstream.url.clone(),+ ProviderLimits {+ execution_timeout: Duration::from_millis(100),+ ..ProviderLimits::default()+ },+ );+ let context = resolved_request_context("provider-secret").await;+ let response = relay+ .forward(+ relay.try_admit().unwrap(),+ context,+ &HeaderMap::new(),+ Bytes::new(),+ )+ .await+ .unwrap();+ // The deadline must cancel upstream even when downstream does not poll its body.+ tokio::time::timeout(Duration::from_secs(1), dropped.notified())+ .await+ .expect("deadline must run without downstream polling");+ let error = to_bytes(response.into_body(), 1024).await.unwrap_err();+ assert!(!error.to_string().contains(&upstream.url));+ assert!(!error.to_string().contains("provider-secret"));+ assert!(relay.try_admit().is_ok());+}diff --git a/ai-gateway/src/resolution/contracts.rs b/ai-gateway/src/resolution/contracts.rsindex 94311d3afb..3474b7d616 100644--- a/ai-gateway/src/resolution/contracts.rs+++ b/ai-gateway/src/resolution/contracts.rs@@ -1,4 +1,4 @@-use super::{ResolveError, valid_token};+use super::{ResolutionError, valid_token};use serde::{Deserialize, Deserializer, Serialize,de::{DeserializeOwned, IntoDeserializer},@@ -47,7 +47,7 @@ struct Auth {#[derive(Deserialize)]#[serde(deny_unknown_fields)]-pub struct Connection {+pub struct ProviderConnection {id: String,#[serde(rename = "provider", deserialize_with = "string_enum")]_provider: Provider,@@ -57,7 +57,7 @@ pub struct Connection {auth: Auth,}-impl Connection {+impl ProviderConnection {pub fn id(&self) -> &str {&self.id}@@ -74,7 +74,7 @@ impl Connection {#[derive(Deserialize)]#[serde(deny_unknown_fields)]-pub struct Attribution {+pub struct RequestAttribution {organization_id: String,project_id: String,key_id: String,@@ -82,7 +82,7 @@ pub struct Attribution {provider_connection_id: String,}-impl Attribution {+impl RequestAttribution {pub fn organization_id(&self) -> &str {&self.organization_id}@@ -102,14 +102,14 @@ impl Attribution {#[derive(Deserialize)]#[serde(deny_unknown_fields)]-pub struct Ingestion {+pub struct IngestionGrant {access_token: String,#[serde(rename = "token_type", deserialize_with = "string_enum")]_token_type: TokenType,expires_at: u64,}-impl Ingestion {+impl IngestionGrant {pub fn access_token(&self) -> &str {&self.access_token}@@ -120,17 +120,17 @@ impl Ingestion {#[derive(Deserialize)]#[serde(deny_unknown_fields)]-struct Response {+struct ResolutionResponse {version: u8,- connection: Connection,- attribution: Attribution,+ connection: ProviderConnection,+ attribution: RequestAttribution,#[serde(deserialize_with = "string_enum")]ingestion_mode: IngestionMode,- ingestion: Ingestion,+ ingestion: IngestionGrant,}/// A validated v1 execution. Construction is restricted to successful resolution.-pub struct ResolvedExecution(Response);+pub struct ResolvedRequestContext(ResolutionResponse);// Web enums are JSON strings. Serde's externally tagged enums also accept objects.fn string_enum<'de, D: Deserializer<'de>, T: DeserializeOwned>(@@ -139,24 +139,25 @@ fn string_enum<'de, D: Deserializer<'de>, T: DeserializeOwned>(T::deserialize(String::deserialize(deserializer)?.into_deserializer())}-impl ResolvedExecution {- pub fn connection(&self) -> &Connection {+impl ResolvedRequestContext {+ pub fn connection(&self) -> &ProviderConnection {&self.0.connection}- pub fn attribution(&self) -> &Attribution {+ pub fn attribution(&self) -> &RequestAttribution {&self.0.attribution}pub fn ingestion_mode(&self) -> IngestionMode {self.0.ingestion_mode}- pub fn ingestion(&self) -> &Ingestion {+ pub fn ingestion(&self) -> &IngestionGrant {&self.0.ingestion}}-impl fmt::Debug for ResolvedExecution {+impl fmt::Debug for ResolvedRequestContext {fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {- f.debug_struct("ResolvedExecution").finish_non_exhaustive()+ f.debug_struct("ResolvedRequestContext")+ .finish_non_exhaustive()}}@@ -164,9 +165,9 @@ pub(super) fn decode(bytes: &[u8],expected_format: ApiFormat,now: u64,-) -> Result<ResolvedExecution, ResolveError> {- let response: Response =- serde_json::from_slice(bytes).map_err(|_| ResolveError::InvalidResponse)?;+) -> Result<ResolvedRequestContext, ResolutionError> {+ let response: ResolutionResponse =+ serde_json::from_slice(bytes).map_err(|_| ResolutionError::InvalidResponse)?;let connection = &response.connection;let attribution = &response.attribution;if response.version != 1@@ -185,7 +186,7 @@ pub(super) fn decode(.iter().any(|value| value.trim().is_empty()){- return Err(ResolveError::InvalidResponse);+ return Err(ResolutionError::InvalidResponse);}- Ok(ResolvedExecution(response))+ Ok(ResolvedRequestContext(response))}diff --git a/ai-gateway/src/resolution/mod.rs b/ai-gateway/src/resolution/mod.rsindex 6094d0937b..83d1c61f19 100644--- a/ai-gateway/src/resolution/mod.rs+++ b/ai-gateway/src/resolution/mod.rs@@ -3,7 +3,8 @@ mod contracts;mod signing;pub use contracts::{- ApiFormat, Attribution, Connection, Ingestion, IngestionMode, MetadataValue, ResolvedExecution,+ ApiFormat, IngestionGrant, IngestionMode, MetadataValue, ProviderConnection,+ RequestAttribution, ResolvedRequestContext,};use reqwest::{Client, Url,@@ -17,27 +18,27 @@ use std::{const RESOLVE_PATH: &str = "/api/internal/ai-gateway/v1/resolve";-struct Limits {+struct ResolutionLimits {timeout: Duration,max_response_bytes: usize,}/// Operator-supplied Web base URL and the existing gateway/Web service key.-pub struct ResolverConfig {+pub struct ControlPlaneConfig {url: Url,service_key: String,- limits: Limits,+ limits: ResolutionLimits,}-impl ResolverConfig {+impl ControlPlaneConfig {/// Configure resolution against a trusted Web base URL, including any deployment prefix.////// # Errors- /// Returns [`ResolveError::Configuration`] for a blank service key or an invalid+ /// Returns [`ResolutionError::Configuration`] for a blank service key or an invalid/// base URL. URLs require HTTPS (HTTP is allowed on loopback), with no userinfo,/// query, or fragment.- pub fn new(web_url: &str, service_key: &str) -> Result<Self, ResolveError> {- let mut url = Url::parse(web_url).map_err(|_| ResolveError::Configuration)?;+ pub fn new(web_url: &str, service_key: &str) -> Result<Self, ResolutionError> {+ let mut url = Url::parse(web_url).map_err(|_| ResolutionError::Configuration)?;let loopback = url.host_str().is_some_and(|host| {host == "localhost"|| host@@ -53,14 +54,14 @@ impl ResolverConfig {|| url.fragment().is_some()|| service_key.trim().is_empty(){- return Err(ResolveError::Configuration);+ return Err(ResolutionError::Configuration);}let resolve_path = format!("{}{RESOLVE_PATH}", url.path().trim_end_matches('/'));url.set_path(&resolve_path);Ok(Self {url,service_key: service_key.to_owned(),- limits: Limits {+ limits: ResolutionLimits {timeout: Duration::from_secs(5),max_response_bytes: 256 * 1024,},@@ -69,17 +70,17 @@ impl ResolverConfig {}/// Reuses the HTTP connection pool; credentials belong exclusively to each request.-pub struct Resolver {+pub struct ControlPlaneClient {client: Client,- config: ResolverConfig,+ config: ControlPlaneConfig,}-impl Resolver {+impl ControlPlaneClient {/// Build a resolver with a reusable HTTP connection pool.////// # Errors- /// Returns [`ResolveError::Configuration`] if the HTTP client cannot be initialized.- pub fn new(config: ResolverConfig) -> Result<Self, ResolveError> {+ /// Returns [`ResolutionError::Configuration`] if the HTTP client cannot be initialized.+ pub fn new(config: ControlPlaneConfig) -> Result<Self, ResolutionError> {let client = Client::builder().redirect(reqwest::redirect::Policy::none()).retry(reqwest::retry::never())@@ -89,24 +90,24 @@ impl Resolver {.no_zstd().no_deflate().build()- .map_err(|_| ResolveError::Configuration)?;+ .map_err(|_| ResolutionError::Configuration)?;Ok(Self { client, config })}/// Resolve a gateway credential and API format into a validated execution contract.////// # Errors- /// Returns a sanitized [`ResolveError`] for invalid credentials, authentication or+ /// Returns a sanitized [`ResolutionError`] for invalid credentials, authentication or/// authorization failures, missing routes, unavailable Web or system time, request/// construction or transport failures, timeouts, or oversized or invalid responses.pub async fn resolve(&self,gateway_key: &str,api_format: ApiFormat,- ) -> Result<ResolvedExecution, ResolveError> {+ ) -> Result<ResolvedRequestContext, ResolutionError> {let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)- .map_err(|_| ResolveError::Unavailable)?;+ .map_err(|_| ResolutionError::Unavailable)?;self.resolve_at(gateway_key, api_format, timestamp).await}@@ -115,9 +116,9 @@ impl Resolver {gateway_key: &str,api_format: ApiFormat,timestamp: Duration,- ) -> Result<ResolvedExecution, ResolveError> {+ ) -> Result<ResolvedRequestContext, ResolutionError> {if gateway_key.len() > 8192 || !valid_token(gateway_key) {- return Err(ResolveError::InvalidCredential);+ return Err(ResolutionError::InvalidCredential);}let started = Instant::now();let body = tokio::time::timeout(@@ -125,7 +126,7 @@ impl Resolver {self.fetch(gateway_key, api_format, timestamp.as_secs()),).await- .map_err(|_| ResolveError::Timeout)??;+ .map_err(|_| ResolutionError::Timeout)??;contracts::decode(&body,api_format,@@ -138,19 +139,19 @@ impl Resolver {gateway_key: &str,api_format: ApiFormat,timestamp: u64,- ) -> Result<Vec<u8>, ResolveError> {+ ) -> Result<Vec<u8>, ResolutionError> {let mut credential = HeaderValue::from_str(&format!("Bearer {gateway_key}"))- .map_err(|_| ResolveError::InvalidCredential)?;+ .map_err(|_| ResolutionError::InvalidCredential)?;credential.set_sensitive(true);let mut signature = HeaderValue::from_str(&signing::authorization(&self.config.service_key,gateway_key,timestamp,))- .map_err(|_| ResolveError::Configuration)?;+ .map_err(|_| ResolutionError::Configuration)?;signature.set_sensitive(true);let body = serde_json::to_vec(&serde_json::json!({ "apiFormat": api_format }))- .map_err(|_| ResolveError::Configuration)?;+ .map_err(|_| ResolutionError::Configuration)?;let mut response = self.client.post(self.config.url.clone())@@ -160,30 +161,30 @@ impl Resolver {.body(body).send().await- .map_err(|_| ResolveError::Transport)?;+ .map_err(|_| ResolutionError::Transport)?;match response.status().as_u16() {200 => {}- 401 => return Err(ResolveError::Authentication),- 403 => return Err(ResolveError::Forbidden),- 404 => return Err(ResolveError::NoRoute),- 429 | 500..=599 => return Err(ResolveError::Unavailable),- _ => return Err(ResolveError::InvalidResponse),+ 401 => return Err(ResolutionError::Authentication),+ 403 => return Err(ResolutionError::Forbidden),+ 404 => return Err(ResolutionError::NoRoute),+ 429 | 500..=599 => return Err(ResolutionError::Unavailable),+ _ => return Err(ResolutionError::InvalidResponse),}let limit = self.config.limits.max_response_bytes;if response.content_length().is_some_and(|size| size > limit as u64){- return Err(ResolveError::ResponseTooLarge);+ return Err(ResolutionError::ResponseTooLarge);}let mut bytes = Vec::new();while let Some(chunk) = response.chunk().await- .map_err(|_| ResolveError::Transport)?+ .map_err(|_| ResolutionError::Transport)?{if chunk.len() > limit - bytes.len() {- return Err(ResolveError::ResponseTooLarge);+ return Err(ResolutionError::ResponseTooLarge);}bytes.extend_from_slice(&chunk);}@@ -197,7 +198,7 @@ fn valid_token(value: &str) -> bool {/// Sanitized failure categories. Upstream bodies and transport errors are never retained.#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum ResolveError {+pub enum ResolutionError {Configuration,InvalidCredential,Authentication,@@ -210,7 +211,7 @@ pub enum ResolveError {ResponseTooLarge,}-impl fmt::Display for ResolveError {+impl fmt::Display for ResolutionError {fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {f.write_str(match self {Self::Configuration => "invalid resolver configuration",@@ -227,7 +228,7 @@ impl fmt::Display for ResolveError {}}-impl std::error::Error for ResolveError {}+impl std::error::Error for ResolutionError {}#[cfg(test)]mod tests;diff --git a/ai-gateway/src/resolution/tests.rs b/ai-gateway/src/resolution/tests.rsindex 0873822f96..ef74457584 100644--- a/ai-gateway/src/resolution/tests.rs+++ b/ai-gateway/src/resolution/tests.rs@@ -54,17 +54,21 @@ impl FakeWeb {Self { url, calls, task }}- fn resolver(&self) -> Resolver {- self.resolver_with_limits(Duration::from_secs(2), 256 * 1024)+ fn control_plane(&self) -> ControlPlaneClient {+ self.control_plane_with_limits(Duration::from_secs(2), 256 * 1024)}- fn resolver_with_limits(&self, timeout: Duration, max_response_bytes: usize) -> Resolver {- let mut config = ResolverConfig::new(&self.url, SERVICE_KEY).unwrap();- config.limits = Limits {+ fn control_plane_with_limits(+ &self,+ timeout: Duration,+ max_response_bytes: usize,+ ) -> ControlPlaneClient {+ let mut config = ControlPlaneConfig::new(&self.url, SERVICE_KEY).unwrap();+ config.limits = ResolutionLimits {timeout,max_response_bytes,};- Resolver::new(config).unwrap()+ ControlPlaneClient::new(config).unwrap()}fn calls(&self) -> usize {@@ -120,7 +124,7 @@ async fn signs_the_exact_key_and_sends_only_the_api_format() {}).await;let context = web- .resolver()+ .control_plane().resolve_at(GATEWAY_KEY, ApiFormat::OpenAiResponses, NOW_TIME).await.unwrap();@@ -165,15 +169,15 @@ async fn denied_and_failed_resolutions_are_classified_without_retries_or_body_le}).await;let error = web- .resolver()+ .control_plane().resolve_at(GATEWAY_KEY, ApiFormat::OpenAiResponses, NOW_TIME).await.unwrap_err();assert!(match status {- 401 => matches!(error, ResolveError::Authentication),- 403 => matches!(error, ResolveError::Forbidden),- 404 => matches!(error, ResolveError::NoRoute),- _ => matches!(error, ResolveError::Unavailable),+ 401 => matches!(error, ResolutionError::Authentication),+ 403 => matches!(error, ResolutionError::Forbidden),+ 404 => matches!(error, ResolutionError::NoRoute),+ _ => matches!(error, ResolutionError::Unavailable),});assert!(!format!("{error:?} {error}").contains("private-upstream-error"));assert_eq!(web.calls(), 1);@@ -197,7 +201,7 @@ async fn never_follows_redirects_with_credentials() {}).await;assert!(- web.resolver()+ web.control_plane().resolve_at(GATEWAY_KEY, ApiFormat::OpenAiResponses, NOW_TIME).await.is_err()@@ -223,10 +227,10 @@ async fn rejects_oversized_declared_and_chunked_responses() {}).await;let result = web- .resolver_with_limits(Duration::from_secs(2), 128)+ .control_plane_with_limits(Duration::from_secs(2), 128).resolve_at(GATEWAY_KEY, ApiFormat::OpenAiResponses, NOW_TIME).await;- assert!(matches!(result, Err(ResolveError::ResponseTooLarge)));+ assert!(matches!(result, Err(ResolutionError::ResponseTooLarge)));assert_eq!(web.calls(), 1);}}@@ -250,12 +254,12 @@ async fn deadline_covers_response_headers_and_body() {.await;let result = tokio::time::timeout(Duration::from_millis(500),- web.resolver_with_limits(Duration::from_millis(30), 256 * 1024)+ web.control_plane_with_limits(Duration::from_millis(30), 256 * 1024).resolve_at(GATEWAY_KEY, ApiFormat::OpenAiResponses, NOW_TIME),).await.expect("resolver must enforce its deadline");- assert!(matches!(result, Err(ResolveError::Timeout)));+ assert!(matches!(result, Err(ResolutionError::Timeout)));assert_eq!(web.calls(), 1);}}@@ -265,10 +269,10 @@ async fn malformed_credentials_never_reach_web() {let web = FakeWeb::start(|_| async { response(success("project-1", "provider-token")) }).await;for key in ["", "has space", "has\r\nheader", "has\ttab"] {let result = web- .resolver()+ .control_plane().resolve_at(key, ApiFormat::OpenAiResponses, NOW_TIME).await;- assert!(matches!(result, Err(ResolveError::InvalidCredential)));+ assert!(matches!(result, Err(ResolutionError::InvalidCredential)));}assert_eq!(web.calls(), 0);}@@ -283,14 +287,14 @@ async fn rejects_a_grant_that_expires_while_resolving_across_a_second_boundary()}).await;let result = web- .resolver()+ .control_plane().resolve_at(GATEWAY_KEY,ApiFormat::OpenAiResponses,NOW_TIME + Duration::from_millis(900),).await;- assert!(matches!(result, Err(ResolveError::InvalidResponse)));+ assert!(matches!(result, Err(ResolutionError::InvalidResponse)));}#[tokio::test]@@ -307,10 +311,10 @@ async fn concurrent_resolutions_keep_credentials_and_contexts_isolated() {}}).await;- let resolver = web.resolver();+ let control_plane = web.control_plane();let (alice, bob) = tokio::join!(- resolver.resolve_at("gw_test_alice", ApiFormat::OpenAiResponses, NOW_TIME),- resolver.resolve_at("gw_test_bob", ApiFormat::OpenAiResponses, NOW_TIME),+ control_plane.resolve_at("gw_test_alice", ApiFormat::OpenAiResponses, NOW_TIME),+ control_plane.resolve_at("gw_test_bob", ApiFormat::OpenAiResponses, NOW_TIME),);let alice = alice.unwrap();let bob = bob.unwrap();@@ -386,10 +390,10 @@ async fn assert_invalid_context(body: Value) {}).await;let result = web- .resolver()+ .control_plane().resolve_at(GATEWAY_KEY, ApiFormat::OpenAiResponses, NOW_TIME).await;- assert!(matches!(result, Err(ResolveError::InvalidResponse)));+ assert!(matches!(result, Err(ResolutionError::InvalidResponse)));assert_eq!(web.calls(), 1);}@@ -400,11 +404,11 @@ async fn rejects_malformed_json_without_exposing_response_contents() {}).await;let error = web- .resolver()+ .control_plane().resolve_at(GATEWAY_KEY, ApiFormat::OpenAiResponses, NOW_TIME).await.unwrap_err();- assert!(matches!(error, ResolveError::InvalidResponse));+ assert!(matches!(error, ResolutionError::InvalidResponse));assert!(!format!("{error:?} {error}").contains("private-provider-secret"));}@@ -412,8 +416,8 @@ async fn rejects_malformed_json_without_exposing_response_contents() {fn web_configuration_rejects_unsafe_or_ambiguous_urls() {for key in ["", " \n\t"] {assert!(matches!(- ResolverConfig::new("https://web.example", key),- Err(ResolveError::Configuration)+ ControlPlaneConfig::new("https://web.example", key),+ Err(ResolutionError::Configuration)));}for url in [@@ -424,8 +428,8 @@ fn web_configuration_rejects_unsafe_or_ambiguous_urls() {"https://web.example#fragment",] {assert!(matches!(- ResolverConfig::new(url, SERVICE_KEY),- Err(ResolveError::Configuration)+ ControlPlaneConfig::new(url, SERVICE_KEY),+ Err(ResolutionError::Configuration)));}for url in [@@ -433,6 +437,6 @@ fn web_configuration_rejects_unsafe_or_ambiguous_urls() {"http://127.0.0.1:3000","http://[::1]:3000",] {- assert!(ResolverConfig::new(url, SERVICE_KEY).is_ok());+ assert!(ControlPlaneConfig::new(url, SERVICE_KEY).is_ok());}}diff --git a/ai-gateway/src/server.rs b/ai-gateway/src/server.rsindex d0cd55e7be..e0a65e9d89 100644--- a/ai-gateway/src/server.rs+++ b/ai-gateway/src/server.rs@@ -13,13 +13,22 @@ use serde::Serialize;use tokio::{net::TcpListener, sync::oneshot};#[derive(Clone, Default)]-pub struct AppState {+pub struct GatewayLifecycleState {ready: Arc<AtomicBool>,+ disabled: bool,}-impl AppState {+impl GatewayLifecycleState {pub fn is_ready(&self) -> bool {- self.ready.load(Ordering::Acquire)+ !self.disabled && self.ready.load(Ordering::Acquire)+ }++ /// Keep liveness available without advertising inference readiness.+ pub fn unconfigured() -> Self {+ Self {+ disabled: true,+ ..Self::default()+ }}}@@ -28,20 +37,26 @@ struct Probe {status: &'static str,}-pub fn router(state: AppState) -> Router {+pub fn router(state: GatewayLifecycleState) -> Router {Router::new().route("/health", get(|| async { Json(Probe { status: "ok" }) })).route("/ready", get(readiness)).with_state(state)}-async fn readiness(State(state): State<AppState>) -> (StatusCode, Json<Probe>) {+async fn readiness(State(state): State<GatewayLifecycleState>) -> (StatusCode, Json<Probe>) {if state.is_ready() {(StatusCode::OK, Json(Probe { status: "ready" }))} else {(StatusCode::SERVICE_UNAVAILABLE,- Json(Probe { status: "draining" }),+ Json(Probe {+ status: if state.disabled {+ "unconfigured"+ } else {+ "draining"+ },+ }),)}}@@ -55,7 +70,7 @@ async fn readiness(State(state): State<AppState>) -> (StatusCode, Json<Probe>) {pub async fn serve(listener: TcpListener,app: Router,- state: AppState,+ state: GatewayLifecycleState,shutdown: impl Future<Output = ()> + Send + 'static,shutdown_timeout: Duration,) -> io::Result<()> {diff --git a/ai-gateway/src/test_support.rs b/ai-gateway/src/test_support.rsnew file mode 100644index 0000000000..a9541d26f3--- /dev/null+++ b/ai-gateway/src/test_support.rs@@ -0,0 +1,102 @@+use std::{+ future::Future,+ sync::{+ Arc,+ atomic::{AtomicUsize, Ordering},+ },+ time::{SystemTime, UNIX_EPOCH},+};++use axum::{+ Router,+ body::Body,+ http::{Request, Response},+ routing::any,+};+use serde_json::json;+use tokio::{net::TcpListener, task::JoinHandle};++use crate::resolution::{+ ApiFormat, ControlPlaneClient, ControlPlaneConfig, ResolvedRequestContext,+};++pub(crate) struct FakeServer {+ pub url: String,+ calls: Arc<AtomicUsize>,+ task: JoinHandle<()>,+}++impl FakeServer {+ pub async fn start<F, Fut>(handler: F) -> Self+ where+ F: Fn(Request<Body>) -> Fut + Clone + Send + Sync + 'static,+ Fut: Future<Output = Response<Body>> + Send + 'static,+ {+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();+ let url = format!("http://{}", listener.local_addr().unwrap());+ let calls = Arc::new(AtomicUsize::new(0));+ let count = calls.clone();+ let app = Router::new().fallback(any(move |request| {+ count.fetch_add(1, Ordering::SeqCst);+ handler(request)+ }));+ let task = tokio::spawn(async move {+ axum::serve(listener, app).await.unwrap();+ });+ Self { url, calls, task }+ }++ pub fn calls(&self) -> usize {+ self.calls.load(Ordering::SeqCst)+ }++ pub fn control_plane(&self) -> ControlPlaneClient {+ ControlPlaneClient::new(ControlPlaneConfig::new(&self.url, "test-service-key").unwrap())+ .unwrap()+ }+}++impl Drop for FakeServer {+ fn drop(&mut self) {+ self.task.abort();+ }+}++pub(crate) fn resolution_response(provider_token: &str) -> Response<Body> {+ let expires_at = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .unwrap()+ .as_secs()+ + 300;+ let body = json!({+ "version": 1,+ "connection": {+ "id": "connection-1", "provider": "openai", "api_format": "openai.responses",+ "base_url": "https://api.openai.com/v1",+ "auth": {"type": "Bearer", "token": provider_token}+ },+ "attribution": {+ "organization_id": "org-1", "project_id": "project-1", "key_id": "key-1", "key_metadata": {},+ "provider_connection_id": "provider-connection-1"+ },+ "ingestion_mode": "usage",+ "ingestion": {"access_token": "private-ingestion-token", "token_type": "Bearer", "expires_at": expires_at}+ });+ Response::builder()+ .header("content-type", "application/json")+ .body(Body::from(body.to_string()))+ .unwrap()+}++pub(crate) async fn resolved_request_context(provider_token: &str) -> ResolvedRequestContext {+ let token = provider_token.to_owned();+ let web = FakeServer::start(move |_| {+ let token = token.clone();+ async move { resolution_response(&token) }+ })+ .await;+ web.control_plane()+ .resolve("gateway-secret", ApiFormat::OpenAiResponses)+ .await+ .unwrap()+}diff --git a/ai-gateway/src/transport/mod.rs b/ai-gateway/src/transport/mod.rsnew file mode 100644index 0000000000..33cc6614ea--- /dev/null+++ b/ai-gateway/src/transport/mod.rs@@ -0,0 +1,282 @@+//! Bounded byte relay. One task owns upstream reads; the body owns its lifetime.+use std::{+ fmt,+ pin::Pin,+ sync::{+ Arc, Mutex,+ atomic::{AtomicBool, Ordering},+ },+ task::{Context, Poll},+};++use axum::{+ body::{Body, Bytes},+ http::{HeaderMap, header},+};+use futures_util::{Stream, StreamExt};+use tokio::{+ sync::{Notify, mpsc},+ task::JoinHandle,+ time::Instant,+};++/// Sanitized failures; upstream URLs, bodies and credentials are never retained.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum ProviderError {+ Configuration,+ Busy,+ Timeout,+ Transport,+}++impl fmt::Display for ProviderError {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.write_str(match self {+ Self::Configuration => "invalid provider configuration",+ Self::Busy => "gateway execution capacity exhausted",+ Self::Timeout => "provider deadline exceeded",+ Self::Transport => "provider transport failed",+ })+ }+}+impl std::error::Error for ProviderError {}++// An allowlist prevents gateway credentials, tenant routing overrides and cookies+// from crossing the boundary. Headers nominated by `Connection` are never end-to-end.+fn selected_headers(source: &HeaderMap, allowed: &[&'static str]) -> HeaderMap {+ let mut selected = HeaderMap::new();+ for &name in allowed {+ let hop_by_hop = source.get_all(header::CONNECTION).iter().any(|value| {+ value.to_str().map_or(true, |value| {+ value+ .split(',')+ .any(|token| token.trim().eq_ignore_ascii_case(name))+ })+ });+ if !hop_by_hop {+ for value in source.get_all(name) {+ selected.append(header::HeaderName::from_static(name), value.clone());+ }+ }+ }+ selected+}++pub(crate) fn request_headers(source: &HeaderMap) -> HeaderMap {+ selected_headers(+ source,+ &[+ "content-type",+ "content-encoding",+ "accept",+ "accept-encoding",+ ],+ )+}++pub(crate) fn response_headers(source: &HeaderMap) -> HeaderMap {+ selected_headers(+ source,+ &[+ "content-type",+ "content-encoding",+ "cache-control",+ "retry-after",+ "x-request-id",+ "openai-processing-ms",+ "openai-version",+ "x-ratelimit-limit-requests",+ "x-ratelimit-limit-tokens",+ "x-ratelimit-remaining-requests",+ "x-ratelimit-remaining-tokens",+ "x-ratelimit-reset-requests",+ "x-ratelimit-reset-tokens",+ ],+ )+}++pub(crate) fn relay<T: Send + 'static>(+ upstream: reqwest::Response,+ deadline: Instant,+ owner: T,+) -> Body {+ relay_stream(+ upstream+ .bytes_stream()+ .map(|chunk| chunk.map_err(|_| ProviderError::Transport)),+ deadline,+ owner,+ )+}++fn relay_stream<S, T>(upstream: S, deadline: Instant, owner: T) -> Body+where+ S: Stream<Item = Result<Bytes, ProviderError>> + Send + 'static,+ T: Send + 'static,+{+ let (sender, receiver) = mpsc::channel(1);+ let failed = Arc::new(AtomicBool::new(false));+ let failure = failed.clone();+ let resources = Arc::new(StreamResources {+ owner: Mutex::new(Some(owner)),+ released: Notify::new(),+ });+ let pump_resources = resources.clone();+ let task = tokio::spawn(async move {+ let pump = async {+ tokio::pin!(upstream);+ while let Some(chunk) = upstream.next().await {+ let chunk = chunk?;+ // One queued chunk plus one pending send; no per-chunk tasks.+ for bytes in chunk.chunks(64 * 1024) {+ if sender.send(Bytes::copy_from_slice(bytes)).await.is_err() {+ return Ok(());+ }+ }+ }+ Ok::<_, ProviderError>(())+ };+ if !matches!(tokio::time::timeout_at(deadline, pump).await, Ok(Ok(()))) {+ failure.store(true, Ordering::Release);+ pump_resources.release();+ }+ drop(sender);+ // Upstream EOF alone does not release admission: the downstream may still+ // have queued bytes. The deadline also covers an unpolled downstream body.+ if tokio::time::timeout_at(deadline, pump_resources.released.notified())+ .await+ .is_err()+ {+ failure.store(true, Ordering::Release);+ pump_resources.release();+ }+ });+ Body::from_stream(ResponseStream {+ receiver,+ task,+ failed,+ resources,+ done: false,+ })+}++struct StreamResources<T> {+ owner: Mutex<Option<T>>,+ released: Notify,+}++impl<T> StreamResources<T> {+ fn release(&self) {+ self.owner.lock().expect("relay owner lock poisoned").take();+ self.released.notify_one();+ }+}++struct ResponseStream<T> {+ receiver: mpsc::Receiver<Bytes>,+ task: JoinHandle<()>,+ failed: Arc<AtomicBool>,+ resources: Arc<StreamResources<T>>,+ done: bool,+}++impl<T> Stream for ResponseStream<T> {+ type Item = Result<Bytes, ProviderError>;++ fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {+ let this = self.get_mut();+ if this.done {+ return Poll::Ready(None);+ }+ match this.receiver.poll_recv(cx) {+ Poll::Ready(None) => {+ this.done = true;+ this.resources.release();+ // The sender publishes a failure before closing its channel.+ Poll::Ready(+ this.failed+ .load(Ordering::Acquire)+ .then_some(Err(ProviderError::Transport)),+ )+ }+ other => other.map(|value| value.map(Ok)),+ }+ }+}++impl<T> Drop for ResponseStream<T> {+ fn drop(&mut self) {+ self.task.abort();+ self.resources.release();+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use axum::body::to_bytes;+ use futures_util::stream;+ use std::{sync::atomic::AtomicUsize, time::Duration};++ struct Owner(Arc<Notify>);+ impl Drop for Owner {+ fn drop(&mut self) {+ self.0.notify_one();+ }+ }++ #[tokio::test]+ async fn slow_downstream_bounds_read_ahead_and_drop_releases_owner() {+ let polled = Arc::new(AtomicUsize::new(0));+ let count = polled.clone();+ let upstream = stream::iter(0..100).map(move |_| {+ count.fetch_add(1, Ordering::SeqCst);+ Ok(Bytes::from(vec![b'x'; 128 * 1024]))+ });+ let released = Arc::new(Notify::new());+ let body = relay_stream(+ upstream,+ Instant::now() + Duration::from_secs(10),+ Owner(released.clone()),+ );+ tokio::task::yield_now().await;+ assert_eq!(polled.load(Ordering::SeqCst), 1);+ drop(body);+ tokio::time::timeout(Duration::from_secs(1), released.notified())+ .await+ .unwrap();+ }++ #[tokio::test]+ async fn queued_bytes_are_delivered_before_an_upstream_error() {+ let released = Arc::new(Notify::new());+ let bytes = Bytes::from_static(b"event: response.output_text.delta\ndata: hello\n\n");+ let body = relay_stream(+ stream::iter([Ok(bytes.clone()), Err(ProviderError::Transport)]),+ Instant::now() + Duration::from_secs(1),+ Owner(released.clone()),+ );+ // Wait until the pump has published its failure with the successful chunk queued.+ tokio::time::timeout(Duration::from_secs(1), released.notified())+ .await+ .unwrap();+ let mut downstream = body.into_data_stream();+ assert_eq!(downstream.next().await.unwrap().unwrap(), bytes);+ assert!(downstream.next().await.unwrap().is_err());+ assert!(downstream.next().await.is_none());+ }++ #[tokio::test]+ async fn deadline_releases_owner_even_after_upstream_eof_with_unpolled_body() {+ let released = Arc::new(Notify::new());+ let body = relay_stream(+ stream::once(async { Ok(Bytes::from_static(b"final bytes")) }),+ Instant::now() + Duration::from_millis(30),+ Owner(released.clone()),+ );+ tokio::time::timeout(Duration::from_secs(1), released.notified())+ .await+ .unwrap();+ assert!(to_bytes(body, 1024).await.is_err());+ }+}diff --git a/ai-gateway/tests/lifecycle.rs b/ai-gateway/tests/lifecycle.rsindex 613d92eac0..b7f12784c9 100644--- a/ai-gateway/tests/lifecycle.rs+++ b/ai-gateway/tests/lifecycle.rs@@ -1,6 +1,6 @@mod support;-use ai_gateway::server::{self, AppState};+use ai_gateway::server::{self, GatewayLifecycleState};use axum::{Router,body::{Body, to_bytes},@@ -17,7 +17,7 @@ use tower::ServiceExt;#[tokio::test]async fn probes_and_unimplemented_routes_over_http() {- let state = AppState::default();+ let state = GatewayLifecycleState::default();let mut server =TestServer::start(server::router(state.clone()), state, Duration::from_secs(1)).await;let health = support::get(server.address, "/health").await;@@ -43,7 +43,7 @@ async fn probes_and_unimplemented_routes_over_http() {async fn shutdown_marks_unready_and_waits_for_in_flight_request() {let entered = Arc::new(Notify::new());let release = Arc::new(Notify::new());- let state = AppState::default();+ let state = GatewayLifecycleState::default();let app = server::router(state.clone()).route("/slow",get({@@ -106,7 +106,12 @@ async fn shutdown_deadline_bounds_a_stuck_handler() {}}),);- let mut server = TestServer::start(app, AppState::default(), Duration::from_millis(50)).await;+ let mut server = TestServer::start(+ app,+ GatewayLifecycleState::default(),+ Duration::from_millis(50),+ )+ .await;let request = tokio::spawn(support::get(server.address, "/stuck"));tokio::time::timeout(Duration::from_secs(2), entered.notified()).await@@ -139,7 +144,12 @@ async fn fake_dependency_records_requests_and_returns_scripted_response() {}}});- let mut fake = TestServer::start(app, AppState::default(), Duration::from_secs(1)).await;+ let mut fake = TestServer::start(+ app,+ GatewayLifecycleState::default(),+ Duration::from_secs(1),+ )+ .await;let response = support::get(fake.address, "/example").await;assert!(response.starts_with("HTTP/1.1 429"));assert!(response.contains("retry-after: 1\r\n"));diff --git a/ai-gateway/tests/startup.rs b/ai-gateway/tests/startup.rsindex 6aee192d52..ba9ff20057 100644--- a/ai-gateway/tests/startup.rs+++ b/ai-gateway/tests/startup.rs@@ -9,6 +9,9 @@ fn invalid_configuration_exits_without_disclosing_the_value() {"LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS","LANGFUSE_LOG_LEVEL","LANGFUSE_LOG_FORMAT",+ "LANGFUSE_AI_GATEWAY_WEB_URL",+ "LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS",+ "LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS",] {let output = Command::new(env!("CARGO_BIN_EXE_ai-gateway")).env_clear()diff --git a/ai-gateway/tests/support/mod.rs b/ai-gateway/tests/support/mod.rsindex 00ebf2d74f..8180ebf130 100644--- a/ai-gateway/tests/support/mod.rs+++ b/ai-gateway/tests/support/mod.rs@@ -1,6 +1,6 @@use std::{io, net::SocketAddr, time::Duration};-use ai_gateway::server::{self, AppState};+use ai_gateway::server::{self, GatewayLifecycleState};use axum::Router;use tokio::{io::{AsyncReadExt, AsyncWriteExt},@@ -11,13 +11,13 @@ use tokio::{pub struct TestServer {pub address: SocketAddr,- pub state: AppState,+ pub state: GatewayLifecycleState,shutdown: Option<oneshot::Sender<()>>,task: Option<JoinHandle<io::Result<()>>>,}impl TestServer {- pub async fn start(app: Router, state: AppState, budget: Duration) -> Self {+ pub async fn start(app: Router, state: GatewayLifecycleState, budget: Duration) -> Self {let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();let address = listener.local_addr().unwrap();let (shutdown, receiver) = oneshot::channel();