Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): relay authenticated OpenAI Responses requests (#17352)

Hassieb Pakzad45f520827a2d
diff --git a/.env.dev.example b/.env.dev.example
index 101ee270b1..1087c02f1f 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -159,6 +159,13 @@ REDIS_AUTH="myredissecret"
# openssl rand -hex 32 used only here
ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000
+# Rust gateway: set the Web base URL to enable inference; include any Web base path.
+# The service key below must match Web. Without this URL, readiness/inference return 503.
+# LANGFUSE_AI_GATEWAY_WEB_URL=http://localhost:3000
+# Independent per-instance concurrency limits; provisional defaults, tune with load tests.
+# LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS=128
+# LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS=128
+
# AI gateway control-plane key rotation. Generate service keys with
# `openssl rand -hex 32` and ES256 PEM keys with
# `openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256`.
diff --git a/.env.prod.example b/.env.prod.example
index 826b246598..9400f5d93d 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -496,3 +496,12 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
# CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET=
# CHB's resource-server identifier; defaults to billing-api.
# CLICKHOUSE_BILLING_AUTH0_AUDIENCE=
+
+# Rust AI gateway. Use the trusted HTTPS Web URL including any NEXT_PUBLIC_BASE_PATH.
+# Without a Web URL the gateway starts, but readiness and inference return 503.
+# LANGFUSE_AI_GATEWAY_WEB_URL=https://langfuse.example.com/app
+# Same shared service key configured in Web. Provider keys are resolved through Web.
+# LANGFUSE_AI_GATEWAY_SERVICE_KEY=
+# Independent per-instance concurrency limits; provisional defaults, tune with load tests.
+# LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS=128
+# LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS=128
diff --git a/ai-gateway/Cargo.lock b/ai-gateway/Cargo.lock
index 99225e5df4..de2918a800 100644
--- a/ai-gateway/Cargo.lock
+++ b/ai-gateway/Cargo.lock
@@ -9,6 +9,7 @@ dependencies = [
"axum",
"futures-util",
"hmac",
+ "http-body-util",
"reqwest",
"serde",
"serde_json",
@@ -257,6 +258,12 @@ version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
+[[package]]
+name = "futures-io"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed"
+
[[package]]
name = "futures-macro"
version = "0.3.34"
@@ -268,6 +275,12 @@ dependencies = [
"syn 3.0.5",
]
+[[package]]
+name = "futures-sink"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d"
+
[[package]]
name = "futures-task"
version = "0.3.34"
@@ -281,8 +294,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
dependencies = [
"futures-core",
+ "futures-io",
"futures-macro",
+ "futures-sink",
"futures-task",
+ "memchr",
"pin-project-lite",
"slab",
]
@@ -771,6 +787,7 @@ dependencies = [
"base64",
"bytes",
"futures-core",
+ "futures-util",
"http",
"http-body",
"http-body-util",
@@ -791,12 +808,14 @@ dependencies = [
"sync_wrapper",
"tokio",
"tokio-rustls",
+ "tokio-util",
"tower",
"tower-http",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
+ "wasm-streams",
"web-sys",
]
@@ -1180,6 +1199,19 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "tokio-util"
+version = "0.7.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "futures-sink",
+ "pin-project-lite",
+ "tokio",
+]
+
[[package]]
name = "tower"
version = "0.5.3"
@@ -1399,6 +1431,19 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "wasm-streams"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
+dependencies = [
+ "futures-util",
+ "js-sys",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
[[package]]
name = "web-sys"
version = "0.3.105"
diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.toml
index f737e62162..316aa4f509 100644
--- a/ai-gateway/Cargo.toml
+++ b/ai-gateway/Cargo.toml
@@ -8,7 +8,9 @@ license = "MIT"
[dependencies]
axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "tokio"] }
hmac = "0.13"
-reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls-native-roots"] }
+http-body-util = "0.1"
+futures-util = "0.3"
+reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls-native-roots", "stream"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
sha2 = "0.11"
@@ -17,7 +19,6 @@ tracing = "0.1.44"
tracing-subscriber = { version = "0.3.23", default-features = false, features = ["fmt", "json"] }
[dev-dependencies]
-futures-util = "0.3"
tokio = { version = "1.53.1", features = ["io-util"] }
tower = { version = "0.5", features = ["util"] }
diff --git a/ai-gateway/README.md b/ai-gateway/README.md
index 6deb790587..f847ed3dab 100644
--- a/ai-gateway/README.md
+++ b/ai-gateway/README.md
@@ -1,9 +1,9 @@
-# AI Gateway service foundation
+# AI Gateway service
-A standalone Rust HTTP process with health probes, bounded shutdown, structured
-logs and a container, with a tested Web resolution client library. Inference and
-customer telemetry are not implemented yet; inference paths return 404. No Web, database or provider
-credentials are needed to build, start or test this package.
+A standalone Rust gateway for `POST /openai/v1/responses`. Web resolves the gateway
+key to a trusted provider connection; Rust relays native JSON or SSE without
+parsing the request or rewriting provider bytes. Customer telemetry is not exported
+yet. Building and testing need no real Web, database or provider credentials.
## Run locally
@@ -27,7 +27,8 @@ keep their built-in watchers. The gateway loads the root `.env` and listens on
the first available port starting at 8080. This lets multiple worktrees run
their development stacks at the same time. Add overrides from the table below
to your root `.env`; exported shell variables take precedence.
-No gateway-specific credentials are needed for this foundation slice.
+Without a Web URL, the process starts with liveness available; readiness and
+inference return 503. To enable inference, configure the Web URL and shared service key.
To work on only the gateway, use `pnpm dev --filter=ai-gateway`.
Both commands watch Rust source changes; do not run both on the same port.
@@ -48,21 +49,25 @@ In another terminal:
```sh
curl --fail http://localhost:8080/health
# {"status":"ok"}
-curl --fail http://localhost:8080/ready
-# {"status":"ready"}
+curl http://localhost:8080/ready
+# 503 without configuration; 200 {"status":"ready"} when configured
```
The binary reads configuration from the process environment. The pnpm development
script loads the root `.env` before starting it; production and direct Cargo runs
-do not load dotenv files. All settings have defaults:
-
-| Variable | Default | Validation |
-| ------------------------------------------------ | -------------- | -------------------------------------------------- |
-| `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS` | `0.0.0.0:8080` | IP address and port; IPv6 uses `[::]:8080` |
-| `LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT` | `false` | `true`, `false`; the pnpm dev task sets `true` |
-| `LANGFUSE_LOG_FORMAT` | `text` | `text`, `json` |
-| `LANGFUSE_LOG_LEVEL` | `info` | `trace`, `debug`, `info`, `warn`, `error`, `fatal` |
-| `LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS` | `10` | Integer from 1 to 300 |
+do not load dotenv files:
+
+| Variable | Default | Validation |
+| ---------------------------------------------- | -------------- | -------------------------------------------------- |
+| `LANGFUSE_AI_GATEWAY_WEB_URL` | unset; inference disabled | Trusted Web base URL, including any deployment prefix |
+| `LANGFUSE_AI_GATEWAY_SERVICE_KEY` | unset | Required with Web URL; same key configured in Web |
+| `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS` | `0.0.0.0:8080` | IP address and port; IPv6 uses `[::]:8080` |
+| `LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT` | `false` | `true`, `false`; the pnpm dev task sets `true` |
+| `LANGFUSE_LOG_FORMAT` | `text` | `text`, `json` |
+| `LANGFUSE_LOG_LEVEL` | `info` | `trace`, `debug`, `info`, `warn`, `error`, `fatal` |
+| `LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS` | `10` | Integer from 1 to 300 |
+| `LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS` | `128` | Positive integer up to Tokio's semaphore capacity; authenticated requests per instance |
+| `LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS` | `128` | Positive integer up to Tokio's semaphore capacity; concurrent Web resolutions per instance |
The gateway shares `LANGFUSE_LOG_LEVEL` with Web and worker. Values are lowercase;
`fatal` maps to Rust's `error` level and therefore includes ordinary error logs.
@@ -79,17 +84,86 @@ Invalid values fail startup without echoing their contents. Logs contain service
lifecycle events, not request bodies or headers. For direct
host-only development, set `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS=127.0.0.1:8080`.
+## Call the gateway
+
+Set these in the repository root `.env` for `pnpm dev` (or export them for Cargo):
+
+```dotenv
+LANGFUSE_AI_GATEWAY_WEB_URL=http://localhost:3000
+LANGFUSE_AI_GATEWAY_SERVICE_KEY=<same-service-key-as-web>
+```
+
+For production use the HTTPS Web URL; include `/app` when Web was built with that
+base path. Web also needs its existing ingestion JWT signer configured, an enabled
+gateway key, and an eligible OpenAI connection. Rust needs neither the JWT signing
+key nor an operator-supplied OpenAI key: provider credentials come from resolution.
+A service key alone does not enable inference without the Web URL.
+
+```sh
+curl http://localhost:8080/openai/v1/responses \
+ -H "Authorization: Bearer $LANGFUSE_GATEWAY_KEY" \
+ -H 'Content-Type: application/json' \
+ -d '{"model":"gpt-4.1-mini","input":"Say hello"}'
+# Add "stream":true to the JSON and use curl -N for SSE.
+```
+
+```python
+from openai import OpenAI
+import os
+
+client = OpenAI(
+ base_url="http://localhost:8080/openai/v1",
+ api_key=os.environ["LANGFUSE_GATEWAY_KEY"],
+ max_retries=0,
+)
+response = client.responses.create(model="gpt-4.1-mini", input="Say hello")
+print(response.output_text)
+```
+
+The gateway forwards to the official OpenAI Responses endpoint only. It does not
+retry, follow redirects, parse `model`/`stream`, or accept client routing overrides.
+Provider errors retain their status and body. Gateway errors use an OpenAI-style
+`{"error":{"message":"...","type":"...","param":null,"code":"..."}}` envelope.
+
+The gateway authenticates through Web before reserving execution capacity or
+reading the request body. Web resolution uses a separate concurrency budget,
+released after resolution finishes; authenticated requests then reserve execution
+capacity through body reading and response completion. Either budget returns 503
+immediately when full, without a waiting queue. Both defaults of 128 are provisional
+guardrails, not measured capacity: tune them independently using load tests for the
+instance resources, request sizes and stream durations. These limits bound work;
+they do not guarantee fairness between clients or tenants.
+
+Other limits are 4 MiB request bodies, 10 seconds to read a request, 5 seconds to
+connect, 120 seconds for provider response headers or an individual upstream read,
+and 600 seconds overall from execution admission. Response size is not capped: a single task pumps chunks
+through a one-slot channel, with chunks at most 64 KiB. It stops reading when that
+channel fills. Completion, disconnect and deadline release admission and context;
+the deadline runs even when the downstream stops polling. A failure after headers
+terminates the response body without adding an error event or retrying.
+There is no separate downstream stall timeout; a client that stops reading can
+retain execution capacity until the overall deadline. A progress-based downstream
+stall policy is deferred to a separate change.
+
+Only request content type/encoding and accept/accept-encoding cross the provider
+boundary, plus the resolved Bearer token. Response content type/encoding, cache
+control, retry-after, request ID and selected OpenAI timing/version/rate-limit
+headers are retained. Cookies, routing overrides, gateway/ingestion credentials,
+hop-by-hop headers and upstream framing are excluded.
+
## Process lifecycle
- `/health` returns 200 while HTTP is serving. It never probes dependencies.
-- `/ready` returns 200 after initialization. On shutdown, readiness changes to
+- `/ready` returns 200 after initialization with inference configured. It does not
+ probe Web or OpenAI; dependency failures are reported per request. Without a Web
+ URL it returns 503. On shutdown, readiness changes to
503 with `{"status":"draining"}` and the listener stops accepting connections.
An external probe may see a closed connection instead of the brief 503 state.
- SIGTERM or Ctrl-C starts graceful shutdown. In-flight requests may finish
within the configured timeout. Deadline expiry exits nonzero and terminates
remaining runtime tasks; ordinary shutdown exits zero.
-- The deadline starts on the shutdown signal, not at process startup. Future
- stream/export slices must integrate their own work into this lifecycle.
+- The shutdown deadline starts on the shutdown signal. New inference requests
+ are rejected while draining; existing response streams share this budget.
## Container
@@ -108,29 +182,29 @@ The runtime image runs as UID/GID 10001, includes CA certificates, and executes
the binary directly so it receives signals. Set the container/orchestrator stop
grace period longer than the gateway shutdown timeout. The smoke test uses an
isolated container and an ephemeral host port, checks read-only/non-root startup,
-probes, missing inference routes and a clean SIGTERM exit, then removes it.
+probes, rejection of unauthenticated inference and a clean SIGTERM exit, then removes it.
## Tests and module boundaries
### Web resolution client
-`resolution::Resolver` resolves a gateway key through an operator-configured Web
-base URL. It is a library boundary; the binary does not initialize it yet. The
+`resolution::ControlPlaneClient` resolves a gateway key through an operator-configured Web
+base URL. The binary initializes it when a Web URL is configured. The
caller supplies the existing `LANGFUSE_AI_GATEWAY_SERVICE_KEY` and a trusted Web
-base URL to `ResolverConfig::new`. HTTPS is required except on loopback for local
+base URL to `ControlPlaneConfig::new`. HTTPS is required except on loopback for local
development. URLs cannot contain credentials, a query or fragment. Include the
Web deployment's `NEXT_PUBLIC_BASE_PATH`, if set: both `https://host/app` and
`https://host/app/` resolve through `/app/api/internal/ai-gateway/v1/resolve`.
```rust,no_run
-use ai_gateway::resolution::{ApiFormat, ResolveError, Resolver, ResolverConfig};
+use ai_gateway::resolution::{ApiFormat, ResolutionError, ControlPlaneClient, ControlPlaneConfig};
async fn example(web_base_url: &str, service_key: &str, gateway_key: &str)
- -> Result<(), ResolveError>
+ -> Result<(), ResolutionError>
{
- let resolver = Resolver::new(ResolverConfig::new(web_base_url, service_key)?)?;
- let execution = resolver.resolve(gateway_key, ApiFormat::OpenAiResponses).await?;
- assert_eq!(execution.connection().base_url(), "https://api.openai.com/v1");
+ let control_plane = ControlPlaneClient::new(ControlPlaneConfig::new(web_base_url, service_key)?)?;
+ let context = control_plane.resolve(gateway_key, ApiFormat::OpenAiResponses).await?;
+ assert_eq!(context.connection().base_url(), "https://api.openai.com/v1");
Ok(())
}
```
@@ -181,6 +255,10 @@ cargo test --locked
- `server.rs`: router, probe state and bounded graceful shutdown. A listener and
shutdown future are injected, so tests do not depend on fixed ports or signals.
- `main.rs`: configuration, logging, signal registration and process exit.
+- `http.rs`: credential extraction, bounded body reads and gateway error envelopes.
+- `inference.rs`: `InferenceService` coordinates resolution, admission and forwarding.
+- `providers/openai.rs`: fixed destination, provider credentials and admission.
+- `transport/mod.rs`: header allowlists, bounded byte relay and response lifetime.
- `resolution/mod.rs`: trusted base URL configuration and bounded Web HTTP client.
- `resolution/contracts.rs`: strict Web response validation and immutable execution context.
- `resolution/signing.rs`: Web v1 HMAC; a literal shared fixture pins byte compatibility.
diff --git a/ai-gateway/scripts/smoke-image.sh b/ai-gateway/scripts/smoke-image.sh
index 57c10d31ba..70c740b016 100644
--- a/ai-gateway/scripts/smoke-image.sh
+++ b/ai-gateway/scripts/smoke-image.sh
@@ -3,7 +3,9 @@ set -euo pipefail
image="${1:?Usage: bash scripts/smoke-image.sh IMAGE}"
container="$(docker run -d --read-only --cap-drop ALL --security-opt no-new-privileges \
- -e LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS=2 -p 127.0.0.1::8080 "$image")"
+ -e LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS=2 \
+ -e LANGFUSE_AI_GATEWAY_WEB_URL=https://web.example \
+ -e LANGFUSE_AI_GATEWAY_SERVICE_KEY=smoke-test-service-key -p 127.0.0.1::8080 "$image")"
cleanup() {
docker logs "$container" || true
docker rm -f "$container" >/dev/null || true
@@ -20,9 +22,9 @@ for route in health ready; do
done
[[ "$(docker exec "$container" id -u)" != 0 ]]
[[ "$(curl --silent --output /dev/null --write-out '%{http_code}' \
- --max-time 2 -X POST "http://$port/openai/v1/responses")" == 404 ]]
+ --max-time 2 -X POST "http://$port/openai/v1/responses")" == 401 ]]
docker stop --time 5 "$container" >/dev/null
[[ "$(docker inspect --format '{{.State.ExitCode}}' "$container")" == 0 ]]
docker logs "$container" 2>&1 | grep -q 'gateway draining'
docker logs "$container" 2>&1 | grep -q 'gateway stopped'
-printf '%s\n' 'Image smoke: non-root, probes, unimplemented inference, SIGTERM passed'
+printf '%s\n' 'Image smoke: non-root, probes, unauthenticated inference, SIGTERM passed'
diff --git a/ai-gateway/src/config.rs b/ai-gateway/src/config.rs
index e9b7c5a092..9deb1dd077 100644
--- a/ai-gateway/src/config.rs
+++ b/ai-gateway/src/config.rs
@@ -1,12 +1,16 @@
+use crate::resolution::ControlPlaneConfig;
use std::{env, error::Error, fmt, net::SocketAddr, time::Duration};
use tracing::level_filters::LevelFilter;
-pub struct Config {
+pub struct GatewayConfig {
+ pub control_plane: Option<ControlPlaneConfig>,
pub listen_address: SocketAddr,
pub auto_increment_listen_port: bool,
pub shutdown_timeout: Duration,
pub log_level: LevelFilter,
pub log_format: LogFormat,
+ pub max_active_requests: usize,
+ pub max_concurrent_resolutions: usize,
}
#[derive(Debug, PartialEq, Eq)]
@@ -16,61 +20,87 @@ pub enum LogFormat {
}
#[derive(Debug, PartialEq, Eq)]
-pub struct ConfigError(&'static str);
+pub struct GatewayConfigError(&'static str);
-impl fmt::Display for ConfigError {
+impl fmt::Display for GatewayConfigError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.0)
}
}
-impl Error for ConfigError {}
+impl Error for GatewayConfigError {}
-impl Config {
+impl GatewayConfig {
/// Read gateway configuration from the process environment, using defaults for absent values.
///
/// # Errors
/// Returns an error if a configured value is not Unicode or fails validation
- /// in [`Self::from_values`]. Error messages never include the supplied values.
- pub fn from_env() -> Result<Self, ConfigError> {
- Self::from_values(
+ /// in [`Self::from_values`], or if a Web URL is configured with an invalid URL
+ /// or missing/blank service key. Error messages never include supplied values.
+ pub fn from_env() -> Result<Self, GatewayConfigError> {
+ let mut config = Self::from_values(
read_env("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS")?.as_deref(),
read_env("LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT")?.as_deref(),
read_env("LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS")?.as_deref(),
read_env("LANGFUSE_LOG_LEVEL")?.as_deref(),
read_env("LANGFUSE_LOG_FORMAT")?.as_deref(),
- )
+ read_env("LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS")?.as_deref(),
+ read_env("LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS")?.as_deref(),
+ )?;
+ if let Some(web_url) =
+ read_env("LANGFUSE_AI_GATEWAY_WEB_URL")?.filter(|url| !url.is_empty())
+ {
+ let service_key = read_env("LANGFUSE_AI_GATEWAY_SERVICE_KEY")?.ok_or(GatewayConfigError(
+ "LANGFUSE_AI_GATEWAY_SERVICE_KEY is required when LANGFUSE_AI_GATEWAY_WEB_URL is set",
+ ))?;
+ config.control_plane = Some(ControlPlaneConfig::new(&web_url, &service_key).map_err(
+ |_| {
+ GatewayConfigError(
+ "invalid LANGFUSE_AI_GATEWAY_WEB_URL or LANGFUSE_AI_GATEWAY_SERVICE_KEY",
+ )
+ },
+ )?);
+ }
+ Ok(config)
}
/// Parse gateway configuration, using defaults for absent values.
///
/// # Errors
- /// Returns an error for an invalid IP address and port, a shutdown timeout outside
- /// 1–300 integer seconds, or an unsupported log level or format.
+ /// Returns an error for an invalid IP address and port, a non-boolean port
+ /// auto-increment setting, a shutdown timeout outside 1–300 integer seconds,
+ /// an unsupported log level or format, or concurrency
+ /// limits outside 1 through [`tokio::sync::Semaphore::MAX_PERMITS`].
pub fn from_values(
listen_address: Option<&str>,
auto_increment_listen_port: Option<&str>,
shutdown_timeout: Option<&str>,
log_level: Option<&str>,
log_format: Option<&str>,
- ) -> Result<Self, ConfigError> {
+ max_active_requests: Option<&str>,
+ max_concurrent_resolutions: Option<&str>,
+ ) -> Result<Self, GatewayConfigError> {
let listen_address = listen_address
.unwrap_or("0.0.0.0:8080")
.parse()
.map_err(|_| {
- ConfigError("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS must be an IP address and port")
+ GatewayConfigError(
+ "LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS must be an IP address and port",
+ )
})?;
- let auto_increment_listen_port =
- parse_boolean(auto_increment_listen_port.unwrap_or("false")).ok_or(ConfigError(
- "LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT must be true or false",
- ))?;
+ let auto_increment_listen_port = parse_boolean(
+ auto_increment_listen_port.unwrap_or("false"),
+ )
+ .ok_or(GatewayConfigError(
+ "LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT must be true or false",
+ ))?;
let seconds: u64 = shutdown_timeout.unwrap_or("10").parse().map_err(|_| {
- ConfigError(
+ GatewayConfigError(
"LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",
)
})?;
if !(1..=300).contains(&seconds) {
- return Err(ConfigError(
+ return Err(GatewayConfigError(
"LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",
));
}
@@ -82,7 +112,7 @@ impl Config {
// Tracing has no separate fatal severity.
"error" | "fatal" => LevelFilter::ERROR,
_ => {
- return Err(ConfigError(
+ return Err(GatewayConfigError(
"LANGFUSE_LOG_LEVEL must be trace, debug, info, warn, error or fatal",
));
}
@@ -90,23 +120,48 @@ impl Config {
let log_format = match log_format.unwrap_or("text") {
"text" => LogFormat::Text,
"json" => LogFormat::Json,
- _ => return Err(ConfigError("LANGFUSE_LOG_FORMAT must be text or json")),
+ _ => {
+ return Err(GatewayConfigError(
+ "LANGFUSE_LOG_FORMAT must be text or json",
+ ));
+ }
};
Ok(Self {
+ control_plane: None,
listen_address,
auto_increment_listen_port,
shutdown_timeout: Duration::from_secs(seconds),
log_level,
log_format,
+ max_active_requests: concurrency_limit(
+ max_active_requests,
+ "LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS must be a positive integer within the semaphore capacity",
+ )?,
+ max_concurrent_resolutions: concurrency_limit(
+ max_concurrent_resolutions,
+ "LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS must be a positive integer within the semaphore capacity",
+ )?,
})
}
}
-fn read_env(name: &'static str) -> Result<Option<String>, ConfigError> {
+fn concurrency_limit(
+ value: Option<&str>,
+ message: &'static str,
+) -> Result<usize, GatewayConfigError> {
+ value
+ .unwrap_or("128")
+ .parse::<usize>()
+ .ok()
+ .filter(|limit| (1..=tokio::sync::Semaphore::MAX_PERMITS).contains(limit))
+ .ok_or(GatewayConfigError(message))
+}
+
+fn read_env(name: &'static str) -> Result<Option<String>, GatewayConfigError> {
match env::var(name) {
Ok(value) => Ok(Some(value)),
Err(env::VarError::NotPresent) => Ok(None),
- Err(env::VarError::NotUnicode(_)) => Err(ConfigError(
+ Err(env::VarError::NotUnicode(_)) => Err(GatewayConfigError(
"gateway configuration contains a non-Unicode environment value",
)),
}
@@ -126,18 +181,22 @@ mod tests {
#[test]
fn configuration_parses_defaults_and_overrides() {
- let default = Config::from_values(None, None, None, None, None).unwrap();
+ let default = GatewayConfig::from_values(None, None, None, None, None, None, None).unwrap();
assert_eq!(default.listen_address, "0.0.0.0:8080".parse().unwrap());
assert!(!default.auto_increment_listen_port);
assert_eq!(default.shutdown_timeout, Duration::from_secs(10));
assert_eq!(default.log_level, LevelFilter::INFO);
assert_eq!(default.log_format, LogFormat::Text);
- let custom = Config::from_values(
+ assert_eq!(default.max_active_requests, 128);
+ assert_eq!(default.max_concurrent_resolutions, 128);
+ let custom = GatewayConfig::from_values(
Some("[::1]:9000"),
Some("true"),
Some("30"),
Some("debug"),
Some("json"),
+ Some("256"),
+ Some("32"),
)
.unwrap();
assert_eq!(custom.listen_address, "[::1]:9000".parse().unwrap());
@@ -145,20 +204,22 @@ mod tests {
assert_eq!(custom.shutdown_timeout, Duration::from_secs(30));
assert_eq!(custom.log_level, LevelFilter::DEBUG);
assert_eq!(custom.log_format, LogFormat::Json);
+ assert_eq!(custom.max_active_requests, 256);
+ assert_eq!(custom.max_concurrent_resolutions, 32);
}
#[test]
fn validates_shared_log_format() {
for (value, expected) in [("text", LogFormat::Text), ("json", LogFormat::Json)] {
assert_eq!(
- Config::from_values(None, None, None, None, Some(value))
+ GatewayConfig::from_values(None, None, None, None, Some(value), None, None)
.unwrap()
.log_format,
expected
);
}
for value in ["", "TEXT", "pretty", "0", "secret-that-must-not-appear"] {
- let error = Config::from_values(None, None, None, None, Some(value))
+ let error = GatewayConfig::from_values(None, None, None, None, Some(value), None, None)
.err()
.unwrap();
assert_eq!(
@@ -178,7 +239,9 @@ mod tests {
("error", LevelFilter::ERROR),
("fatal", LevelFilter::ERROR),
] {
- let config = Config::from_values(None, None, None, Some(value), None).unwrap();
+ let config =
+ GatewayConfig::from_values(None, None, None, Some(value), None, None, None)
+ .unwrap();
assert_eq!(config.log_level, expected, "{value}");
}
}
@@ -199,18 +262,58 @@ mod tests {
(None, Some("-1"), None),
(Some("localhost:8080"), None, None),
] {
- let error = Config::from_values(address, None, timeout, level, None)
+ let error = GatewayConfig::from_values(address, None, timeout, level, None, None, None)
.err()
.unwrap();
assert!(!error.to_string().contains(sensitive_input));
assert!(!format!("{error:?}").contains(sensitive_input));
}
for value in ["", "TRUE", "1", sensitive_input] {
- let error = Config::from_values(None, Some(value), None, None, None)
+ let error = GatewayConfig::from_values(None, Some(value), None, None, None, None, None)
.err()
.unwrap();
assert!(!error.to_string().contains(sensitive_input));
assert!(!format!("{error:?}").contains(sensitive_input));
}
}
+
+ #[test]
+ fn concurrency_limits_reject_invalid_values_and_accept_boundaries() {
+ let maximum = tokio::sync::Semaphore::MAX_PERMITS.to_string();
+ let overflow = (tokio::sync::Semaphore::MAX_PERMITS + 1).to_string();
+ for value in ["1", maximum.as_str()] {
+ let config =
+ GatewayConfig::from_values(None, None, None, None, None, Some(value), Some(value))
+ .unwrap();
+ assert_eq!(config.max_active_requests, value.parse::<usize>().unwrap());
+ assert_eq!(
+ config.max_concurrent_resolutions,
+ value.parse::<usize>().unwrap()
+ );
+ }
+ for value in [
+ "0",
+ "-1",
+ "1.5",
+ "",
+ "secret-that-must-not-appear",
+ overflow.as_str(),
+ ] {
+ for (active, resolutions, name) in [
+ (Some(value), None, "LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS"),
+ (
+ None,
+ Some(value),
+ "LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS",
+ ),
+ ] {
+ let error =
+ GatewayConfig::from_values(None, None, None, None, None, active, resolutions)
+ .err()
+ .unwrap();
+ assert!(error.to_string().contains(name));
+ assert!(!error.to_string().contains("secret-that-must-not-appear"));
+ }
+ }
+ }
}
diff --git a/ai-gateway/src/http.rs b/ai-gateway/src/http.rs
new file mode 100644
index 0000000000..f56f4138cd
--- /dev/null
+++ b/ai-gateway/src/http.rs
@@ -0,0 +1,202 @@
+//! Public OpenAI-compatible envelope around the opaque resolve/execute flow.
+use std::{error::Error, sync::Arc, time::Duration};
+
+use axum::{
+ Json, Router,
+ body::{Body, to_bytes},
+ extract::{Request, State},
+ http::{HeaderMap, StatusCode, header},
+ response::{IntoResponse, Response},
+ routing::post,
+};
+use serde::Serialize;
+
+use crate::{
+ inference::{InferenceService, RequestPreparationError},
+ providers::openai::ProviderError,
+ resolution::ResolutionError,
+ server::GatewayLifecycleState,
+};
+
+const MAX_REQUEST_BYTES: usize = 4 * 1024 * 1024;
+const REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(10);
+
+#[derive(Clone)]
+struct InferenceRouteState {
+ inference: Option<Arc<InferenceService>>,
+ lifecycle: GatewayLifecycleState,
+}
+
+/// Build inference routes. An unconfigured or draining service returns 503.
+pub fn router(inference: Option<InferenceService>, lifecycle: GatewayLifecycleState) -> Router {
+ Router::new()
+ .route("/openai/v1/responses", post(handle_responses))
+ .with_state(InferenceRouteState {
+ inference: inference.map(Arc::new),
+ lifecycle,
+ })
+}
+
+async fn handle_responses(
+ State(state): State<InferenceRouteState>,
+ request: Request,
+) -> Result<Response, InferenceHttpError> {
+ let inference = state
+ .inference
+ .as_ref()
+ .filter(|_| state.lifecycle.is_ready())
+ .ok_or(InferenceHttpError::Unavailable)?;
+ let gateway_key = gateway_key(request.headers())?.to_owned();
+ let (permit, context) = inference
+ .resolve_and_admit(&gateway_key)
+ .await
+ .map_err(|error| match error {
+ RequestPreparationError::Resolution(error) => InferenceHttpError::Resolution(error),
+ RequestPreparationError::Provider(error) => InferenceHttpError::Provider(error),
+ })?;
+ let (parts, body) = request.into_parts();
+ let bytes = tokio::time::timeout(REQUEST_READ_TIMEOUT, to_bytes(body, MAX_REQUEST_BYTES))
+ .await
+ .map_err(|_| InferenceHttpError::RequestTimeout)?
+ .map_err(|error| {
+ if error
+ .source()
+ .is_some_and(<dyn Error + 'static>::is::<http_body_util::LengthLimitError>)
+ {
+ InferenceHttpError::TooLarge
+ } else {
+ InferenceHttpError::InvalidBody
+ }
+ })?;
+ inference
+ .forward(permit, context, &parts.headers, bytes)
+ .await
+ .map_err(InferenceHttpError::Provider)
+}
+
+fn gateway_key(headers: &HeaderMap) -> Result<&str, InferenceHttpError> {
+ let mut values = headers.get_all(header::AUTHORIZATION).iter();
+ let value = values
+ .next()
+ .and_then(|value| value.to_str().ok())
+ .ok_or(InferenceHttpError::Credential)?;
+ if values.next().is_some() {
+ return Err(InferenceHttpError::Credential);
+ }
+ let (scheme, token) = value
+ .split_once(' ')
+ .ok_or(InferenceHttpError::Credential)?;
+ if !scheme.eq_ignore_ascii_case("Bearer")
+ || token.is_empty()
+ || token.len() > 8192
+ || !token
+ .bytes()
+ .all(|byte| byte.is_ascii_graphic() && byte != b',')
+ {
+ return Err(InferenceHttpError::Credential);
+ }
+ Ok(token)
+}
+
+enum InferenceHttpError {
+ Unavailable,
+ Credential,
+ TooLarge,
+ RequestTimeout,
+ InvalidBody,
+ Resolution(ResolutionError),
+ Provider(ProviderError),
+}
+
+#[derive(Serialize)]
+struct OpenAiErrorResponse {
+ error: OpenAiErrorDetail,
+}
+#[derive(Serialize)]
+struct OpenAiErrorDetail {
+ message: &'static str,
+ #[serde(rename = "type")]
+ kind: &'static str,
+ param: Option<&'static str>,
+ code: &'static str,
+}
+
+impl IntoResponse for InferenceHttpError {
+ fn into_response(self) -> Response<Body> {
+ use ResolutionError as R;
+ let (status, message, kind, code) = match self {
+ Self::Credential | Self::Resolution(R::InvalidCredential | R::Authentication) => (
+ StatusCode::UNAUTHORIZED,
+ "Invalid gateway credential",
+ "authentication_error",
+ "invalid_api_key",
+ ),
+ Self::Resolution(R::Forbidden) => (
+ StatusCode::FORBIDDEN,
+ "Gateway access forbidden",
+ "permission_error",
+ "permission_denied",
+ ),
+ Self::Resolution(R::NoRoute) => (
+ StatusCode::NOT_FOUND,
+ "No eligible provider connection",
+ "invalid_request_error",
+ "no_route",
+ ),
+ Self::TooLarge => (
+ StatusCode::PAYLOAD_TOO_LARGE,
+ "Request body exceeds gateway limit",
+ "invalid_request_error",
+ "request_too_large",
+ ),
+ Self::RequestTimeout => (
+ StatusCode::REQUEST_TIMEOUT,
+ "Request body read deadline exceeded",
+ "invalid_request_error",
+ "request_timeout",
+ ),
+ Self::InvalidBody => (
+ StatusCode::BAD_REQUEST,
+ "Invalid request body",
+ "invalid_request_error",
+ "invalid_body",
+ ),
+ Self::Unavailable
+ | Self::Resolution(R::Unavailable | R::Configuration)
+ | Self::Provider(ProviderError::Configuration | ProviderError::Busy) => (
+ StatusCode::SERVICE_UNAVAILABLE,
+ "Gateway temporarily unavailable",
+ "server_error",
+ "gateway_unavailable",
+ ),
+ Self::Resolution(R::Timeout) | Self::Provider(ProviderError::Timeout) => (
+ StatusCode::GATEWAY_TIMEOUT,
+ "Upstream deadline exceeded",
+ "server_error",
+ "upstream_timeout",
+ ),
+ Self::Resolution(R::Transport | R::InvalidResponse | R::ResponseTooLarge)
+ | Self::Provider(ProviderError::Transport) => (
+ StatusCode::BAD_GATEWAY,
+ "Upstream request failed",
+ "server_error",
+ "upstream_error",
+ ),
+ };
+ (
+ status,
+ Json(OpenAiErrorResponse {
+ error: OpenAiErrorDetail {
+ message,
+ kind,
+ param: None,
+ code,
+ },
+ }),
+ )
+ .into_response()
+ }
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/ai-gateway/src/http/tests.rs b/ai-gateway/src/http/tests.rs
new file mode 100644
index 0000000000..ad6aa91780
--- /dev/null
+++ b/ai-gateway/src/http/tests.rs
@@ -0,0 +1,578 @@
+use super::*;
+use crate::{
+ providers::openai::{OpenAiProvider, ProviderLimits},
+ test_support::{FakeServer, resolution_response},
+};
+use futures_util::{StreamExt, stream};
+use std::{convert::Infallible, task::Poll};
+use tokio::{net::TcpListener, sync::oneshot, task::JoinHandle};
+use tower::ServiceExt;
+
+#[tokio::test]
+async fn invalid_credentials_are_rejected_before_admission_or_body_reads() {
+ let web = FakeServer::start(|_| async {
+ Response::builder()
+ .status(StatusCode::UNAUTHORIZED)
+ .body(Body::empty())
+ .unwrap()
+ })
+ .await;
+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
+ let provider_client = OpenAiProvider::for_test(
+ provider.url.clone(),
+ ProviderLimits {
+ active: 1,
+ ..ProviderLimits::default()
+ },
+ );
+ let permit = provider_client.try_admit().unwrap();
+ let gateway = Gateway::start(Some(InferenceService::for_test(
+ web.control_plane(),
+ provider_client,
+ 1,
+ )))
+ .await;
+ // Exercise both an exhausted and an available execution pool.
+ for occupied in [Some(permit), None] {
+ let body = Body::from_stream(stream::poll_fn(
+ |_| -> Poll<Option<Result<&'static str, Infallible>>> {
+ panic!("an invalid credential must be rejected before polling the body");
+ },
+ ));
+ let response = tokio::time::timeout(
+ Duration::from_secs(2),
+ gateway.app.clone().oneshot(
+ Request::post("/openai/v1/responses")
+ .header(header::AUTHORIZATION, "Bearer invalid-key")
+ .body(body)
+ .unwrap(),
+ ),
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
+ drop(occupied);
+ }
+ assert_eq!(web.calls(), 2);
+ assert_eq!(provider.calls(), 0);
+}
+
+struct Gateway {
+ url: String,
+ shutdown: Option<oneshot::Sender<()>>,
+ task: JoinHandle<std::io::Result<()>>,
+ app: Router,
+}
+
+#[tokio::test]
+async fn resolution_capacity_is_bounded_and_released_on_cancellation_and_failure() {
+ let entered = Arc::new(tokio::sync::Notify::new());
+ let started = entered.clone();
+ let web = FakeServer::start(move |request| {
+ let started = started.clone();
+ async move {
+ match request.headers()[header::AUTHORIZATION].to_str().unwrap() {
+ "Bearer stalled" => {
+ started.notify_one();
+ std::future::pending().await
+ }
+ "Bearer invalid" => Response::builder()
+ .status(StatusCode::UNAUTHORIZED)
+ .body(Body::empty())
+ .unwrap(),
+ _ => resolution_response("provider"),
+ }
+ }
+ })
+ .await;
+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
+ let service = InferenceService::for_test(
+ web.control_plane(),
+ OpenAiProvider::for_test(provider.url.clone(), ProviderLimits::default()),
+ 1,
+ );
+ let gateway = Gateway::start(Some(service)).await;
+ let request = |key| {
+ Request::post("/openai/v1/responses")
+ .header(header::AUTHORIZATION, key)
+ .body(Body::from("{}"))
+ .unwrap()
+ };
+ let mut stalled = Box::pin(gateway.app.clone().oneshot(request("Bearer stalled")));
+ tokio::time::timeout(Duration::from_secs(2), async {
+ tokio::select! {
+ () = entered.notified() => {},
+ _ = &mut stalled => panic!("resolution should be pending"),
+ }
+ })
+ .await
+ .unwrap();
+ let busy = tokio::time::timeout(
+ Duration::from_secs(2),
+ gateway.app.clone().oneshot(request("Bearer valid")),
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(busy.status(), StatusCode::SERVICE_UNAVAILABLE);
+ assert_eq!(web.calls(), 1);
+ assert_eq!(provider.calls(), 0);
+
+ drop(stalled);
+ for (key, status) in [
+ ("Bearer invalid", StatusCode::UNAUTHORIZED),
+ ("Bearer valid", StatusCode::OK),
+ ] {
+ let response = tokio::time::timeout(
+ Duration::from_secs(2),
+ gateway.app.clone().oneshot(request(key)),
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(response.status(), status);
+ }
+ assert_eq!(web.calls(), 3);
+ assert_eq!(provider.calls(), 1);
+}
+
+impl Gateway {
+ async fn start(inference: Option<InferenceService>) -> Self {
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let url = format!("http://{}", listener.local_addr().unwrap());
+ let state = if inference.is_some() {
+ GatewayLifecycleState::default()
+ } else {
+ GatewayLifecycleState::unconfigured()
+ };
+ let app = crate::server::router(state.clone()).merge(router(inference, state.clone()));
+ let (shutdown, signal) = oneshot::channel();
+ let task = tokio::spawn(crate::server::serve(
+ listener,
+ app.clone(),
+ state,
+ async {
+ let _ = signal.await;
+ },
+ Duration::from_secs(1),
+ ));
+ let client = reqwest::Client::new();
+ for _ in 0..100 {
+ if client.get(format!("{url}/health")).send().await.is_ok() {
+ return Self {
+ url,
+ shutdown: Some(shutdown),
+ task,
+ app,
+ };
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ panic!("gateway did not start");
+ }
+
+ fn post(&self) -> reqwest::RequestBuilder {
+ reqwest::Client::new().post(format!("{}/openai/v1/responses", self.url))
+ }
+}
+
+impl Drop for Gateway {
+ fn drop(&mut self) {
+ self.task.abort();
+ }
+}
+
+fn inference(web: &FakeServer, provider: &FakeServer) -> InferenceService {
+ InferenceService::for_test(
+ web.control_plane(),
+ OpenAiProvider::for_test(
+ format!("{}/v1/responses", provider.url),
+ ProviderLimits::default(),
+ ),
+ 128,
+ )
+}
+
+#[tokio::test]
+async fn native_json_and_sse_traverse_resolution_and_relay() {
+ let web = FakeServer::start(|request| async move {
+ assert_eq!(request.uri(), "/api/internal/ai-gateway/v1/resolve");
+ assert!(
+ request
+ .headers()
+ .contains_key("langfuse-gateway-authorization")
+ );
+ let key = request.headers()[header::AUTHORIZATION]
+ .to_str()
+ .unwrap()
+ .to_owned();
+ assert_eq!(
+ to_bytes(request.into_body(), 1024).await.unwrap(),
+ r#"{"apiFormat":"openai.responses"}"#
+ );
+ resolution_response(if key == "Bearer gateway-json" {
+ "provider-json"
+ } else {
+ "provider-sse"
+ })
+ })
+ .await;
+ let provider = FakeServer::start(|request| async move {
+ assert_eq!(request.uri(), "/v1/responses");
+ assert!(!request.headers().contains_key("x-langfuse-project-id"));
+ assert!(
+ !request
+ .headers()
+ .contains_key("langfuse-gateway-authorization")
+ );
+ let key = request.headers()[header::AUTHORIZATION]
+ .to_str()
+ .unwrap()
+ .to_owned();
+ let body = to_bytes(request.into_body(), 4096).await.unwrap();
+ if key == "Bearer provider-json" {
+ assert_eq!(body, "{ \"model\":\"unchanged\", \"unknown\":true }");
+ Response::builder()
+ .header("content-type", "application/json")
+ .body(Body::from("{\"output\":[]}"))
+ .unwrap()
+ } else {
+ assert_eq!(key, "Bearer provider-sse");
+ assert_eq!(body, r#"{"model":"anything","stream":true}"#);
+ Response::builder()
+ .header("content-type", "text/event-stream")
+ .body(Body::from_stream(stream::iter([
+ Ok::<_, Infallible>("event: arbitrary\ndata: "),
+ Ok("hello\n\n"),
+ ])))
+ .unwrap()
+ }
+ })
+ .await;
+ let gateway = Gateway::start(Some(inference(&web, &provider))).await;
+ let json = gateway
+ .post()
+ .bearer_auth("gateway-json")
+ .header("x-langfuse-project-id", "attacker-project")
+ .body("{ \"model\":\"unchanged\", \"unknown\":true }")
+ .send();
+ let sse = gateway
+ .post()
+ .bearer_auth("gateway-sse")
+ .body(r#"{"model":"anything","stream":true}"#)
+ .send();
+ let (json, sse) = tokio::join!(json, sse);
+ let json = json.unwrap();
+ let sse = sse.unwrap();
+ assert_eq!(json.status(), StatusCode::OK);
+ assert_eq!(json.text().await.unwrap(), r#"{"output":[]}"#);
+ assert_eq!(sse.headers()[header::CONTENT_TYPE], "text/event-stream");
+ assert_eq!(
+ sse.text().await.unwrap(),
+ "event: arbitrary\ndata: hello\n\n"
+ );
+ assert_eq!(web.calls(), 2);
+ assert_eq!(provider.calls(), 2);
+}
+
+#[tokio::test]
+async fn malformed_credentials_skip_resolution_and_oversized_body_releases_capacity() {
+ let web = FakeServer::start(|_| async { resolution_response("provider-secret") }).await;
+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
+ let service = InferenceService::for_test(
+ web.control_plane(),
+ OpenAiProvider::for_test(
+ provider.url.clone(),
+ ProviderLimits {
+ active: 1,
+ ..ProviderLimits::default()
+ },
+ ),
+ 1,
+ );
+ let gateway = Gateway::start(Some(service)).await;
+ for authorization in [
+ None,
+ Some("Basic abc"),
+ Some("Bearer "),
+ Some("Bearer key,other"),
+ Some("Bearer key extra"),
+ ] {
+ let mut request = gateway.post();
+ if let Some(value) = authorization {
+ request = request.header(header::AUTHORIZATION, value);
+ }
+ let response = request.send().await.unwrap();
+ assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
+ let body: serde_json::Value =
+ serde_json::from_slice(&response.bytes().await.unwrap()).unwrap();
+ assert_eq!(body["error"]["code"], "invalid_api_key");
+ }
+ let mut headers = HeaderMap::new();
+ headers.append(header::AUTHORIZATION, "Bearer one".parse().unwrap());
+ headers.append(header::AUTHORIZATION, "Bearer two".parse().unwrap());
+ assert_eq!(
+ gateway
+ .post()
+ .headers(headers)
+ .send()
+ .await
+ .unwrap()
+ .status(),
+ StatusCode::UNAUTHORIZED
+ );
+ assert_eq!(web.calls(), 0);
+ let response = gateway
+ .post()
+ .bearer_auth("valid-key")
+ .body(vec![b'x'; MAX_REQUEST_BYTES + 1])
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE);
+ assert_eq!(web.calls(), 1);
+ assert_eq!(provider.calls(), 0);
+ let next = gateway
+ .post()
+ .bearer_auth("valid-key")
+ .body("{}")
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(next.status(), StatusCode::OK);
+ assert_eq!(provider.calls(), 1);
+}
+
+#[tokio::test]
+async fn resolution_failures_are_sanitized_and_never_call_provider() {
+ let web = FakeServer::start(|request| async move {
+ let key = request.headers()[header::AUTHORIZATION].to_str().unwrap();
+ match key {
+ "Bearer malformed" => Response::new(Body::from("secret malformed response")),
+ "Bearer invalid-origin" => {
+ let body = to_bytes(resolution_response("provider-secret").into_body(), 4096)
+ .await
+ .unwrap();
+ let mut body: serde_json::Value = serde_json::from_slice(&body).unwrap();
+ body["connection"]["base_url"] = "https://attacker.example/v1".into();
+ Response::new(Body::from(body.to_string()))
+ }
+ _ => {
+ let status: u16 = key.strip_prefix("Bearer ").unwrap().parse().unwrap();
+ Response::builder()
+ .status(status)
+ .body(Body::from("secret upstream error"))
+ .unwrap()
+ }
+ }
+ })
+ .await;
+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
+ let gateway = Gateway::start(Some(inference(&web, &provider))).await;
+ for (key, status) in [
+ ("401", 401),
+ ("403", 403),
+ ("404", 404),
+ ("503", 503),
+ ("malformed", 502),
+ ("invalid-origin", 502),
+ ] {
+ let response = gateway
+ .post()
+ .bearer_auth(key)
+ .body("{}")
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(response.status().as_u16(), status);
+ let body = response.text().await.unwrap();
+ assert!(!body.contains("secret"));
+ assert!(serde_json::from_str::<serde_json::Value>(&body).unwrap()["error"].is_object());
+ }
+ assert_eq!(provider.calls(), 0);
+}
+
+#[tokio::test]
+async fn slow_request_body_and_resolution_have_bounded_waits() {
+ let web = FakeServer::start(|request| async move {
+ if request.headers()[header::AUTHORIZATION] != "Bearer slow-resolution" {
+ return resolution_response("provider");
+ }
+ Response::new(Body::from_stream(stream::pending::<
+ Result<&'static str, Infallible>,
+ >()))
+ })
+ .await;
+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
+ let gateway = Gateway::start(Some(inference(&web, &provider))).await;
+ let slow_body = gateway
+ .post()
+ .bearer_auth("key")
+ .body(reqwest::Body::wrap_stream(stream::pending::<
+ Result<&'static str, Infallible>,
+ >()))
+ .send();
+ let slow_resolution = gateway
+ .post()
+ .bearer_auth("slow-resolution")
+ .body("{}")
+ .send();
+ let (body, resolution) = tokio::time::timeout(Duration::from_secs(12), async {
+ tokio::join!(slow_body, slow_resolution)
+ })
+ .await
+ .unwrap();
+ assert_eq!(body.unwrap().status(), StatusCode::REQUEST_TIMEOUT);
+ assert_eq!(resolution.unwrap().status(), StatusCode::GATEWAY_TIMEOUT);
+ assert_eq!(web.calls(), 2);
+ assert_eq!(provider.calls(), 0);
+}
+
+#[tokio::test]
+async fn unconfigured_gateway_is_live_but_not_ready_and_inference_is_unavailable() {
+ let gateway = Gateway::start(None).await;
+ let client = reqwest::Client::new();
+ assert_eq!(
+ client
+ .get(format!("{}/health", gateway.url))
+ .send()
+ .await
+ .unwrap()
+ .status(),
+ StatusCode::OK
+ );
+ assert_eq!(
+ client
+ .get(format!("{}/ready", gateway.url))
+ .send()
+ .await
+ .unwrap()
+ .status(),
+ StatusCode::SERVICE_UNAVAILABLE
+ );
+ assert_eq!(
+ gateway.post().send().await.unwrap().status(),
+ StatusCode::SERVICE_UNAVAILABLE
+ );
+}
+
+#[tokio::test]
+async fn active_admission_is_held_through_stream_and_shutdown_drains_it() {
+ let (release, released) = tokio::sync::watch::channel(false);
+ let web = FakeServer::start(|_| async { resolution_response("provider") }).await;
+ let provider = FakeServer::start(move |_| {
+ let mut released = released.clone();
+ async move {
+ let stream = stream::once(async { Ok::<_, Infallible>("data: first\n\n") }).chain(
+ stream::once(async move {
+ released.wait_for(|done| *done).await.unwrap();
+ Ok("data: last\n\n")
+ }),
+ );
+ Response::builder()
+ .header("content-type", "text/event-stream")
+ .body(Body::from_stream(stream))
+ .unwrap()
+ }
+ })
+ .await;
+ let service = InferenceService::for_test(
+ web.control_plane(),
+ OpenAiProvider::for_test(
+ provider.url.clone(),
+ ProviderLimits {
+ active: 1,
+ ..ProviderLimits::default()
+ },
+ ),
+ 1,
+ );
+ let mut gateway = Gateway::start(Some(service)).await;
+ let mut response = gateway
+ .post()
+ .bearer_auth("key")
+ .body("{}")
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(response.chunk().await.unwrap().unwrap(), "data: first\n\n");
+ assert_eq!(
+ gateway
+ .post()
+ .bearer_auth("key")
+ .body("{}")
+ .send()
+ .await
+ .unwrap()
+ .status(),
+ StatusCode::SERVICE_UNAVAILABLE
+ );
+ assert_eq!(web.calls(), 2);
+ assert_eq!(provider.calls(), 1);
+ gateway.shutdown.take().unwrap().send(()).unwrap();
+ assert!(!gateway.task.is_finished());
+ release.send(true).unwrap();
+ assert_eq!(response.text().await.unwrap(), "data: last\n\n");
+ tokio::time::timeout(Duration::from_secs(2), &mut gateway.task)
+ .await
+ .unwrap()
+ .unwrap()
+ .unwrap();
+}
+
+#[tokio::test]
+async fn client_disconnect_releases_capacity_for_the_next_request() {
+ let web = FakeServer::start(|_| async { resolution_response("provider") }).await;
+ let provider = FakeServer::start(|_| async {
+ let stream =
+ stream::once(async { Ok::<_, Infallible>("data: first\n\n") }).chain(stream::pending());
+ Response::builder()
+ .header("content-type", "text/event-stream")
+ .body(Body::from_stream(stream))
+ .unwrap()
+ })
+ .await;
+ let service = InferenceService::for_test(
+ web.control_plane(),
+ OpenAiProvider::for_test(
+ provider.url.clone(),
+ ProviderLimits {
+ active: 1,
+ ..ProviderLimits::default()
+ },
+ ),
+ 1,
+ );
+ let gateway = Gateway::start(Some(service)).await;
+ let mut first = gateway
+ .post()
+ .bearer_auth("key")
+ .body("{}")
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(first.chunk().await.unwrap().unwrap(), "data: first\n\n");
+ drop(first);
+ let mut next = tokio::time::timeout(Duration::from_secs(2), async {
+ loop {
+ let response = gateway
+ .post()
+ .bearer_auth("key")
+ .body("{}")
+ .send()
+ .await
+ .unwrap();
+ if response.status() == StatusCode::OK {
+ break response;
+ }
+ assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .unwrap();
+ assert_eq!(next.chunk().await.unwrap().unwrap(), "data: first\n\n");
+ assert!(web.calls() >= 2);
+ assert_eq!(provider.calls(), 2);
+}
diff --git a/ai-gateway/src/inference.rs b/ai-gateway/src/inference.rs
new file mode 100644
index 0000000000..fce29bf6bc
--- /dev/null
+++ b/ai-gateway/src/inference.rs
@@ -0,0 +1,91 @@
+//! Resolve the caller's credential, then execute using only the trusted context.
+use axum::{
+ body::{Body, Bytes},
+ http::{HeaderMap, Response},
+};
+use tokio::sync::Semaphore;
+
+use crate::{
+ providers::openai::{OpenAiProvider, ProviderError, RequestPermit},
+ resolution::{
+ ApiFormat, ControlPlaneClient, ControlPlaneConfig, ResolutionError, ResolvedRequestContext,
+ },
+};
+
+pub struct InferenceService {
+ control_plane: ControlPlaneClient,
+ provider: OpenAiProvider,
+ resolution_capacity: Semaphore,
+}
+
+pub(crate) enum RequestPreparationError {
+ Resolution(ResolutionError),
+ Provider(ProviderError),
+}
+
+impl InferenceService {
+ /// Initialize pooled clients for resolution and provider requests.
+ ///
+ /// # Errors
+ /// Returns a sanitized configuration error for invalid capacities or client setup.
+ pub fn new(
+ config: ControlPlaneConfig,
+ max_active_requests: usize,
+ max_concurrent_resolutions: usize,
+ ) -> Result<Self, ResolutionError> {
+ if !(1..=Semaphore::MAX_PERMITS).contains(&max_concurrent_resolutions) {
+ return Err(ResolutionError::Configuration);
+ }
+ Ok(Self {
+ control_plane: ControlPlaneClient::new(config)?,
+ provider: OpenAiProvider::new(max_active_requests)
+ .map_err(|_| ResolutionError::Configuration)?,
+ resolution_capacity: Semaphore::new(max_concurrent_resolutions),
+ })
+ }
+
+ /// Authenticate with a separate bounded budget before reserving execution capacity.
+ pub(crate) async fn resolve_and_admit(
+ &self,
+ gateway_key: &str,
+ ) -> Result<(RequestPermit, ResolvedRequestContext), RequestPreparationError> {
+ let context = {
+ let _permit = self
+ .resolution_capacity
+ .try_acquire()
+ .map_err(|_| RequestPreparationError::Resolution(ResolutionError::Unavailable))?;
+ self.control_plane
+ .resolve(gateway_key, ApiFormat::OpenAiResponses)
+ .await
+ .map_err(RequestPreparationError::Resolution)?
+ };
+ let permit = self
+ .provider
+ .try_admit()
+ .map_err(RequestPreparationError::Provider)?;
+ Ok((permit, context))
+ }
+
+ pub(crate) async fn forward(
+ &self,
+ permit: RequestPermit,
+ context: ResolvedRequestContext,
+ headers: &HeaderMap,
+ body: Bytes,
+ ) -> Result<Response<Body>, ProviderError> {
+ self.provider.forward(permit, context, headers, body).await
+ }
+
+ #[cfg(test)]
+ pub(crate) fn for_test(
+ control_plane: ControlPlaneClient,
+ provider: OpenAiProvider,
+ resolutions: usize,
+ ) -> Self {
+ Self {
+ control_plane,
+ provider,
+ resolution_capacity: Semaphore::new(resolutions),
+ }
+ }
+}
diff --git a/ai-gateway/src/lib.rs b/ai-gateway/src/lib.rs
index ff115b8a31..60740a9181 100644
--- a/ai-gateway/src/lib.rs
+++ b/ai-gateway/src/lib.rs
@@ -1,3 +1,10 @@
pub mod config;
+pub mod http;
+pub mod inference;
+pub mod providers;
pub mod resolution;
pub mod server;
+mod transport;
+
+#[cfg(test)]
+mod test_support;
diff --git a/ai-gateway/src/main.rs b/ai-gateway/src/main.rs
index fa0511ea9a..3d4873dabd 100644
--- a/ai-gateway/src/main.rs
+++ b/ai-gateway/src/main.rs
@@ -1,8 +1,10 @@
use std::{error::Error, io, net::SocketAddr, process::ExitCode};
use ai_gateway::{
- config::{Config, LogFormat},
- server::{self, AppState},
+ config::{GatewayConfig, LogFormat},
+ http,
+ inference::InferenceService,
+ server::{self, GatewayLifecycleState},
};
use tokio::net::TcpListener;
@@ -18,7 +20,7 @@ async fn main() -> ExitCode {
}
async fn run() -> Result<(), Box<dyn Error>> {
- let config = Config::from_env()?;
+ let config = GatewayConfig::from_env()?;
let logging = tracing_subscriber::fmt()
.with_max_level(config.log_level)
.with_target(false);
@@ -26,18 +28,27 @@ async fn run() -> Result<(), Box<dyn Error>> {
LogFormat::Text => logging.compact().init(),
LogFormat::Json => logging.json().init(),
}
+ let inference = config
+ .control_plane
+ .map(|control_plane| {
+ InferenceService::new(
+ control_plane,
+ config.max_active_requests,
+ config.max_concurrent_resolutions,
+ )
+ })
+ .transpose()?;
+ let inference_enabled = inference.is_some();
+ let state = if inference_enabled {
+ GatewayLifecycleState::default()
+ } else {
+ GatewayLifecycleState::unconfigured()
+ };
+ let app = server::router(state.clone()).merge(http::router(inference, state.clone()));
let shutdown = shutdown_signal()?;
let listener = bind_listener(config.listen_address, config.auto_increment_listen_port).await?;
- tracing::info!(address = %listener.local_addr()?, "gateway listening");
- let state = AppState::default();
- server::serve(
- listener,
- server::router(state.clone()),
- state,
- shutdown,
- config.shutdown_timeout,
- )
- .await?;
+ tracing::info!(address = %listener.local_addr()?, inference_enabled, "gateway listening");
+ server::serve(listener, app, state, shutdown, config.shutdown_timeout).await?;
tracing::info!("gateway stopped");
Ok(())
}
diff --git a/ai-gateway/src/providers/mod.rs b/ai-gateway/src/providers/mod.rs
new file mode 100644
index 0000000000..d8c308735b
--- /dev/null
+++ b/ai-gateway/src/providers/mod.rs
@@ -0,0 +1 @@
+pub mod openai;
diff --git a/ai-gateway/src/providers/openai.rs b/ai-gateway/src/providers/openai.rs
new file mode 100644
index 0000000000..e8882905b4
--- /dev/null
+++ b/ai-gateway/src/providers/openai.rs
@@ -0,0 +1,163 @@
+//! Native Responses transport. Provider requests use only resolved credentials.
+use std::{sync::Arc, time::Duration};
+
+use axum::{
+ body::{Body, Bytes},
+ http::{HeaderMap, HeaderValue, Response, header},
+};
+use reqwest::Client;
+use tokio::{
+ sync::{OwnedSemaphorePermit, Semaphore},
+ time::Instant,
+};
+
+use crate::{resolution::ResolvedRequestContext, transport};
+
+pub use crate::transport::ProviderError;
+
+pub(crate) struct ProviderLimits {
+ pub active: usize,
+ pub execution_timeout: Duration,
+ pub headers_timeout: Duration,
+ pub read_timeout: Duration,
+}
+
+impl Default for ProviderLimits {
+ fn default() -> Self {
+ Self {
+ active: 128,
+ execution_timeout: Duration::from_secs(600),
+ headers_timeout: Duration::from_secs(120),
+ read_timeout: Duration::from_secs(120),
+ }
+ }
+}
+
+/// An admitted execution. Dropping it releases capacity; there is no waiting queue.
+pub struct RequestPermit {
+ _permit: OwnedSemaphorePermit,
+ deadline: Instant,
+}
+
+/// A pooled client for the official `OpenAI` Responses endpoint.
+pub struct OpenAiProvider {
+ client: Client,
+ capacity: Arc<Semaphore>,
+ limits: ProviderLimits,
+ #[cfg(test)]
+ endpoint: String,
+}
+
+impl OpenAiProvider {
+ /// Construct the provider transport with bounded admission and transport waits.
+ ///
+ /// # Errors
+ /// Returns [`ProviderError::Configuration`] when the HTTPS client cannot be initialized.
+ pub fn new(max_active_requests: usize) -> Result<Self, ProviderError> {
+ Self::with_limits(ProviderLimits {
+ active: max_active_requests,
+ ..ProviderLimits::default()
+ })
+ }
+
+ fn with_limits(limits: ProviderLimits) -> Result<Self, ProviderError> {
+ if !(1..=Semaphore::MAX_PERMITS).contains(&limits.active) {
+ return Err(ProviderError::Configuration);
+ }
+ let client = Client::builder()
+ .redirect(reqwest::redirect::Policy::none())
+ .retry(reqwest::retry::never())
+ .no_proxy()
+ .no_gzip()
+ .no_brotli()
+ .no_deflate()
+ .no_zstd()
+ .connect_timeout(Duration::from_secs(5))
+ .read_timeout(limits.read_timeout)
+ .pool_max_idle_per_host(16)
+ .build()
+ .map_err(|_| ProviderError::Configuration)?;
+ Ok(Self {
+ client,
+ capacity: Arc::new(Semaphore::new(limits.active)),
+ limits,
+ #[cfg(test)]
+ endpoint: "https://api.openai.com/v1/responses".to_owned(),
+ })
+ }
+
+ /// Reserve capacity for an authenticated request before reading its body.
+ ///
+ /// # Errors
+ /// Returns [`ProviderError::Busy`] immediately when all execution slots are occupied.
+ pub fn try_admit(&self) -> Result<RequestPermit, ProviderError> {
+ let permit = self
+ .capacity
+ .clone()
+ .try_acquire_owned()
+ .map_err(|_| ProviderError::Busy)?;
+ Ok(RequestPermit {
+ _permit: permit,
+ deadline: Instant::now() + self.limits.execution_timeout,
+ })
+ }
+
+ /// Execute once and stream native status, safe headers and entity bytes.
+ /// The response body owns admission and the trusted context until completion/drop.
+ ///
+ /// # Errors
+ /// Returns a sanitized error for invalid credentials, transport failures or a
+ /// deadline before response headers arrive. Later failures terminate the body.
+ pub async fn forward(
+ &self,
+ permit: RequestPermit,
+ context: ResolvedRequestContext,
+ headers: &HeaderMap,
+ body: Bytes,
+ ) -> Result<Response<Body>, ProviderError> {
+ #[cfg(not(test))]
+ let endpoint = "https://api.openai.com/v1/responses";
+ #[cfg(test)]
+ let endpoint = &self.endpoint;
+
+ let mut authorization =
+ HeaderValue::from_str(&format!("Bearer {}", context.connection().provider_token()))
+ .map_err(|_| ProviderError::Configuration)?;
+ authorization.set_sensitive(true);
+ let response = tokio::time::timeout_at(
+ permit
+ .deadline
+ .min(Instant::now() + self.limits.headers_timeout),
+ self.client
+ .post(endpoint)
+ .headers(transport::request_headers(headers))
+ .header(header::AUTHORIZATION, authorization)
+ .body(body)
+ .send(),
+ )
+ .await
+ .map_err(|_| ProviderError::Timeout)?
+ .map_err(|error| {
+ if error.is_timeout() {
+ ProviderError::Timeout
+ } else {
+ ProviderError::Transport
+ }
+ })?;
+ let mut downstream = Response::new(Body::empty());
+ *downstream.status_mut() = response.status();
+ *downstream.headers_mut() = transport::response_headers(response.headers());
+ *downstream.body_mut() = transport::relay(response, permit.deadline, (permit, context));
+ Ok(downstream)
+ }
+
+ #[cfg(test)]
+ pub(crate) fn for_test(endpoint: String, limits: ProviderLimits) -> Self {
+ let mut provider = Self::with_limits(limits).unwrap();
+ provider.endpoint = endpoint;
+ provider
+ }
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/ai-gateway/src/providers/openai/tests.rs b/ai-gateway/src/providers/openai/tests.rs
new file mode 100644
index 0000000000..4c1bde86b8
--- /dev/null
+++ b/ai-gateway/src/providers/openai/tests.rs
@@ -0,0 +1,333 @@
+use super::*;
+use crate::test_support::{FakeServer, resolved_request_context};
+use axum::{body::to_bytes, http::StatusCode};
+use futures_util::{StreamExt, stream};
+use std::{convert::Infallible, future::pending};
+use tokio::sync::Notify;
+
+fn provider(server: &FakeServer, active: usize) -> OpenAiProvider {
+ OpenAiProvider::for_test(
+ format!("{}/v1/responses", server.url),
+ ProviderLimits {
+ active,
+ ..ProviderLimits::default()
+ },
+ )
+}
+
+#[tokio::test]
+async fn preserves_opaque_bytes_and_isolates_request_and_response_headers() {
+ const REQUEST: &[u8] = b"{ \"model\": \"opaque\", \"future_field\": [true, 3] }\n";
+ const RESPONSE: &[u8] = b"\x1f\x8b\x08\x00opaque-encoded-response";
+ let upstream = FakeServer::start(|request| async move {
+ assert_eq!(request.uri(), "/v1/responses");
+ assert_eq!(request.method(), "POST");
+ assert_eq!(
+ request.headers()[header::AUTHORIZATION],
+ "Bearer provider-secret"
+ );
+ assert_eq!(request.headers()[header::CONTENT_TYPE], "application/json");
+ for name in [
+ "cookie",
+ "langfuse-gateway-authorization",
+ "openai-organization",
+ "openai-project",
+ "x-api-key",
+ "x-forwarded-host",
+ "accept-encoding",
+ ] {
+ assert!(!request.headers().contains_key(name), "forwarded {name}");
+ }
+ assert_eq!(to_bytes(request.into_body(), 1024).await.unwrap(), REQUEST);
+ Response::builder()
+ .status(201)
+ .header("content-type", "application/json")
+ .header("content-encoding", "gzip")
+ .header("x-request-id", "request-123")
+ .header("set-cookie", "private-cookie")
+ .header("connection", "openai-version")
+ .header("openai-version", "private-hop-value")
+ .body(Body::from(RESPONSE))
+ .unwrap()
+ })
+ .await;
+ let relay = provider(&upstream, 1);
+ let mut headers = HeaderMap::new();
+ for (name, value) in [
+ ("authorization", "Bearer gateway-secret"),
+ ("cookie", "session=secret"),
+ ("langfuse-gateway-authorization", "service-signature"),
+ ("openai-organization", "wrong-org"),
+ ("openai-project", "wrong-project"),
+ ("x-api-key", "wrong-key"),
+ ("x-forwarded-host", "attacker.example"),
+ ("content-type", "application/json"),
+ ("accept-encoding", "gzip"),
+ ("connection", "accept-encoding"),
+ ] {
+ headers.insert(
+ axum::http::HeaderName::from_static(name),
+ HeaderValue::from_static(value),
+ );
+ }
+ let response = relay
+ .forward(
+ relay.try_admit().unwrap(),
+ resolved_request_context("provider-secret").await,
+ &headers,
+ Bytes::from_static(REQUEST),
+ )
+ .await
+ .unwrap();
+ assert_eq!(response.status(), StatusCode::CREATED);
+ assert_eq!(response.headers()["content-encoding"], "gzip");
+ assert_eq!(response.headers()["x-request-id"], "request-123");
+ for name in [
+ "set-cookie",
+ "connection",
+ "openai-version",
+ "content-length",
+ "transfer-encoding",
+ ] {
+ assert!(!response.headers().contains_key(name));
+ }
+ assert_eq!(
+ to_bytes(response.into_body(), 1024).await.unwrap(),
+ RESPONSE
+ );
+ assert!(relay.try_admit().is_ok());
+ assert_eq!(upstream.calls(), 1);
+}
+
+#[tokio::test]
+async fn preserves_provider_failures_without_retries_or_redirects() {
+ let target = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
+ for status in [307, 429, 500, 503] {
+ let location = target.url.clone();
+ let upstream = FakeServer::start(move |_| {
+ let location = location.clone();
+ async move {
+ Response::builder()
+ .status(status)
+ .header("location", location)
+ .header("retry-after", "7")
+ .body(Body::from("native-provider-body"))
+ .unwrap()
+ }
+ })
+ .await;
+ let relay = provider(&upstream, 1);
+ let response = relay
+ .forward(
+ relay.try_admit().unwrap(),
+ resolved_request_context("provider-secret").await,
+ &HeaderMap::new(),
+ Bytes::new(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(response.status().as_u16(), status);
+ assert_eq!(response.headers()["retry-after"], "7");
+ assert!(!response.headers().contains_key("location"));
+ assert_eq!(
+ to_bytes(response.into_body(), 1024).await.unwrap(),
+ "native-provider-body"
+ );
+ assert_eq!(upstream.calls(), 1);
+ }
+ assert_eq!(target.calls(), 0);
+}
+
+#[tokio::test]
+async fn streams_sse_incrementally_and_holds_admission_until_eof() {
+ let release = Arc::new(Notify::new());
+ let upstream_release = release.clone();
+ let upstream = FakeServer::start(move |_| {
+ let release = upstream_release.clone();
+ async move {
+ let chunks = stream::once(async { Ok::<_, Infallible>("data: first\n\n") }).chain(
+ stream::once(async move {
+ release.notified().await;
+ Ok("data: [DONE]\n\n")
+ }),
+ );
+ Response::builder()
+ .header("content-type", "text/event-stream")
+ .body(Body::from_stream(chunks))
+ .unwrap()
+ }
+ })
+ .await;
+ let relay = provider(&upstream, 1);
+ let response = relay
+ .forward(
+ relay.try_admit().unwrap(),
+ resolved_request_context("provider-secret").await,
+ &HeaderMap::new(),
+ Bytes::new(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(response.headers()["content-type"], "text/event-stream");
+ let mut body = response.into_body().into_data_stream();
+ assert_eq!(
+ tokio::time::timeout(Duration::from_secs(1), body.next())
+ .await
+ .unwrap()
+ .unwrap()
+ .unwrap(),
+ "data: first\n\n"
+ );
+ assert!(matches!(relay.try_admit(), Err(ProviderError::Busy)));
+ release.notify_one();
+ let mut remaining = Vec::new();
+ while let Some(chunk) = body.next().await {
+ remaining.extend_from_slice(&chunk.unwrap());
+ }
+ assert_eq!(remaining, b"data: [DONE]\n\n");
+ assert!(relay.try_admit().is_ok());
+}
+
+#[tokio::test]
+async fn simultaneous_requests_keep_provider_credentials_and_bodies_isolated() {
+ let upstream = FakeServer::start(|request| async move {
+ let authorization = request.headers()[header::AUTHORIZATION]
+ .to_str()
+ .unwrap()
+ .to_owned();
+ let body = to_bytes(request.into_body(), 1024).await.unwrap();
+ Response::new(Body::from(format!(
+ "{authorization}:{}",
+ String::from_utf8(body.to_vec()).unwrap()
+ )))
+ })
+ .await;
+ let relay = provider(&upstream, 2);
+ let alice = resolved_request_context("alice-secret").await;
+ let bob = resolved_request_context("bob-secret").await;
+ let headers = HeaderMap::new();
+ let (alice, bob) = tokio::join!(
+ relay.forward(
+ relay.try_admit().unwrap(),
+ alice,
+ &headers,
+ Bytes::from_static(b"alice-body")
+ ),
+ relay.forward(
+ relay.try_admit().unwrap(),
+ bob,
+ &headers,
+ Bytes::from_static(b"bob-body")
+ ),
+ );
+ assert_eq!(
+ to_bytes(alice.unwrap().into_body(), 1024).await.unwrap(),
+ "Bearer alice-secret:alice-body"
+ );
+ assert_eq!(
+ to_bytes(bob.unwrap().into_body(), 1024).await.unwrap(),
+ "Bearer bob-secret:bob-body"
+ );
+ assert_eq!(upstream.calls(), 2);
+}
+
+struct Dropped(Arc<Notify>);
+impl Drop for Dropped {
+ fn drop(&mut self) {
+ self.0.notify_one();
+ }
+}
+
+async fn stalled_provider(dropped: Arc<Notify>) -> FakeServer {
+ FakeServer::start(move |_| {
+ let guard = Dropped(dropped.clone());
+ async move {
+ let tail = stream::once(async move {
+ let _guard = guard;
+ pending::<Result<&str, Infallible>>().await
+ });
+ Response::new(Body::from_stream(
+ stream::once(async { Ok::<_, Infallible>("first") }).chain(tail),
+ ))
+ }
+ })
+ .await
+}
+
+#[tokio::test]
+async fn dropping_downstream_cancels_upstream_and_releases_admission() {
+ let dropped = Arc::new(Notify::new());
+ let upstream = stalled_provider(dropped.clone()).await;
+ let relay = provider(&upstream, 1);
+ let response = relay
+ .forward(
+ relay.try_admit().unwrap(),
+ resolved_request_context("provider-secret").await,
+ &HeaderMap::new(),
+ Bytes::new(),
+ )
+ .await
+ .unwrap();
+ let mut body = response.into_body().into_data_stream();
+ assert_eq!(body.next().await.unwrap().unwrap(), "first");
+ assert!(matches!(relay.try_admit(), Err(ProviderError::Busy)));
+ drop(body);
+ assert!(relay.try_admit().is_ok());
+ tokio::time::timeout(Duration::from_secs(1), dropped.notified())
+ .await
+ .expect("upstream body should be dropped on cancellation");
+}
+
+#[tokio::test]
+async fn deadlines_bound_headers_and_stalled_bodies_without_exposing_transport_details() {
+ let upstream = FakeServer::start(|_| async {
+ pending::<()>().await;
+ Response::new(Body::empty())
+ })
+ .await;
+ let relay = OpenAiProvider::for_test(
+ upstream.url.clone(),
+ ProviderLimits {
+ headers_timeout: Duration::from_millis(30),
+ ..ProviderLimits::default()
+ },
+ );
+ let response = relay
+ .forward(
+ relay.try_admit().unwrap(),
+ resolved_request_context("provider-secret").await,
+ &HeaderMap::new(),
+ Bytes::new(),
+ )
+ .await;
+ assert!(matches!(response, Err(ProviderError::Timeout)));
+ assert!(relay.try_admit().is_ok());
+
+ let dropped = Arc::new(Notify::new());
+ let upstream = stalled_provider(dropped.clone()).await;
+ let relay = OpenAiProvider::for_test(
+ upstream.url.clone(),
+ ProviderLimits {
+ execution_timeout: Duration::from_millis(100),
+ ..ProviderLimits::default()
+ },
+ );
+ let context = resolved_request_context("provider-secret").await;
+ let response = relay
+ .forward(
+ relay.try_admit().unwrap(),
+ context,
+ &HeaderMap::new(),
+ Bytes::new(),
+ )
+ .await
+ .unwrap();
+ // The deadline must cancel upstream even when downstream does not poll its body.
+ tokio::time::timeout(Duration::from_secs(1), dropped.notified())
+ .await
+ .expect("deadline must run without downstream polling");
+ let error = to_bytes(response.into_body(), 1024).await.unwrap_err();
+ assert!(!error.to_string().contains(&upstream.url));
+ assert!(!error.to_string().contains("provider-secret"));
+ assert!(relay.try_admit().is_ok());
+}
diff --git a/ai-gateway/src/resolution/contracts.rs b/ai-gateway/src/resolution/contracts.rs
index 94311d3afb..3474b7d616 100644
--- a/ai-gateway/src/resolution/contracts.rs
+++ b/ai-gateway/src/resolution/contracts.rs
@@ -1,4 +1,4 @@
-use super::{ResolveError, valid_token};
+use super::{ResolutionError, valid_token};
use serde::{
Deserialize, Deserializer, Serialize,
de::{DeserializeOwned, IntoDeserializer},
@@ -47,7 +47,7 @@ struct Auth {
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
-pub struct Connection {
+pub struct ProviderConnection {
id: String,
#[serde(rename = "provider", deserialize_with = "string_enum")]
_provider: Provider,
@@ -57,7 +57,7 @@ pub struct Connection {
auth: Auth,
}
-impl Connection {
+impl ProviderConnection {
pub fn id(&self) -> &str {
&self.id
}
@@ -74,7 +74,7 @@ impl Connection {
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
-pub struct Attribution {
+pub struct RequestAttribution {
organization_id: String,
project_id: String,
key_id: String,
@@ -82,7 +82,7 @@ pub struct Attribution {
provider_connection_id: String,
}
-impl Attribution {
+impl RequestAttribution {
pub fn organization_id(&self) -> &str {
&self.organization_id
}
@@ -102,14 +102,14 @@ impl Attribution {
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
-pub struct Ingestion {
+pub struct IngestionGrant {
access_token: String,
#[serde(rename = "token_type", deserialize_with = "string_enum")]
_token_type: TokenType,
expires_at: u64,
}
-impl Ingestion {
+impl IngestionGrant {
pub fn access_token(&self) -> &str {
&self.access_token
}
@@ -120,17 +120,17 @@ impl Ingestion {
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
-struct Response {
+struct ResolutionResponse {
version: u8,
- connection: Connection,
- attribution: Attribution,
+ connection: ProviderConnection,
+ attribution: RequestAttribution,
#[serde(deserialize_with = "string_enum")]
ingestion_mode: IngestionMode,
- ingestion: Ingestion,
+ ingestion: IngestionGrant,
}
/// A validated v1 execution. Construction is restricted to successful resolution.
-pub struct ResolvedExecution(Response);
+pub struct ResolvedRequestContext(ResolutionResponse);
// Web enums are JSON strings. Serde's externally tagged enums also accept objects.
fn string_enum<'de, D: Deserializer<'de>, T: DeserializeOwned>(
@@ -139,24 +139,25 @@ fn string_enum<'de, D: Deserializer<'de>, T: DeserializeOwned>(
T::deserialize(String::deserialize(deserializer)?.into_deserializer())
}
-impl ResolvedExecution {
- pub fn connection(&self) -> &Connection {
+impl ResolvedRequestContext {
+ pub fn connection(&self) -> &ProviderConnection {
&self.0.connection
}
- pub fn attribution(&self) -> &Attribution {
+ pub fn attribution(&self) -> &RequestAttribution {
&self.0.attribution
}
pub fn ingestion_mode(&self) -> IngestionMode {
self.0.ingestion_mode
}
- pub fn ingestion(&self) -> &Ingestion {
+ pub fn ingestion(&self) -> &IngestionGrant {
&self.0.ingestion
}
}
-impl fmt::Debug for ResolvedExecution {
+impl fmt::Debug for ResolvedRequestContext {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- f.debug_struct("ResolvedExecution").finish_non_exhaustive()
+ f.debug_struct("ResolvedRequestContext")
+ .finish_non_exhaustive()
}
}
@@ -164,9 +165,9 @@ pub(super) fn decode(
bytes: &[u8],
expected_format: ApiFormat,
now: u64,
-) -> Result<ResolvedExecution, ResolveError> {
- let response: Response =
- serde_json::from_slice(bytes).map_err(|_| ResolveError::InvalidResponse)?;
+) -> Result<ResolvedRequestContext, ResolutionError> {
+ let response: ResolutionResponse =
+ serde_json::from_slice(bytes).map_err(|_| ResolutionError::InvalidResponse)?;
let connection = &response.connection;
let attribution = &response.attribution;
if response.version != 1
@@ -185,7 +186,7 @@ pub(super) fn decode(
.iter()
.any(|value| value.trim().is_empty())
{
- return Err(ResolveError::InvalidResponse);
+ return Err(ResolutionError::InvalidResponse);
}
- Ok(ResolvedExecution(response))
+ Ok(ResolvedRequestContext(response))
}
diff --git a/ai-gateway/src/resolution/mod.rs b/ai-gateway/src/resolution/mod.rs
index 6094d0937b..83d1c61f19 100644
--- a/ai-gateway/src/resolution/mod.rs
+++ b/ai-gateway/src/resolution/mod.rs
@@ -3,7 +3,8 @@ mod contracts;
mod signing;
pub use contracts::{
- ApiFormat, Attribution, Connection, Ingestion, IngestionMode, MetadataValue, ResolvedExecution,
+ ApiFormat, IngestionGrant, IngestionMode, MetadataValue, ProviderConnection,
+ RequestAttribution, ResolvedRequestContext,
};
use reqwest::{
Client, Url,
@@ -17,27 +18,27 @@ use std::{
const RESOLVE_PATH: &str = "/api/internal/ai-gateway/v1/resolve";
-struct Limits {
+struct ResolutionLimits {
timeout: Duration,
max_response_bytes: usize,
}
/// Operator-supplied Web base URL and the existing gateway/Web service key.
-pub struct ResolverConfig {
+pub struct ControlPlaneConfig {
url: Url,
service_key: String,
- limits: Limits,
+ limits: ResolutionLimits,
}
-impl ResolverConfig {
+impl ControlPlaneConfig {
/// Configure resolution against a trusted Web base URL, including any deployment prefix.
///
/// # Errors
- /// Returns [`ResolveError::Configuration`] for a blank service key or an invalid
+ /// Returns [`ResolutionError::Configuration`] for a blank service key or an invalid
/// base URL. URLs require HTTPS (HTTP is allowed on loopback), with no userinfo,
/// query, or fragment.
- pub fn new(web_url: &str, service_key: &str) -> Result<Self, ResolveError> {
- let mut url = Url::parse(web_url).map_err(|_| ResolveError::Configuration)?;
+ pub fn new(web_url: &str, service_key: &str) -> Result<Self, ResolutionError> {
+ let mut url = Url::parse(web_url).map_err(|_| ResolutionError::Configuration)?;
let loopback = url.host_str().is_some_and(|host| {
host == "localhost"
|| host
@@ -53,14 +54,14 @@ impl ResolverConfig {
|| url.fragment().is_some()
|| service_key.trim().is_empty()
{
- return Err(ResolveError::Configuration);
+ return Err(ResolutionError::Configuration);
}
let resolve_path = format!("{}{RESOLVE_PATH}", url.path().trim_end_matches('/'));
url.set_path(&resolve_path);
Ok(Self {
url,
service_key: service_key.to_owned(),
- limits: Limits {
+ limits: ResolutionLimits {
timeout: Duration::from_secs(5),
max_response_bytes: 256 * 1024,
},
@@ -69,17 +70,17 @@ impl ResolverConfig {
}
/// Reuses the HTTP connection pool; credentials belong exclusively to each request.
-pub struct Resolver {
+pub struct ControlPlaneClient {
client: Client,
- config: ResolverConfig,
+ config: ControlPlaneConfig,
}
-impl Resolver {
+impl ControlPlaneClient {
/// Build a resolver with a reusable HTTP connection pool.
///
/// # Errors
- /// Returns [`ResolveError::Configuration`] if the HTTP client cannot be initialized.
- pub fn new(config: ResolverConfig) -> Result<Self, ResolveError> {
+ /// Returns [`ResolutionError::Configuration`] if the HTTP client cannot be initialized.
+ pub fn new(config: ControlPlaneConfig) -> Result<Self, ResolutionError> {
let client = Client::builder()
.redirect(reqwest::redirect::Policy::none())
.retry(reqwest::retry::never())
@@ -89,24 +90,24 @@ impl Resolver {
.no_zstd()
.no_deflate()
.build()
- .map_err(|_| ResolveError::Configuration)?;
+ .map_err(|_| ResolutionError::Configuration)?;
Ok(Self { client, config })
}
/// Resolve a gateway credential and API format into a validated execution contract.
///
/// # Errors
- /// Returns a sanitized [`ResolveError`] for invalid credentials, authentication or
+ /// Returns a sanitized [`ResolutionError`] for invalid credentials, authentication or
/// authorization failures, missing routes, unavailable Web or system time, request
/// construction or transport failures, timeouts, or oversized or invalid responses.
pub async fn resolve(
&self,
gateway_key: &str,
api_format: ApiFormat,
- ) -> Result<ResolvedExecution, ResolveError> {
+ ) -> Result<ResolvedRequestContext, ResolutionError> {
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
- .map_err(|_| ResolveError::Unavailable)?;
+ .map_err(|_| ResolutionError::Unavailable)?;
self.resolve_at(gateway_key, api_format, timestamp).await
}
@@ -115,9 +116,9 @@ impl Resolver {
gateway_key: &str,
api_format: ApiFormat,
timestamp: Duration,
- ) -> Result<ResolvedExecution, ResolveError> {
+ ) -> Result<ResolvedRequestContext, ResolutionError> {
if gateway_key.len() > 8192 || !valid_token(gateway_key) {
- return Err(ResolveError::InvalidCredential);
+ return Err(ResolutionError::InvalidCredential);
}
let started = Instant::now();
let body = tokio::time::timeout(
@@ -125,7 +126,7 @@ impl Resolver {
self.fetch(gateway_key, api_format, timestamp.as_secs()),
)
.await
- .map_err(|_| ResolveError::Timeout)??;
+ .map_err(|_| ResolutionError::Timeout)??;
contracts::decode(
&body,
api_format,
@@ -138,19 +139,19 @@ impl Resolver {
gateway_key: &str,
api_format: ApiFormat,
timestamp: u64,
- ) -> Result<Vec<u8>, ResolveError> {
+ ) -> Result<Vec<u8>, ResolutionError> {
let mut credential = HeaderValue::from_str(&format!("Bearer {gateway_key}"))
- .map_err(|_| ResolveError::InvalidCredential)?;
+ .map_err(|_| ResolutionError::InvalidCredential)?;
credential.set_sensitive(true);
let mut signature = HeaderValue::from_str(&signing::authorization(
&self.config.service_key,
gateway_key,
timestamp,
))
- .map_err(|_| ResolveError::Configuration)?;
+ .map_err(|_| ResolutionError::Configuration)?;
signature.set_sensitive(true);
let body = serde_json::to_vec(&serde_json::json!({ "apiFormat": api_format }))
- .map_err(|_| ResolveError::Configuration)?;
+ .map_err(|_| ResolutionError::Configuration)?;
let mut response = self
.client
.post(self.config.url.clone())
@@ -160,30 +161,30 @@ impl Resolver {
.body(body)
.send()
.await
- .map_err(|_| ResolveError::Transport)?;
+ .map_err(|_| ResolutionError::Transport)?;
match response.status().as_u16() {
200 => {}
- 401 => return Err(ResolveError::Authentication),
- 403 => return Err(ResolveError::Forbidden),
- 404 => return Err(ResolveError::NoRoute),
- 429 | 500..=599 => return Err(ResolveError::Unavailable),
- _ => return Err(ResolveError::InvalidResponse),
+ 401 => return Err(ResolutionError::Authentication),
+ 403 => return Err(ResolutionError::Forbidden),
+ 404 => return Err(ResolutionError::NoRoute),
+ 429 | 500..=599 => return Err(ResolutionError::Unavailable),
+ _ => return Err(ResolutionError::InvalidResponse),
}
let limit = self.config.limits.max_response_bytes;
if response
.content_length()
.is_some_and(|size| size > limit as u64)
{
- return Err(ResolveError::ResponseTooLarge);
+ return Err(ResolutionError::ResponseTooLarge);
}
let mut bytes = Vec::new();
while let Some(chunk) = response
.chunk()
.await
- .map_err(|_| ResolveError::Transport)?
+ .map_err(|_| ResolutionError::Transport)?
{
if chunk.len() > limit - bytes.len() {
- return Err(ResolveError::ResponseTooLarge);
+ return Err(ResolutionError::ResponseTooLarge);
}
bytes.extend_from_slice(&chunk);
}
@@ -197,7 +198,7 @@ fn valid_token(value: &str) -> bool {
/// Sanitized failure categories. Upstream bodies and transport errors are never retained.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum ResolveError {
+pub enum ResolutionError {
Configuration,
InvalidCredential,
Authentication,
@@ -210,7 +211,7 @@ pub enum ResolveError {
ResponseTooLarge,
}
-impl fmt::Display for ResolveError {
+impl fmt::Display for ResolutionError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
Self::Configuration => "invalid resolver configuration",
@@ -227,7 +228,7 @@ impl fmt::Display for ResolveError {
}
}
-impl std::error::Error for ResolveError {}
+impl std::error::Error for ResolutionError {}
#[cfg(test)]
mod tests;
diff --git a/ai-gateway/src/resolution/tests.rs b/ai-gateway/src/resolution/tests.rs
index 0873822f96..ef74457584 100644
--- a/ai-gateway/src/resolution/tests.rs
+++ b/ai-gateway/src/resolution/tests.rs
@@ -54,17 +54,21 @@ impl FakeWeb {
Self { url, calls, task }
}
- fn resolver(&self) -> Resolver {
- self.resolver_with_limits(Duration::from_secs(2), 256 * 1024)
+ fn control_plane(&self) -> ControlPlaneClient {
+ self.control_plane_with_limits(Duration::from_secs(2), 256 * 1024)
}
- fn resolver_with_limits(&self, timeout: Duration, max_response_bytes: usize) -> Resolver {
- let mut config = ResolverConfig::new(&self.url, SERVICE_KEY).unwrap();
- config.limits = Limits {
+ fn control_plane_with_limits(
+ &self,
+ timeout: Duration,
+ max_response_bytes: usize,
+ ) -> ControlPlaneClient {
+ let mut config = ControlPlaneConfig::new(&self.url, SERVICE_KEY).unwrap();
+ config.limits = ResolutionLimits {
timeout,
max_response_bytes,
};
- Resolver::new(config).unwrap()
+ ControlPlaneClient::new(config).unwrap()
}
fn calls(&self) -> usize {
@@ -120,7 +124,7 @@ async fn signs_the_exact_key_and_sends_only_the_api_format() {
}).await;
let context = web
- .resolver()
+ .control_plane()
.resolve_at(GATEWAY_KEY, ApiFormat::OpenAiResponses, NOW_TIME)
.await
.unwrap();
@@ -165,15 +169,15 @@ async fn denied_and_failed_resolutions_are_classified_without_retries_or_body_le
})
.await;
let error = web
- .resolver()
+ .control_plane()
.resolve_at(GATEWAY_KEY, ApiFormat::OpenAiResponses, NOW_TIME)
.await
.unwrap_err();
assert!(match status {
- 401 => matches!(error, ResolveError::Authentication),
- 403 => matches!(error, ResolveError::Forbidden),
- 404 => matches!(error, ResolveError::NoRoute),
- _ => matches!(error, ResolveError::Unavailable),
+ 401 => matches!(error, ResolutionError::Authentication),
+ 403 => matches!(error, ResolutionError::Forbidden),
+ 404 => matches!(error, ResolutionError::NoRoute),
+ _ => matches!(error, ResolutionError::Unavailable),
});
assert!(!format!("{error:?} {error}").contains("private-upstream-error"));
assert_eq!(web.calls(), 1);
@@ -197,7 +201,7 @@ async fn never_follows_redirects_with_credentials() {
})
.await;
assert!(
- web.resolver()
+ web.control_plane()
.resolve_at(GATEWAY_KEY, ApiFormat::OpenAiResponses, NOW_TIME)
.await
.is_err()
@@ -223,10 +227,10 @@ async fn rejects_oversized_declared_and_chunked_responses() {
})
.await;
let result = web
- .resolver_with_limits(Duration::from_secs(2), 128)
+ .control_plane_with_limits(Duration::from_secs(2), 128)
.resolve_at(GATEWAY_KEY, ApiFormat::OpenAiResponses, NOW_TIME)
.await;
- assert!(matches!(result, Err(ResolveError::ResponseTooLarge)));
+ assert!(matches!(result, Err(ResolutionError::ResponseTooLarge)));
assert_eq!(web.calls(), 1);
}
}
@@ -250,12 +254,12 @@ async fn deadline_covers_response_headers_and_body() {
.await;
let result = tokio::time::timeout(
Duration::from_millis(500),
- web.resolver_with_limits(Duration::from_millis(30), 256 * 1024)
+ web.control_plane_with_limits(Duration::from_millis(30), 256 * 1024)
.resolve_at(GATEWAY_KEY, ApiFormat::OpenAiResponses, NOW_TIME),
)
.await
.expect("resolver must enforce its deadline");
- assert!(matches!(result, Err(ResolveError::Timeout)));
+ assert!(matches!(result, Err(ResolutionError::Timeout)));
assert_eq!(web.calls(), 1);
}
}
@@ -265,10 +269,10 @@ async fn malformed_credentials_never_reach_web() {
let web = FakeWeb::start(|_| async { response(success("project-1", "provider-token")) }).await;
for key in ["", "has space", "has\r\nheader", "has\ttab"] {
let result = web
- .resolver()
+ .control_plane()
.resolve_at(key, ApiFormat::OpenAiResponses, NOW_TIME)
.await;
- assert!(matches!(result, Err(ResolveError::InvalidCredential)));
+ assert!(matches!(result, Err(ResolutionError::InvalidCredential)));
}
assert_eq!(web.calls(), 0);
}
@@ -283,14 +287,14 @@ async fn rejects_a_grant_that_expires_while_resolving_across_a_second_boundary()
})
.await;
let result = web
- .resolver()
+ .control_plane()
.resolve_at(
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
NOW_TIME + Duration::from_millis(900),
)
.await;
- assert!(matches!(result, Err(ResolveError::InvalidResponse)));
+ assert!(matches!(result, Err(ResolutionError::InvalidResponse)));
}
#[tokio::test]
@@ -307,10 +311,10 @@ async fn concurrent_resolutions_keep_credentials_and_contexts_isolated() {
}
})
.await;
- let resolver = web.resolver();
+ let control_plane = web.control_plane();
let (alice, bob) = tokio::join!(
- resolver.resolve_at("gw_test_alice", ApiFormat::OpenAiResponses, NOW_TIME),
- resolver.resolve_at("gw_test_bob", ApiFormat::OpenAiResponses, NOW_TIME),
+ control_plane.resolve_at("gw_test_alice", ApiFormat::OpenAiResponses, NOW_TIME),
+ control_plane.resolve_at("gw_test_bob", ApiFormat::OpenAiResponses, NOW_TIME),
);
let alice = alice.unwrap();
let bob = bob.unwrap();
@@ -386,10 +390,10 @@ async fn assert_invalid_context(body: Value) {
})
.await;
let result = web
- .resolver()
+ .control_plane()
.resolve_at(GATEWAY_KEY, ApiFormat::OpenAiResponses, NOW_TIME)
.await;
- assert!(matches!(result, Err(ResolveError::InvalidResponse)));
+ assert!(matches!(result, Err(ResolutionError::InvalidResponse)));
assert_eq!(web.calls(), 1);
}
@@ -400,11 +404,11 @@ async fn rejects_malformed_json_without_exposing_response_contents() {
})
.await;
let error = web
- .resolver()
+ .control_plane()
.resolve_at(GATEWAY_KEY, ApiFormat::OpenAiResponses, NOW_TIME)
.await
.unwrap_err();
- assert!(matches!(error, ResolveError::InvalidResponse));
+ assert!(matches!(error, ResolutionError::InvalidResponse));
assert!(!format!("{error:?} {error}").contains("private-provider-secret"));
}
@@ -412,8 +416,8 @@ async fn rejects_malformed_json_without_exposing_response_contents() {
fn web_configuration_rejects_unsafe_or_ambiguous_urls() {
for key in ["", " \n\t"] {
assert!(matches!(
- ResolverConfig::new("https://web.example", key),
- Err(ResolveError::Configuration)
+ ControlPlaneConfig::new("https://web.example", key),
+ Err(ResolutionError::Configuration)
));
}
for url in [
@@ -424,8 +428,8 @@ fn web_configuration_rejects_unsafe_or_ambiguous_urls() {
"https://web.example#fragment",
] {
assert!(matches!(
- ResolverConfig::new(url, SERVICE_KEY),
- Err(ResolveError::Configuration)
+ ControlPlaneConfig::new(url, SERVICE_KEY),
+ Err(ResolutionError::Configuration)
));
}
for url in [
@@ -433,6 +437,6 @@ fn web_configuration_rejects_unsafe_or_ambiguous_urls() {
"http://127.0.0.1:3000",
"http://[::1]:3000",
] {
- assert!(ResolverConfig::new(url, SERVICE_KEY).is_ok());
+ assert!(ControlPlaneConfig::new(url, SERVICE_KEY).is_ok());
}
}
diff --git a/ai-gateway/src/server.rs b/ai-gateway/src/server.rs
index d0cd55e7be..e0a65e9d89 100644
--- a/ai-gateway/src/server.rs
+++ b/ai-gateway/src/server.rs
@@ -13,13 +13,22 @@ use serde::Serialize;
use tokio::{net::TcpListener, sync::oneshot};
#[derive(Clone, Default)]
-pub struct AppState {
+pub struct GatewayLifecycleState {
ready: Arc<AtomicBool>,
+ disabled: bool,
}
-impl AppState {
+impl GatewayLifecycleState {
pub fn is_ready(&self) -> bool {
- self.ready.load(Ordering::Acquire)
+ !self.disabled && self.ready.load(Ordering::Acquire)
+ }
+
+ /// Keep liveness available without advertising inference readiness.
+ pub fn unconfigured() -> Self {
+ Self {
+ disabled: true,
+ ..Self::default()
+ }
}
}
@@ -28,20 +37,26 @@ struct Probe {
status: &'static str,
}
-pub fn router(state: AppState) -> Router {
+pub fn router(state: GatewayLifecycleState) -> Router {
Router::new()
.route("/health", get(|| async { Json(Probe { status: "ok" }) }))
.route("/ready", get(readiness))
.with_state(state)
}
-async fn readiness(State(state): State<AppState>) -> (StatusCode, Json<Probe>) {
+async fn readiness(State(state): State<GatewayLifecycleState>) -> (StatusCode, Json<Probe>) {
if state.is_ready() {
(StatusCode::OK, Json(Probe { status: "ready" }))
} else {
(
StatusCode::SERVICE_UNAVAILABLE,
- Json(Probe { status: "draining" }),
+ Json(Probe {
+ status: if state.disabled {
+ "unconfigured"
+ } else {
+ "draining"
+ },
+ }),
)
}
}
@@ -55,7 +70,7 @@ async fn readiness(State(state): State<AppState>) -> (StatusCode, Json<Probe>) {
pub async fn serve(
listener: TcpListener,
app: Router,
- state: AppState,
+ state: GatewayLifecycleState,
shutdown: impl Future<Output = ()> + Send + 'static,
shutdown_timeout: Duration,
) -> io::Result<()> {
diff --git a/ai-gateway/src/test_support.rs b/ai-gateway/src/test_support.rs
new file mode 100644
index 0000000000..a9541d26f3
--- /dev/null
+++ b/ai-gateway/src/test_support.rs
@@ -0,0 +1,102 @@
+use std::{
+ future::Future,
+ sync::{
+ Arc,
+ atomic::{AtomicUsize, Ordering},
+ },
+ time::{SystemTime, UNIX_EPOCH},
+};
+
+use axum::{
+ Router,
+ body::Body,
+ http::{Request, Response},
+ routing::any,
+};
+use serde_json::json;
+use tokio::{net::TcpListener, task::JoinHandle};
+
+use crate::resolution::{
+ ApiFormat, ControlPlaneClient, ControlPlaneConfig, ResolvedRequestContext,
+};
+
+pub(crate) struct FakeServer {
+ pub url: String,
+ calls: Arc<AtomicUsize>,
+ task: JoinHandle<()>,
+}
+
+impl FakeServer {
+ pub async fn start<F, Fut>(handler: F) -> Self
+ where
+ F: Fn(Request<Body>) -> Fut + Clone + Send + Sync + 'static,
+ Fut: Future<Output = Response<Body>> + Send + 'static,
+ {
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let url = format!("http://{}", listener.local_addr().unwrap());
+ let calls = Arc::new(AtomicUsize::new(0));
+ let count = calls.clone();
+ let app = Router::new().fallback(any(move |request| {
+ count.fetch_add(1, Ordering::SeqCst);
+ handler(request)
+ }));
+ let task = tokio::spawn(async move {
+ axum::serve(listener, app).await.unwrap();
+ });
+ Self { url, calls, task }
+ }
+
+ pub fn calls(&self) -> usize {
+ self.calls.load(Ordering::SeqCst)
+ }
+
+ pub fn control_plane(&self) -> ControlPlaneClient {
+ ControlPlaneClient::new(ControlPlaneConfig::new(&self.url, "test-service-key").unwrap())
+ .unwrap()
+ }
+}
+
+impl Drop for FakeServer {
+ fn drop(&mut self) {
+ self.task.abort();
+ }
+}
+
+pub(crate) fn resolution_response(provider_token: &str) -> Response<Body> {
+ let expires_at = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap()
+ .as_secs()
+ + 300;
+ let body = json!({
+ "version": 1,
+ "connection": {
+ "id": "connection-1", "provider": "openai", "api_format": "openai.responses",
+ "base_url": "https://api.openai.com/v1",
+ "auth": {"type": "Bearer", "token": provider_token}
+ },
+ "attribution": {
+ "organization_id": "org-1", "project_id": "project-1", "key_id": "key-1", "key_metadata": {},
+ "provider_connection_id": "provider-connection-1"
+ },
+ "ingestion_mode": "usage",
+ "ingestion": {"access_token": "private-ingestion-token", "token_type": "Bearer", "expires_at": expires_at}
+ });
+ Response::builder()
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .unwrap()
+}
+
+pub(crate) async fn resolved_request_context(provider_token: &str) -> ResolvedRequestContext {
+ let token = provider_token.to_owned();
+ let web = FakeServer::start(move |_| {
+ let token = token.clone();
+ async move { resolution_response(&token) }
+ })
+ .await;
+ web.control_plane()
+ .resolve("gateway-secret", ApiFormat::OpenAiResponses)
+ .await
+ .unwrap()
+}
diff --git a/ai-gateway/src/transport/mod.rs b/ai-gateway/src/transport/mod.rs
new file mode 100644
index 0000000000..33cc6614ea
--- /dev/null
+++ b/ai-gateway/src/transport/mod.rs
@@ -0,0 +1,282 @@
+//! Bounded byte relay. One task owns upstream reads; the body owns its lifetime.
+use std::{
+ fmt,
+ pin::Pin,
+ sync::{
+ Arc, Mutex,
+ atomic::{AtomicBool, Ordering},
+ },
+ task::{Context, Poll},
+};
+
+use axum::{
+ body::{Body, Bytes},
+ http::{HeaderMap, header},
+};
+use futures_util::{Stream, StreamExt};
+use tokio::{
+ sync::{Notify, mpsc},
+ task::JoinHandle,
+ time::Instant,
+};
+
+/// Sanitized failures; upstream URLs, bodies and credentials are never retained.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ProviderError {
+ Configuration,
+ Busy,
+ Timeout,
+ Transport,
+}
+
+impl fmt::Display for ProviderError {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.write_str(match self {
+ Self::Configuration => "invalid provider configuration",
+ Self::Busy => "gateway execution capacity exhausted",
+ Self::Timeout => "provider deadline exceeded",
+ Self::Transport => "provider transport failed",
+ })
+ }
+}
+impl std::error::Error for ProviderError {}
+
+// An allowlist prevents gateway credentials, tenant routing overrides and cookies
+// from crossing the boundary. Headers nominated by `Connection` are never end-to-end.
+fn selected_headers(source: &HeaderMap, allowed: &[&'static str]) -> HeaderMap {
+ let mut selected = HeaderMap::new();
+ for &name in allowed {
+ let hop_by_hop = source.get_all(header::CONNECTION).iter().any(|value| {
+ value.to_str().map_or(true, |value| {
+ value
+ .split(',')
+ .any(|token| token.trim().eq_ignore_ascii_case(name))
+ })
+ });
+ if !hop_by_hop {
+ for value in source.get_all(name) {
+ selected.append(header::HeaderName::from_static(name), value.clone());
+ }
+ }
+ }
+ selected
+}
+
+pub(crate) fn request_headers(source: &HeaderMap) -> HeaderMap {
+ selected_headers(
+ source,
+ &[
+ "content-type",
+ "content-encoding",
+ "accept",
+ "accept-encoding",
+ ],
+ )
+}
+
+pub(crate) fn response_headers(source: &HeaderMap) -> HeaderMap {
+ selected_headers(
+ source,
+ &[
+ "content-type",
+ "content-encoding",
+ "cache-control",
+ "retry-after",
+ "x-request-id",
+ "openai-processing-ms",
+ "openai-version",
+ "x-ratelimit-limit-requests",
+ "x-ratelimit-limit-tokens",
+ "x-ratelimit-remaining-requests",
+ "x-ratelimit-remaining-tokens",
+ "x-ratelimit-reset-requests",
+ "x-ratelimit-reset-tokens",
+ ],
+ )
+}
+
+pub(crate) fn relay<T: Send + 'static>(
+ upstream: reqwest::Response,
+ deadline: Instant,
+ owner: T,
+) -> Body {
+ relay_stream(
+ upstream
+ .bytes_stream()
+ .map(|chunk| chunk.map_err(|_| ProviderError::Transport)),
+ deadline,
+ owner,
+ )
+}
+
+fn relay_stream<S, T>(upstream: S, deadline: Instant, owner: T) -> Body
+where
+ S: Stream<Item = Result<Bytes, ProviderError>> + Send + 'static,
+ T: Send + 'static,
+{
+ let (sender, receiver) = mpsc::channel(1);
+ let failed = Arc::new(AtomicBool::new(false));
+ let failure = failed.clone();
+ let resources = Arc::new(StreamResources {
+ owner: Mutex::new(Some(owner)),
+ released: Notify::new(),
+ });
+ let pump_resources = resources.clone();
+ let task = tokio::spawn(async move {
+ let pump = async {
+ tokio::pin!(upstream);
+ while let Some(chunk) = upstream.next().await {
+ let chunk = chunk?;
+ // One queued chunk plus one pending send; no per-chunk tasks.
+ for bytes in chunk.chunks(64 * 1024) {
+ if sender.send(Bytes::copy_from_slice(bytes)).await.is_err() {
+ return Ok(());
+ }
+ }
+ }
+ Ok::<_, ProviderError>(())
+ };
+ if !matches!(tokio::time::timeout_at(deadline, pump).await, Ok(Ok(()))) {
+ failure.store(true, Ordering::Release);
+ pump_resources.release();
+ }
+ drop(sender);
+ // Upstream EOF alone does not release admission: the downstream may still
+ // have queued bytes. The deadline also covers an unpolled downstream body.
+ if tokio::time::timeout_at(deadline, pump_resources.released.notified())
+ .await
+ .is_err()
+ {
+ failure.store(true, Ordering::Release);
+ pump_resources.release();
+ }
+ });
+ Body::from_stream(ResponseStream {
+ receiver,
+ task,
+ failed,
+ resources,
+ done: false,
+ })
+}
+
+struct StreamResources<T> {
+ owner: Mutex<Option<T>>,
+ released: Notify,
+}
+
+impl<T> StreamResources<T> {
+ fn release(&self) {
+ self.owner.lock().expect("relay owner lock poisoned").take();
+ self.released.notify_one();
+ }
+}
+
+struct ResponseStream<T> {
+ receiver: mpsc::Receiver<Bytes>,
+ task: JoinHandle<()>,
+ failed: Arc<AtomicBool>,
+ resources: Arc<StreamResources<T>>,
+ done: bool,
+}
+
+impl<T> Stream for ResponseStream<T> {
+ type Item = Result<Bytes, ProviderError>;
+
+ fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+ let this = self.get_mut();
+ if this.done {
+ return Poll::Ready(None);
+ }
+ match this.receiver.poll_recv(cx) {
+ Poll::Ready(None) => {
+ this.done = true;
+ this.resources.release();
+ // The sender publishes a failure before closing its channel.
+ Poll::Ready(
+ this.failed
+ .load(Ordering::Acquire)
+ .then_some(Err(ProviderError::Transport)),
+ )
+ }
+ other => other.map(|value| value.map(Ok)),
+ }
+ }
+}
+
+impl<T> Drop for ResponseStream<T> {
+ fn drop(&mut self) {
+ self.task.abort();
+ self.resources.release();
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use axum::body::to_bytes;
+ use futures_util::stream;
+ use std::{sync::atomic::AtomicUsize, time::Duration};
+
+ struct Owner(Arc<Notify>);
+ impl Drop for Owner {
+ fn drop(&mut self) {
+ self.0.notify_one();
+ }
+ }
+
+ #[tokio::test]
+ async fn slow_downstream_bounds_read_ahead_and_drop_releases_owner() {
+ let polled = Arc::new(AtomicUsize::new(0));
+ let count = polled.clone();
+ let upstream = stream::iter(0..100).map(move |_| {
+ count.fetch_add(1, Ordering::SeqCst);
+ Ok(Bytes::from(vec![b'x'; 128 * 1024]))
+ });
+ let released = Arc::new(Notify::new());
+ let body = relay_stream(
+ upstream,
+ Instant::now() + Duration::from_secs(10),
+ Owner(released.clone()),
+ );
+ tokio::task::yield_now().await;
+ assert_eq!(polled.load(Ordering::SeqCst), 1);
+ drop(body);
+ tokio::time::timeout(Duration::from_secs(1), released.notified())
+ .await
+ .unwrap();
+ }
+
+ #[tokio::test]
+ async fn queued_bytes_are_delivered_before_an_upstream_error() {
+ let released = Arc::new(Notify::new());
+ let bytes = Bytes::from_static(b"event: response.output_text.delta\ndata: hello\n\n");
+ let body = relay_stream(
+ stream::iter([Ok(bytes.clone()), Err(ProviderError::Transport)]),
+ Instant::now() + Duration::from_secs(1),
+ Owner(released.clone()),
+ );
+ // Wait until the pump has published its failure with the successful chunk queued.
+ tokio::time::timeout(Duration::from_secs(1), released.notified())
+ .await
+ .unwrap();
+ let mut downstream = body.into_data_stream();
+ assert_eq!(downstream.next().await.unwrap().unwrap(), bytes);
+ assert!(downstream.next().await.unwrap().is_err());
+ assert!(downstream.next().await.is_none());
+ }
+
+ #[tokio::test]
+ async fn deadline_releases_owner_even_after_upstream_eof_with_unpolled_body() {
+ let released = Arc::new(Notify::new());
+ let body = relay_stream(
+ stream::once(async { Ok(Bytes::from_static(b"final bytes")) }),
+ Instant::now() + Duration::from_millis(30),
+ Owner(released.clone()),
+ );
+ tokio::time::timeout(Duration::from_secs(1), released.notified())
+ .await
+ .unwrap();
+ assert!(to_bytes(body, 1024).await.is_err());
+ }
+}
diff --git a/ai-gateway/tests/lifecycle.rs b/ai-gateway/tests/lifecycle.rs
index 613d92eac0..b7f12784c9 100644
--- a/ai-gateway/tests/lifecycle.rs
+++ b/ai-gateway/tests/lifecycle.rs
@@ -1,6 +1,6 @@
mod support;
-use ai_gateway::server::{self, AppState};
+use ai_gateway::server::{self, GatewayLifecycleState};
use axum::{
Router,
body::{Body, to_bytes},
@@ -17,7 +17,7 @@ use tower::ServiceExt;
#[tokio::test]
async fn probes_and_unimplemented_routes_over_http() {
- let state = AppState::default();
+ let state = GatewayLifecycleState::default();
let mut server =
TestServer::start(server::router(state.clone()), state, Duration::from_secs(1)).await;
let health = support::get(server.address, "/health").await;
@@ -43,7 +43,7 @@ async fn probes_and_unimplemented_routes_over_http() {
async fn shutdown_marks_unready_and_waits_for_in_flight_request() {
let entered = Arc::new(Notify::new());
let release = Arc::new(Notify::new());
- let state = AppState::default();
+ let state = GatewayLifecycleState::default();
let app = server::router(state.clone()).route(
"/slow",
get({
@@ -106,7 +106,12 @@ async fn shutdown_deadline_bounds_a_stuck_handler() {
}
}),
);
- let mut server = TestServer::start(app, AppState::default(), Duration::from_millis(50)).await;
+ let mut server = TestServer::start(
+ app,
+ GatewayLifecycleState::default(),
+ Duration::from_millis(50),
+ )
+ .await;
let request = tokio::spawn(support::get(server.address, "/stuck"));
tokio::time::timeout(Duration::from_secs(2), entered.notified())
.await
@@ -139,7 +144,12 @@ async fn fake_dependency_records_requests_and_returns_scripted_response() {
}
}
});
- let mut fake = TestServer::start(app, AppState::default(), Duration::from_secs(1)).await;
+ let mut fake = TestServer::start(
+ app,
+ GatewayLifecycleState::default(),
+ Duration::from_secs(1),
+ )
+ .await;
let response = support::get(fake.address, "/example").await;
assert!(response.starts_with("HTTP/1.1 429"));
assert!(response.contains("retry-after: 1\r\n"));
diff --git a/ai-gateway/tests/startup.rs b/ai-gateway/tests/startup.rs
index 6aee192d52..ba9ff20057 100644
--- a/ai-gateway/tests/startup.rs
+++ b/ai-gateway/tests/startup.rs
@@ -9,6 +9,9 @@ fn invalid_configuration_exits_without_disclosing_the_value() {
"LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS",
"LANGFUSE_LOG_LEVEL",
"LANGFUSE_LOG_FORMAT",
+ "LANGFUSE_AI_GATEWAY_WEB_URL",
+ "LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS",
+ "LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS",
] {
let output = Command::new(env!("CARGO_BIN_EXE_ai-gateway"))
.env_clear()
diff --git a/ai-gateway/tests/support/mod.rs b/ai-gateway/tests/support/mod.rs
index 00ebf2d74f..8180ebf130 100644
--- a/ai-gateway/tests/support/mod.rs
+++ b/ai-gateway/tests/support/mod.rs
@@ -1,6 +1,6 @@
use std::{io, net::SocketAddr, time::Duration};
-use ai_gateway::server::{self, AppState};
+use ai_gateway::server::{self, GatewayLifecycleState};
use axum::Router;
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
@@ -11,13 +11,13 @@ use tokio::{
pub struct TestServer {
pub address: SocketAddr,
- pub state: AppState,
+ pub state: GatewayLifecycleState,
shutdown: Option<oneshot::Sender<()>>,
task: Option<JoinHandle<io::Result<()>>>,
}
impl TestServer {
- pub async fn start(app: Router, state: AppState, budget: Duration) -> Self {
+ pub async fn start(app: Router, state: GatewayLifecycleState, budget: Duration) -> Self {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let (shutdown, receiver) = oneshot::channel();