🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
use crate::resolution::ControlPlaneConfig;
use std::{env, net::SocketAddr, ops::RangeInclusive, time::Duration};
use tracing::level_filters::LevelFilter;
pub struct GatewayConfig {
pub control_plane: Option<ControlPlaneConfig>,
pub listen_address: SocketAddr,
pub auto_increment_listen_port: bool,
/// Connection drain after the shutdown signal. Behind an ALB, deregistration has
/// already drained streams on SIGTERM; on Kubernetes this drain is the only one.
pub connection_drain_timeout: Duration,
/// The connection drain timeout was read from its deprecated name.
pub uses_deprecated_connection_drain_timeout_name: bool,
/// Budget for flushing telemetry, starting once connection draining ends. The
/// orchestrator grace period must cover both timeouts (ECS `stopTimeout` is at most 120 s).
pub telemetry_shutdown_timeout: Duration,
pub log_level: LevelFilter,
pub log_format: LogFormat,
pub telemetry_buffer_bytes: usize,
pub request_limits: RequestLimits,
pub execution_limits: ExecutionLimits,
}
const MIB: usize = 1024 * 1024;
const SEMAPHORE_CAPACITY: usize = tokio::sync::Semaphore::MAX_PERMITS;
pub const DEFAULT_MAX_REQUEST_BYTES: usize = 16 * MIB;
const DEFAULT_MEMORY_BUDGET_BYTES: usize = 1024 * MIB;
/// Covers response capture (up to 1 MiB retained output plus one inspected event of up
/// to 1 MiB) and connection buffers. Staging responses average 8 KB (Anthropic) to
/// 210 KB (`OpenAI`), so this is above the typical request.
const DEFAULT_REQUEST_OVERHEAD_BYTES: usize = 2 * MIB;
const DEFAULT_MAX_EXECUTION_SECONDS: u32 = 10 * 60;
const DEFAULT_STALL_TIMEOUT_SECONDS: u32 = 120;
const DEFAULT_SHUTDOWN_TIMEOUT_SECONDS: u64 = 10;
/// Memory bounds per instance. Admission is limited only by the memory budget.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct RequestLimits {
pub max_request_bytes: usize,
/// Bytes all in-flight requests may reserve together.
pub memory_budget_bytes: usize,
/// Fixed bytes every in-flight request reserves on top of its body.
pub request_overhead_bytes: usize,
}
impl Default for RequestLimits {
fn default() -> Self {
Self {
max_request_bytes: DEFAULT_MAX_REQUEST_BYTES,
memory_budget_bytes: DEFAULT_MEMORY_BUDGET_BYTES,
request_overhead_bytes: DEFAULT_REQUEST_OVERHEAD_BYTES,
}
}
}
/// Provider execution deadlines.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ExecutionLimits {
/// Ceiling from execution admission until the response completes.
pub max_execution_time: Duration,
/// Longest wait without progress: stream response headers, upstream reads, downstream writes.
pub stall_timeout: Duration,
}
impl Default for ExecutionLimits {
fn default() -> Self {
Self {
max_execution_time: Duration::from_secs(DEFAULT_MAX_EXECUTION_SECONDS.into()),
stall_timeout: Duration::from_secs(DEFAULT_STALL_TIMEOUT_SECONDS.into()),
}
}
}
#[derive(Debug, PartialEq, Eq)]
pub enum LogFormat {
Text,
Json,
}
#[derive(Debug, PartialEq, Eq, thiserror::Error)]
#[error("{0}")]
pub struct GatewayConfigError(&'static str);
impl GatewayConfig {
/// Read gateway configuration from the process environment, using defaults for absent values.
///
/// # Errors
/// Returns an error if a configured value is not Unicode or fails validation
/// in [`Self::from_values`], or if a Web URL is configured with an invalid URL
/// or missing/blank service key, or a non-boolean HTTP Web URL opt-in. Error
/// messages never include supplied values.
pub fn from_env() -> Result<Self, GatewayConfigError> {
let (connection_drain_timeout, uses_deprecated_connection_drain_timeout_name) =
connection_drain_timeout_value(
read_env("LANGFUSE_AI_GATEWAY_CONNECTION_DRAIN_TIMEOUT_SECONDS")?,
read_env("LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS")?,
);
let mut config = Self::from_values(
read_env("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS")?.as_deref(),
read_env("LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT")?.as_deref(),
connection_drain_timeout.as_deref(),
read_env("LANGFUSE_LOG_LEVEL")?.as_deref(),
read_env("LANGFUSE_LOG_FORMAT")?.as_deref(),
)?;
config.uses_deprecated_connection_drain_timeout_name =
uses_deprecated_connection_drain_timeout_name;
config.telemetry_buffer_bytes = telemetry_buffer_bytes(
read_env("LANGFUSE_AI_GATEWAY_TELEMETRY_BUFFER_BYTES")?.as_deref(),
)?;
config.telemetry_shutdown_timeout = telemetry_shutdown_timeout(
read_env("LANGFUSE_AI_GATEWAY_TELEMETRY_SHUTDOWN_TIMEOUT_SECONDS")?.as_deref(),
)?;
config.request_limits = request_limits(
read_env("LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES")?.as_deref(),
read_env("LANGFUSE_AI_GATEWAY_MEMORY_BUDGET_BYTES")?.as_deref(),
read_env("LANGFUSE_AI_GATEWAY_REQUEST_MEMORY_OVERHEAD_BYTES")?.as_deref(),
)?;
config.execution_limits = execution_limits(
read_env("LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS")?.as_deref(),
read_env("LANGFUSE_AI_GATEWAY_STALL_TIMEOUT_SECONDS")?.as_deref(),
)?;
config.control_plane = control_plane(
read_env("LANGFUSE_AI_GATEWAY_WEB_URL")?.as_deref(),
read_env("LANGFUSE_AI_GATEWAY_SERVICE_KEY")?.as_deref(),
read_env("LANGFUSE_AI_GATEWAY_ALLOW_HTTP_WEB_URL")?.as_deref(),
)?;
Ok(config)
}
/// Parse gateway configuration, using defaults for absent values.
///
/// # Errors
/// Returns an error for an invalid IP address and port, a non-boolean port
/// auto-increment setting, a connection drain timeout outside 1–300 integer seconds,
/// or an unsupported log level or format.
pub fn from_values(
listen_address: Option<&str>,
auto_increment_listen_port: Option<&str>,
connection_drain_timeout: Option<&str>,
log_level: Option<&str>,
log_format: Option<&str>,
) -> Result<Self, GatewayConfigError> {
let listen_address = listen_address
.unwrap_or("0.0.0.0:8080")
.parse()
.map_err(|_| {
GatewayConfigError(
"LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS must be an IP address and port",
)
})?;
let auto_increment_listen_port = parse_boolean(
auto_increment_listen_port.unwrap_or("false"),
)
.ok_or(GatewayConfigError(
"LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT must be true or false",
))?;
let seconds = integer_setting(
connection_drain_timeout,
DEFAULT_SHUTDOWN_TIMEOUT_SECONDS,
1..=300,
"LANGFUSE_AI_GATEWAY_CONNECTION_DRAIN_TIMEOUT_SECONDS (or the deprecated LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS) must be an integer from 1 to 300",
)?;
let log_level = match log_level.unwrap_or("info") {
"debug" => LevelFilter::DEBUG,
"info" => LevelFilter::INFO,
"warn" => LevelFilter::WARN,
// Tracing has no separate fatal severity.
"error" | "fatal" => LevelFilter::ERROR,
_ => {
return Err(GatewayConfigError(
"LANGFUSE_LOG_LEVEL must be debug, info, warn, error or fatal",
));
}
};
let log_format = match log_format.unwrap_or("text") {
"text" => LogFormat::Text,
"json" => LogFormat::Json,
_ => {
return Err(GatewayConfigError(
"LANGFUSE_LOG_FORMAT must be text or json",
));
}
};
Ok(Self {
control_plane: None,
listen_address,
auto_increment_listen_port,
connection_drain_timeout: Duration::from_secs(seconds),
uses_deprecated_connection_drain_timeout_name: false,
telemetry_shutdown_timeout: Duration::from_secs(DEFAULT_SHUTDOWN_TIMEOUT_SECONDS),
log_level,
log_format,
telemetry_buffer_bytes: crate::telemetry::DEFAULT_RETAINED_BYTES,
request_limits: RequestLimits::default(),
execution_limits: ExecutionLimits::default(),
})
}
}
fn control_plane(
web_url: Option<&str>,
service_key: Option<&str>,
allow_http_web_url: Option<&str>,
) -> Result<Option<ControlPlaneConfig>, GatewayConfigError> {
let allow_http_web_url = parse_boolean(allow_http_web_url.unwrap_or("false")).ok_or(
GatewayConfigError("LANGFUSE_AI_GATEWAY_ALLOW_HTTP_WEB_URL must be true or false"),
)?;
let Some(web_url) = web_url.filter(|url| !url.is_empty()) else {
return Ok(None);
};
let service_key = service_key.ok_or(GatewayConfigError(
"LANGFUSE_AI_GATEWAY_SERVICE_KEY is required when LANGFUSE_AI_GATEWAY_WEB_URL is set",
))?;
let control_plane = if allow_http_web_url {
ControlPlaneConfig::with_plain_http(web_url, service_key)
} else {
ControlPlaneConfig::new(web_url, service_key)
};
control_plane.map(Some).map_err(|_| {
GatewayConfigError("invalid LANGFUSE_AI_GATEWAY_WEB_URL or LANGFUSE_AI_GATEWAY_SERVICE_KEY")
})
}
fn request_limits(
max_request_bytes: Option<&str>,
memory_budget_bytes: Option<&str>,
request_overhead_bytes: Option<&str>,
) -> Result<RequestLimits, GatewayConfigError> {
let defaults = RequestLimits::default();
// The overhead and each body growth are single semaphore acquisitions, which take a u32.
let max_reservation = usize::try_from(u32::MAX)
.unwrap_or(usize::MAX)
.min(SEMAPHORE_CAPACITY);
let max_request_bytes = integer_setting(
max_request_bytes,
defaults.max_request_bytes,
MIB..=max_reservation,
"LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES must be an integer of at least 1048576 (1 MiB) and within a single semaphore reservation",
)?;
let request_overhead_bytes = integer_setting(
request_overhead_bytes,
defaults.request_overhead_bytes,
0..=max_reservation,
"LANGFUSE_AI_GATEWAY_REQUEST_MEMORY_OVERHEAD_BYTES must be a non-negative integer within a single semaphore reservation",
)?;
// A maximal request must fit, or it could never be admitted.
let largest_request = max_request_bytes.saturating_add(request_overhead_bytes);
let memory_budget_bytes = integer_setting(
memory_budget_bytes,
defaults.memory_budget_bytes,
largest_request..=SEMAPHORE_CAPACITY,
"LANGFUSE_AI_GATEWAY_MEMORY_BUDGET_BYTES must be an integer of at least LANGFUSE_AI_GATEWAY_MAX_REQUEST_BYTES plus LANGFUSE_AI_GATEWAY_REQUEST_MEMORY_OVERHEAD_BYTES within the semaphore capacity",
)?;
Ok(RequestLimits {
max_request_bytes,
memory_budget_bytes,
request_overhead_bytes,
})
}
fn execution_limits(
max_execution_time: Option<&str>,
stall_timeout: Option<&str>,
) -> Result<ExecutionLimits, GatewayConfigError> {
// `u32` seconds keep every deadline far from `Instant` overflow.
let max_execution_time: u32 = integer_setting(
max_execution_time,
DEFAULT_MAX_EXECUTION_SECONDS,
60..=u32::MAX,
"LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS must be an integer of at least 60",
)?;
let stall_timeout: u32 = integer_setting(
stall_timeout,
DEFAULT_STALL_TIMEOUT_SECONDS,
1..=max_execution_time,
"LANGFUSE_AI_GATEWAY_STALL_TIMEOUT_SECONDS must be an integer from 1 to LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS",
)?;
Ok(ExecutionLimits {
max_execution_time: Duration::from_secs(max_execution_time.into()),
stall_timeout: Duration::from_secs(stall_timeout.into()),
})
}
fn telemetry_buffer_bytes(value: Option<&str>) -> Result<usize, GatewayConfigError> {
integer_setting(
value,
crate::telemetry::DEFAULT_RETAINED_BYTES,
4 * MIB..=4096 * MIB,
"LANGFUSE_AI_GATEWAY_TELEMETRY_BUFFER_BYTES must be an integer from 4194304 (4 MiB) to 4294967296 (4 GiB)",
)
}
/// The current name wins; the deprecated name keeps released deployments working.
fn connection_drain_timeout_value(
current: Option<String>,
deprecated: Option<String>,
) -> (Option<String>, bool) {
match (current, deprecated) {
(Some(value), _) => (Some(value), false),
(None, Some(value)) => (Some(value), true),
(None, None) => (None, false),
}
}
fn telemetry_shutdown_timeout(value: Option<&str>) -> Result<Duration, GatewayConfigError> {
integer_setting(
value,
DEFAULT_SHUTDOWN_TIMEOUT_SECONDS,
1..=300,
"LANGFUSE_AI_GATEWAY_TELEMETRY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",
)
.map(Duration::from_secs)
}
/// An unset value takes `default`. Errors carry only `message`, never the supplied value.
fn integer_setting<T: std::str::FromStr + PartialOrd>(
value: Option<&str>,
default: T,
range: RangeInclusive<T>,
message: &'static str,
) -> Result<T, GatewayConfigError> {
value
.map_or(Some(default), |value| value.parse().ok())
.filter(|value| range.contains(value))
.ok_or(GatewayConfigError(message))
}
fn read_env(name: &'static str) -> Result<Option<String>, GatewayConfigError> {
match env::var(name) {
Ok(value) => Ok(Some(value)),
Err(env::VarError::NotPresent) => Ok(None),
Err(env::VarError::NotUnicode(_)) => Err(GatewayConfigError(
"gateway configuration contains a non-Unicode environment value",
)),
}
}
fn parse_boolean(value: &str) -> Option<bool> {
match value {
"true" => Some(true),
"false" => Some(false),
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn configuration_parses_defaults_and_overrides() {
let default = GatewayConfig::from_values(None, None, None, None, None).unwrap();
assert_eq!(default.listen_address, "0.0.0.0:8080".parse().unwrap());
assert!(!default.auto_increment_listen_port);
assert_eq!(default.connection_drain_timeout, Duration::from_secs(10));
assert_eq!(default.log_level, LevelFilter::INFO);
assert_eq!(default.log_format, LogFormat::Text);
assert_eq!(default.telemetry_buffer_bytes, 64 * 1024 * 1024);
let custom = GatewayConfig::from_values(
Some("[::1]:9000"),
Some("true"),
Some("30"),
Some("debug"),
Some("json"),
)
.unwrap();
assert_eq!(custom.listen_address, "[::1]:9000".parse().unwrap());
assert!(custom.auto_increment_listen_port);
assert_eq!(custom.connection_drain_timeout, Duration::from_secs(30));
assert_eq!(custom.log_level, LevelFilter::DEBUG);
assert_eq!(custom.log_format, LogFormat::Json);
}
#[test]
fn memory_budget_must_fit_a_maximal_request_with_its_overhead() {
assert_eq!(
request_limits(None, None, None).unwrap(),
RequestLimits {
max_request_bytes: 16 * MIB,
memory_budget_bytes: 1024 * MIB,
request_overhead_bytes: 2 * MIB,
}
);
let fitting = request_limits(Some("1048576"), Some("1048576"), Some("0")).unwrap();
assert_eq!(fitting.memory_budget_bytes, MIB);
assert_eq!(fitting.request_overhead_bytes, 0);
let budget = "LANGFUSE_AI_GATEWAY_MEMORY_BUDGET_BYTES";
let overhead = "LANGFUSE_AI_GATEWAY_REQUEST_MEMORY_OVERHEAD_BYTES";
for (max_request, budget_value, overhead_value, name) in [
(Some("1048576"), Some("1048576"), Some("1"), budget),
// One byte short of the default 16 MiB request plus 2 MiB overhead.
(None, Some("18874367"), None, budget),
(None, None, Some("-1"), overhead),
(None, None, Some("secret-that-must-not-appear"), overhead),
(None, Some("secret-that-must-not-appear"), None, budget),
] {
let error = request_limits(max_request, budget_value, overhead_value)
.err()
.unwrap();
assert!(error.to_string().starts_with(name), "{error}");
assert!(!error.to_string().contains("secret-that-must-not-appear"));
}
}
#[test]
fn validates_shared_log_format() {
for (value, expected) in [("text", LogFormat::Text), ("json", LogFormat::Json)] {
assert_eq!(
GatewayConfig::from_values(None, None, None, None, Some(value))
.unwrap()
.log_format,
expected
);
}
for value in ["", "TEXT", "pretty", "0", "secret-that-must-not-appear"] {
let error = GatewayConfig::from_values(None, None, None, None, Some(value))
.err()
.unwrap();
assert_eq!(
error.to_string(),
"LANGFUSE_LOG_FORMAT must be text or json"
);
}
}
#[test]
fn accepts_shared_langfuse_log_levels() {
for (value, expected) in [
("debug", LevelFilter::DEBUG),
("info", LevelFilter::INFO),
("warn", LevelFilter::WARN),
("error", LevelFilter::ERROR),
("fatal", LevelFilter::ERROR),
] {
let config = GatewayConfig::from_values(None, None, None, Some(value), None).unwrap();
assert_eq!(config.log_level, expected, "{value}");
}
}
#[test]
fn rejects_invalid_configuration_without_echoing_input() {
let sensitive_input = "secret-that-must-not-appear";
for (address, timeout, level) in [
(Some(sensitive_input), None, None),
(None, Some(sensitive_input), None),
(None, None, Some(sensitive_input)),
(None, None, Some("off")),
(None, None, Some("trace")),
(None, None, Some("INFO")),
(None, None, Some("3")),
(None, None, Some("")),
(None, Some("0"), None),
(None, Some("301"), None),
(None, Some("-1"), None),
(Some("localhost:8080"), None, None),
] {
let error = GatewayConfig::from_values(address, None, timeout, level, None)
.err()
.unwrap();
assert!(!error.to_string().contains(sensitive_input));
assert!(!format!("{error:?}").contains(sensitive_input));
}
for value in ["", "TRUE", "1", sensitive_input] {
let error = GatewayConfig::from_values(None, Some(value), None, None, None)
.err()
.unwrap();
assert!(!error.to_string().contains(sensitive_input));
assert!(!format!("{error:?}").contains(sensitive_input));
}
}
#[test]
fn plain_http_web_url_requires_the_explicit_opt_in() {
let in_cluster = Some("http://langfuse-web.langfuse.svc.cluster.local:3000");
let key = Some("service-key");
for opt_in in [None, Some("false")] {
let error = control_plane(in_cluster, key, opt_in).err().unwrap();
assert_eq!(
error.to_string(),
"invalid LANGFUSE_AI_GATEWAY_WEB_URL or LANGFUSE_AI_GATEWAY_SERVICE_KEY"
);
}
let allowed = control_plane(in_cluster, key, Some("true"))
.unwrap()
.unwrap();
assert!(allowed.is_plain_http());
assert_eq!(
allowed
.endpoint("/api/internal/ai-gateway/v1/ingestion")
.as_str(),
"http://langfuse-web.langfuse.svc.cluster.local:3000/api/internal/ai-gateway/v1/ingestion"
);
for (url, opt_in) in [
("https://cloud.langfuse.com", Some("true")),
("http://localhost:3000", None),
] {
let config = control_plane(Some(url), key, opt_in).unwrap().unwrap();
assert!(!config.is_plain_http(), "{url}");
}
assert!(control_plane(None, None, Some("true")).unwrap().is_none());
for value in ["", "TRUE", "1", "secret-that-must-not-appear"] {
let error = control_plane(in_cluster, key, Some(value)).err().unwrap();
assert_eq!(
error.to_string(),
"LANGFUSE_AI_GATEWAY_ALLOW_HTTP_WEB_URL must be true or false"
);
}
}
#[test]
fn telemetry_buffer_is_read_in_bytes_within_bounds() {
assert_eq!(telemetry_buffer_bytes(None).unwrap(), 64 * 1024 * 1024);
for value in [4_194_304, 67_108_864, 4_294_967_296] {
assert_eq!(
telemetry_buffer_bytes(Some(&value.to_string())).unwrap(),
value
);
}
for value in [
"",
"0",
"64",
"4194303",
"4294967297",
"-1",
"1.5",
"64MiB",
"secret-that-must-not-appear",
] {
let error = telemetry_buffer_bytes(Some(value)).err().unwrap();
assert_eq!(
error.to_string(),
"LANGFUSE_AI_GATEWAY_TELEMETRY_BUFFER_BYTES must be an integer from 4194304 (4 MiB) to 4294967296 (4 GiB)"
);
}
}
#[test]
fn connection_drain_timeout_falls_back_to_its_deprecated_name() {
let value = |text: &str| Some(text.to_owned());
assert_eq!(connection_drain_timeout_value(None, None), (None, false));
assert_eq!(
connection_drain_timeout_value(value("5"), None),
(value("5"), false)
);
assert_eq!(
connection_drain_timeout_value(None, value("7")),
(value("7"), true)
);
assert_eq!(
connection_drain_timeout_value(value("5"), value("7")),
(value("5"), false)
);
}
#[test]
fn telemetry_shutdown_timeout_is_read_in_seconds_within_bounds() {
assert_eq!(
GatewayConfig::from_values(None, None, None, None, None)
.unwrap()
.telemetry_shutdown_timeout,
Duration::from_secs(10)
);
assert_eq!(
telemetry_shutdown_timeout(None).unwrap(),
Duration::from_secs(10)
);
for value in [1, 120, 300] {
assert_eq!(
telemetry_shutdown_timeout(Some(&value.to_string())).unwrap(),
Duration::from_secs(value)
);
}
for value in ["", "0", "301", "-1", "1.5", "secret-that-must-not-appear"] {
let error = telemetry_shutdown_timeout(Some(value)).err().unwrap();
assert_eq!(
error.to_string(),
"LANGFUSE_AI_GATEWAY_TELEMETRY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300"
);
}
}
}