🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(ai-gateway): avoid local development port conflicts (#17432)
Hassieb Pakzada180d76a7a6c
diff --git a/ai-gateway/README.md b/ai-gateway/README.mdindex 55a46ff2f2..6deb790587 100644--- a/ai-gateway/README.md+++ b/ai-gateway/README.md@@ -24,8 +24,9 @@ pnpm devThis starts the gateway alongside the other development services. Turbo Watchrecompiles and restarts the gateway after Rust source changes; Web and workerkeep their built-in watchers. The gateway loads the root `.env` and listens on-port 8080 by default. Add overrides from the table below to your root `.env`;-exported shell variables take precedence.+the first available port starting at 8080. This lets multiple worktrees run+their development stacks at the same time. Add overrides from the table below+to your root `.env`; exported shell variables take precedence.No gateway-specific credentials are needed for this foundation slice.To work on only the gateway, use `pnpm dev --filter=ai-gateway`.@@ -33,8 +34,9 @@ Both commands watch Rust source changes; do not run both on the same port.To start once without watching, run `pnpm dev` from `ai-gateway/`. Cargo alsoworks independently of Node/pnpm; it reads exported environment variables only,-without automatically loading `.env`. Run it from this directory so rustup-selects the pinned toolchain:+without automatically loading `.env`. Direct Cargo and production starts fail+if the configured port is occupied unless port auto-increment is explicitly+enabled. Run it from this directory so rustup selects the pinned toolchain:```shcd ai-gateway@@ -54,12 +56,13 @@ The binary reads configuration from the process environment. The pnpm developmenscript loads the root `.env` before starting it; production and direct Cargo runsdo not load dotenv files. All settings have defaults:-| Variable | Default | Validation |-| ---------------------------------------------- | -------------- | -------------------------------------------------- |-| `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS` | `0.0.0.0:8080` | IP address and port; IPv6 uses `[::]:8080` |-| `LANGFUSE_LOG_FORMAT` | `text` | `text`, `json` |-| `LANGFUSE_LOG_LEVEL` | `info` | `trace`, `debug`, `info`, `warn`, `error`, `fatal` |-| `LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS` | `10` | Integer from 1 to 300 |+| Variable | Default | Validation |+| ------------------------------------------------ | -------------- | -------------------------------------------------- |+| `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS` | `0.0.0.0:8080` | IP address and port; IPv6 uses `[::]:8080` |+| `LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT` | `false` | `true`, `false`; the pnpm dev task sets `true` |+| `LANGFUSE_LOG_FORMAT` | `text` | `text`, `json` |+| `LANGFUSE_LOG_LEVEL` | `info` | `trace`, `debug`, `info`, `warn`, `error`, `fatal` |+| `LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS` | `10` | Integer from 1 to 300 |The gateway shares `LANGFUSE_LOG_LEVEL` with Web and worker. Values are lowercase;`fatal` maps to Rust's `error` level and therefore includes ordinary error logs.diff --git a/ai-gateway/package.json b/ai-gateway/package.jsonindex 8e48d235b5..06c6ae9322 100644--- a/ai-gateway/package.json+++ b/ai-gateway/package.json@@ -2,7 +2,7 @@"name": "ai-gateway","private": true,"scripts": {- "dev": "dotenv -e ../.env -- cargo run --locked",+ "dev": "dotenv -e ../.env -v LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT=true -- cargo run --locked","build": "cargo build --release --locked","typecheck": "cargo check --locked --all-targets","lint": "cargo fmt --all -- --check && cargo clippy --locked --all-targets -- -D warnings",diff --git a/ai-gateway/src/config.rs b/ai-gateway/src/config.rsindex 8b5d42d5f0..e9b7c5a092 100644--- a/ai-gateway/src/config.rs+++ b/ai-gateway/src/config.rs@@ -3,6 +3,7 @@ use tracing::level_filters::LevelFilter;pub struct Config {pub listen_address: SocketAddr,+ pub auto_increment_listen_port: bool,pub shutdown_timeout: Duration,pub log_level: LevelFilter,pub log_format: LogFormat,@@ -34,6 +35,7 @@ impl Config {pub fn from_env() -> Result<Self, ConfigError> {Self::from_values(read_env("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS")?.as_deref(),+ read_env("LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT")?.as_deref(),read_env("LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS")?.as_deref(),read_env("LANGFUSE_LOG_LEVEL")?.as_deref(),read_env("LANGFUSE_LOG_FORMAT")?.as_deref(),@@ -47,6 +49,7 @@ impl Config {/// 1–300 integer seconds, or an unsupported log level or format.pub fn from_values(listen_address: Option<&str>,+ auto_increment_listen_port: Option<&str>,shutdown_timeout: Option<&str>,log_level: Option<&str>,log_format: Option<&str>,@@ -57,6 +60,10 @@ impl Config {.map_err(|_| {ConfigError("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS must be an IP address and port")})?;+ let auto_increment_listen_port =+ parse_boolean(auto_increment_listen_port.unwrap_or("false")).ok_or(ConfigError(+ "LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT must be true or false",+ ))?;let seconds: u64 = shutdown_timeout.unwrap_or("10").parse().map_err(|_| {ConfigError("LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",@@ -87,6 +94,7 @@ impl Config {};Ok(Self {listen_address,+ auto_increment_listen_port,shutdown_timeout: Duration::from_secs(seconds),log_level,log_format,@@ -104,21 +112,36 @@ fn read_env(name: &'static str) -> Result<Option<String>, ConfigError> {}}+fn parse_boolean(value: &str) -> Option<bool> {+ match value {+ "true" => Some(true),+ "false" => Some(false),+ _ => None,+ }+}+#[cfg(test)]mod tests {use super::*;#[test]fn configuration_parses_defaults_and_overrides() {- let default = Config::from_values(None, None, None, None).unwrap();+ let default = Config::from_values(None, None, None, None, None).unwrap();assert_eq!(default.listen_address, "0.0.0.0:8080".parse().unwrap());+ assert!(!default.auto_increment_listen_port);assert_eq!(default.shutdown_timeout, Duration::from_secs(10));assert_eq!(default.log_level, LevelFilter::INFO);assert_eq!(default.log_format, LogFormat::Text);- let custom =- Config::from_values(Some("[::1]:9000"), Some("30"), Some("debug"), Some("json"))- .unwrap();+ let custom = Config::from_values(+ Some("[::1]:9000"),+ Some("true"),+ Some("30"),+ Some("debug"),+ Some("json"),+ )+ .unwrap();assert_eq!(custom.listen_address, "[::1]:9000".parse().unwrap());+ assert!(custom.auto_increment_listen_port);assert_eq!(custom.shutdown_timeout, Duration::from_secs(30));assert_eq!(custom.log_level, LevelFilter::DEBUG);assert_eq!(custom.log_format, LogFormat::Json);@@ -128,14 +151,14 @@ mod tests {fn validates_shared_log_format() {for (value, expected) in [("text", LogFormat::Text), ("json", LogFormat::Json)] {assert_eq!(- Config::from_values(None, None, None, Some(value))+ Config::from_values(None, None, None, None, Some(value)).unwrap().log_format,expected);}for value in ["", "TEXT", "pretty", "0", "secret-that-must-not-appear"] {- let error = Config::from_values(None, None, None, Some(value))+ let error = Config::from_values(None, None, None, None, Some(value)).err().unwrap();assert_eq!(@@ -155,7 +178,7 @@ mod tests {("error", LevelFilter::ERROR),("fatal", LevelFilter::ERROR),] {- let config = Config::from_values(None, None, Some(value), None).unwrap();+ let config = Config::from_values(None, None, None, Some(value), None).unwrap();assert_eq!(config.log_level, expected, "{value}");}}@@ -176,7 +199,14 @@ mod tests {(None, Some("-1"), None),(Some("localhost:8080"), None, None),] {- let error = Config::from_values(address, timeout, level, None)+ let error = Config::from_values(address, None, timeout, level, None)+ .err()+ .unwrap();+ assert!(!error.to_string().contains(sensitive_input));+ assert!(!format!("{error:?}").contains(sensitive_input));+ }+ for value in ["", "TRUE", "1", sensitive_input] {+ let error = Config::from_values(None, Some(value), None, None, None).err().unwrap();assert!(!error.to_string().contains(sensitive_input));diff --git a/ai-gateway/src/main.rs b/ai-gateway/src/main.rsindex 44e0d6819e..fa0511ea9a 100644--- a/ai-gateway/src/main.rs+++ b/ai-gateway/src/main.rs@@ -1,4 +1,4 @@-use std::{error::Error, process::ExitCode};+use std::{error::Error, io, net::SocketAddr, process::ExitCode};use ai_gateway::{config::{Config, LogFormat},@@ -27,7 +27,7 @@ async fn run() -> Result<(), Box<dyn Error>> {LogFormat::Json => logging.json().init(),}let shutdown = shutdown_signal()?;- let listener = TcpListener::bind(config.listen_address).await?;+ let listener = bind_listener(config.listen_address, config.auto_increment_listen_port).await?;tracing::info!(address = %listener.local_addr()?, "gateway listening");let state = AppState::default();server::serve(@@ -42,6 +42,35 @@ async fn run() -> Result<(), Box<dyn Error>> {Ok(())}+async fn bind_listener(+ requested_address: SocketAddr,+ auto_increment_port: bool,+) -> io::Result<TcpListener> {+ let mut address = requested_address;+ loop {+ match TcpListener::bind(address).await {+ Ok(listener) => {+ if address != requested_address {+ tracing::warn!(+ %requested_address,+ selected_address = %address,+ "gateway listen port unavailable; using next available port"+ );+ }+ return Ok(listener);+ }+ Err(error)+ if auto_increment_port+ && error.kind() == io::ErrorKind::AddrInUse+ && address.port() < u16::MAX =>+ {+ address.set_port(address.port() + 1);+ }+ Err(error) => return Err(error),+ }+ }+}+#[cfg(unix)]fn shutdown_signal() -> std::io::Result<impl Future<Output = ()> + Send> {use tokio::signal::unix::{SignalKind, signal};diff --git a/ai-gateway/tests/startup.rs b/ai-gateway/tests/startup.rsindex 999e70e276..6aee192d52 100644--- a/ai-gateway/tests/startup.rs+++ b/ai-gateway/tests/startup.rs@@ -5,6 +5,7 @@ fn invalid_configuration_exits_without_disclosing_the_value() {// Occupy the listener so an ignored setting cannot leave a child server running.let listener = TcpListener::bind("127.0.0.1:0").unwrap();for name in [+ "LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT","LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS","LANGFUSE_LOG_LEVEL","LANGFUSE_LOG_FORMAT",@@ -45,6 +46,52 @@ fn occupied_listener_exits_unsuccessfully() {);}+#[test]+fn occupied_listener_uses_next_port_when_enabled() {+ use std::{+ io::{BufRead, BufReader},+ process::Stdio,+ sync::mpsc,+ time::Duration,+ };++ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let occupied_address = listener.local_addr().unwrap();+ assert_ne!(occupied_address.port(), u16::MAX);++ let mut child = Command::new(env!("CARGO_BIN_EXE_ai-gateway"))+ .env_clear()+ .env(+ "LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS",+ occupied_address.to_string(),+ )+ .env("LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT", "true")+ .stdout(Stdio::piped())+ .stderr(Stdio::null())+ .spawn()+ .unwrap();+ let stdout = child.stdout.take().unwrap();+ let (sender, receiver) = mpsc::channel();+ let reader = std::thread::spawn(move || {+ let mut lines = BufReader::new(stdout).lines();+ let result = lines.find(|line| {+ line.as_ref()+ .is_ok_and(|line| line.contains("gateway listening"))+ });+ let _ = sender.send(result);+ });+ let line = receiver+ .recv_timeout(Duration::from_secs(10))+ .expect("gateway must start on the next available port")+ .expect("gateway must emit a startup log")+ .unwrap();+ let _ = child.kill();+ child.wait().unwrap();+ reader.join().unwrap();++ assert!(!line.contains(&occupied_address.to_string()), "{line}");+}+#[test]fn lifecycle_logs_follow_the_shared_format() {use std::{