Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): allow an opt-in plain HTTP web URL for in-cluster traffic (#19224)

Tobias Wochinger8aeb85020aa1
diff --git a/.env.prod.example b/.env.prod.example
index 32697bc620..5f722b7c00 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -599,6 +599,10 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
# Rust AI gateway. Use the trusted HTTPS Web URL including any NEXT_PUBLIC_BASE_PATH.
# Without a Web URL the gateway starts, but readiness and inference return 503.
# LANGFUSE_AI_GATEWAY_WEB_URL=https://langfuse.example.com/app
+# Set to true to allow a plain http:// Web URL off loopback. Internal traffic only, e.g.
+# Web's in-cluster Kubernetes Service; never a URL that crosses the public internet, since
+# resolve responses carry provider API keys and the service key travels unencrypted.
+# LANGFUSE_AI_GATEWAY_ALLOW_HTTP_WEB_URL=false
# Same shared service key configured in Web. Provider keys are resolved through Web.
# LANGFUSE_AI_GATEWAY_SERVICE_KEY=
# Web only: enable the gateway for every organization. When false, only
diff --git a/ai-gateway/README.md b/ai-gateway/README.md
index 26c90c10f0..b72875acf1 100644
--- a/ai-gateway/README.md
+++ b/ai-gateway/README.md
@@ -73,6 +73,7 @@ do not load dotenv files:
| ---------------------------------------------- | -------------- | -------------------------------------------------- |
| `LANGFUSE_AI_GATEWAY_WEB_URL` | unset; inference disabled | Trusted Web base URL, including any deployment prefix |
| `LANGFUSE_AI_GATEWAY_SERVICE_KEY` | unset | Required with Web URL; same key configured in Web |
+| `LANGFUSE_AI_GATEWAY_ALLOW_HTTP_WEB_URL` | `false` | `true`, `false`; `true` also accepts a plain `http://` Web URL on any host. Only for internal traffic, such as Web's in-cluster Kubernetes Service (`http://<release>-web.<namespace>.svc.cluster.local:3000`) or another private network; never for a URL that crosses the public internet, because resolve responses carry provider API keys and requests carry the service key and ingestion tokens unencrypted. Resolution, model catalogs and trace uploads all use this URL. The gateway logs a warning at startup when it is used |
| `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS` | `0.0.0.0:8080` | IP address and port; IPv6 uses `[::]:8080` |
| `LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT` | `false` | `true`, `false`; the pnpm dev task sets `true` |
| `LANGFUSE_LOG_FORMAT` | `text` | `text`, `json` |
@@ -860,13 +861,7 @@ the file locally: `cargo about generate --locked --fail about.hbs`.
### Web resolution client
-`resolution::ControlPlaneClient` resolves a gateway key through an operator-configured Web
-base URL. The binary initializes it when a Web URL is configured. The
-caller supplies the existing `LANGFUSE_AI_GATEWAY_SERVICE_KEY` and a trusted Web
-base URL to `ControlPlaneConfig::new`. HTTPS is required except on loopback for local
-development. URLs cannot contain credentials, a query or fragment. Include the
-Web deployment's `NEXT_PUBLIC_BASE_PATH`, if set: both `https://host/app` and
-`https://host/app/` resolve through `/app/api/internal/ai-gateway/v1/resolve`.
+`resolution::ControlPlaneClient` resolves a gateway key through an operator-configured Web base URL. The binary initializes it when a Web URL is configured. The caller supplies the existing `LANGFUSE_AI_GATEWAY_SERVICE_KEY` and a trusted Web base URL to `ControlPlaneConfig::new`. HTTPS is required except on loopback for local development; `ControlPlaneConfig::with_plain_http`, which the binary uses when `LANGFUSE_AI_GATEWAY_ALLOW_HTTP_WEB_URL=true`, also accepts HTTP on any host, for internal traffic only. URLs cannot contain credentials, a query or fragment. Include the Web deployment's `NEXT_PUBLIC_BASE_PATH`, if set: both `https://host/app` and `https://host/app/` resolve through `/app/api/internal/ai-gateway/v1/resolve`.
```rust,no_run
use ai_gateway::resolution::{ApiFormat, ResolutionError, ControlPlaneClient, ControlPlaneConfig};
diff --git a/ai-gateway/src/config.rs b/ai-gateway/src/config.rs
index f5fabd7e3b..11b1750eb9 100644
--- a/ai-gateway/src/config.rs
+++ b/ai-gateway/src/config.rs
@@ -78,7 +78,8 @@ impl GatewayConfig {
/// # Errors
/// Returns an error if a configured value is not Unicode or fails validation
/// in [`Self::from_values`], or if a Web URL is configured with an invalid URL
- /// or missing/blank service key. Error messages never include supplied values.
+ /// or missing/blank service key, or a non-boolean HTTP Web URL opt-in. Error
+ /// messages never include supplied values.
pub fn from_env() -> Result<Self, GatewayConfigError> {
let mut config = Self::from_values(
read_env("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS")?.as_deref(),
@@ -99,20 +100,11 @@ impl GatewayConfig {
read_env("LANGFUSE_AI_GATEWAY_MAX_EXECUTION_SECONDS")?.as_deref(),
read_env("LANGFUSE_AI_GATEWAY_STALL_TIMEOUT_SECONDS")?.as_deref(),
)?;
- if let Some(web_url) =
- read_env("LANGFUSE_AI_GATEWAY_WEB_URL")?.filter(|url| !url.is_empty())
- {
- let service_key = read_env("LANGFUSE_AI_GATEWAY_SERVICE_KEY")?.ok_or(GatewayConfigError(
- "LANGFUSE_AI_GATEWAY_SERVICE_KEY is required when LANGFUSE_AI_GATEWAY_WEB_URL is set",
- ))?;
- config.control_plane = Some(ControlPlaneConfig::new(&web_url, &service_key).map_err(
- |_| {
- GatewayConfigError(
- "invalid LANGFUSE_AI_GATEWAY_WEB_URL or LANGFUSE_AI_GATEWAY_SERVICE_KEY",
- )
- },
- )?);
- }
+ config.control_plane = control_plane(
+ read_env("LANGFUSE_AI_GATEWAY_WEB_URL")?.as_deref(),
+ read_env("LANGFUSE_AI_GATEWAY_SERVICE_KEY")?.as_deref(),
+ read_env("LANGFUSE_AI_GATEWAY_ALLOW_HTTP_WEB_URL")?.as_deref(),
+ )?;
Ok(config)
}
@@ -184,6 +176,30 @@ impl GatewayConfig {
}
}
+fn control_plane(
+ web_url: Option<&str>,
+ service_key: Option<&str>,
+ allow_http_web_url: Option<&str>,
+) -> Result<Option<ControlPlaneConfig>, GatewayConfigError> {
+ let allow_http_web_url = parse_boolean(allow_http_web_url.unwrap_or("false")).ok_or(
+ GatewayConfigError("LANGFUSE_AI_GATEWAY_ALLOW_HTTP_WEB_URL must be true or false"),
+ )?;
+ let Some(web_url) = web_url.filter(|url| !url.is_empty()) else {
+ return Ok(None);
+ };
+ let service_key = service_key.ok_or(GatewayConfigError(
+ "LANGFUSE_AI_GATEWAY_SERVICE_KEY is required when LANGFUSE_AI_GATEWAY_WEB_URL is set",
+ ))?;
+ let control_plane = if allow_http_web_url {
+ ControlPlaneConfig::with_plain_http(web_url, service_key)
+ } else {
+ ControlPlaneConfig::new(web_url, service_key)
+ };
+ control_plane.map(Some).map_err(|_| {
+ GatewayConfigError("invalid LANGFUSE_AI_GATEWAY_WEB_URL or LANGFUSE_AI_GATEWAY_SERVICE_KEY")
+ })
+}
+
fn request_limits(
max_request_bytes: Option<&str>,
memory_budget_bytes: Option<&str>,
@@ -410,6 +426,44 @@ mod tests {
}
}
+ #[test]
+ fn plain_http_web_url_requires_the_explicit_opt_in() {
+ let in_cluster = Some("http://langfuse-web.langfuse.svc.cluster.local:3000");
+ let key = Some("service-key");
+ for opt_in in [None, Some("false")] {
+ let error = control_plane(in_cluster, key, opt_in).err().unwrap();
+ assert_eq!(
+ error.to_string(),
+ "invalid LANGFUSE_AI_GATEWAY_WEB_URL or LANGFUSE_AI_GATEWAY_SERVICE_KEY"
+ );
+ }
+ let allowed = control_plane(in_cluster, key, Some("true"))
+ .unwrap()
+ .unwrap();
+ assert!(allowed.is_plain_http());
+ assert_eq!(
+ allowed
+ .endpoint("/api/internal/ai-gateway/v1/ingestion")
+ .as_str(),
+ "http://langfuse-web.langfuse.svc.cluster.local:3000/api/internal/ai-gateway/v1/ingestion"
+ );
+ for (url, opt_in) in [
+ ("https://cloud.langfuse.com", Some("true")),
+ ("http://localhost:3000", None),
+ ] {
+ let config = control_plane(Some(url), key, opt_in).unwrap().unwrap();
+ assert!(!config.is_plain_http(), "{url}");
+ }
+ assert!(control_plane(None, None, Some("true")).unwrap().is_none());
+ for value in ["", "TRUE", "1", "secret-that-must-not-appear"] {
+ let error = control_plane(in_cluster, key, Some(value)).err().unwrap();
+ assert_eq!(
+ error.to_string(),
+ "LANGFUSE_AI_GATEWAY_ALLOW_HTTP_WEB_URL must be true or false"
+ );
+ }
+ }
+
#[test]
fn telemetry_buffer_is_read_in_bytes_within_bounds() {
assert_eq!(telemetry_buffer_bytes(None).unwrap(), 64 * 1024 * 1024);
diff --git a/ai-gateway/src/main.rs b/ai-gateway/src/main.rs
index 307242c32c..d83caa9b93 100644
--- a/ai-gateway/src/main.rs
+++ b/ai-gateway/src/main.rs
@@ -5,6 +5,7 @@ use ai_gateway::{
http,
inference::InferenceService,
observability,
+ resolution::ControlPlaneConfig,
server::{self, GatewayLifecycleState},
};
use tokio::net::TcpListener;
@@ -27,6 +28,15 @@ async fn main() -> ExitCode {
async fn run() -> Result<(), Box<dyn Error>> {
let mut config = GatewayConfig::from_env()?;
let observability = observability::init(&config)?;
+ if config
+ .control_plane
+ .as_ref()
+ .is_some_and(ControlPlaneConfig::is_plain_http)
+ {
+ tracing::warn!(
+ "LANGFUSE_AI_GATEWAY_WEB_URL uses plain HTTP off loopback (LANGFUSE_AI_GATEWAY_ALLOW_HTTP_WEB_URL=true); provider API keys, the service key and ingestion tokens travel unencrypted. Only use it for internal traffic such as Web's in-cluster Kubernetes Service, never for a URL that crosses the public internet"
+ );
+ }
let inference = config
.control_plane
.take()
diff --git a/ai-gateway/src/resolution/mod.rs b/ai-gateway/src/resolution/mod.rs
index ec9f06a2e2..064cc71556 100644
--- a/ai-gateway/src/resolution/mod.rs
+++ b/ai-gateway/src/resolution/mod.rs
@@ -36,6 +36,7 @@ struct ControlPlaneLimits {
/// Operator-supplied Web base URL and the existing gateway/Web service key.
pub struct ControlPlaneConfig {
web_url: Url,
+ plain_http: bool,
service_key: SecretString,
resolve_limits: ControlPlaneLimits,
models_limits: ControlPlaneLimits,
@@ -49,6 +50,28 @@ impl ControlPlaneConfig {
/// base URL. URLs require HTTPS (HTTP is allowed on loopback), with no userinfo,
/// query, or fragment.
pub fn new(web_url: &str, service_key: &str) -> Result<Self, ResolutionError> {
+ Self::from_parts(web_url, service_key, false)
+ }
+
+ /// Like [`Self::new`], but also accepts plain HTTP on any host.
+ ///
+ /// Only for internal traffic, such as Web's in-cluster Kubernetes Service or another
+ /// private network. Never use it for a URL that crosses the public internet: resolve
+ /// responses carry provider API keys, and requests carry the service key and
+ /// ingestion tokens, all unencrypted over HTTP.
+ ///
+ /// # Errors
+ /// Returns [`ResolutionError::Configuration`] for a blank service key or an invalid
+ /// base URL. URLs require HTTP or HTTPS, with no userinfo, query, or fragment.
+ pub fn with_plain_http(web_url: &str, service_key: &str) -> Result<Self, ResolutionError> {
+ Self::from_parts(web_url, service_key, true)
+ }
+
+ fn from_parts(
+ web_url: &str,
+ service_key: &str,
+ allow_plain_http: bool,
+ ) -> Result<Self, ResolutionError> {
let url = Url::parse(web_url).map_err(|_| ResolutionError::Configuration)?;
let loopback = url.host_str().is_some_and(|host| {
host == "localhost"
@@ -57,7 +80,8 @@ impl ControlPlaneConfig {
.parse::<IpAddr>()
.is_ok_and(|ip| ip.is_loopback())
});
- if !(url.scheme() == "https" || (url.scheme() == "http" && loopback))
+ let plain_http = url.scheme() == "http" && !loopback;
+ if !(url.scheme() == "https" || (url.scheme() == "http" && (loopback || allow_plain_http)))
|| url.host_str().is_none()
|| !url.username().is_empty()
|| url.password().is_some()
@@ -69,6 +93,7 @@ impl ControlPlaneConfig {
}
Ok(Self {
web_url: url,
+ plain_http,
service_key: service_key.into(),
resolve_limits: ControlPlaneLimits {
timeout: Duration::from_secs(5),
@@ -88,6 +113,13 @@ impl ControlPlaneConfig {
url
}
+ /// Whether the Web URL is plain HTTP off loopback, so provider API keys, the service
+ /// key and ingestion tokens travel unencrypted.
+ #[must_use]
+ pub fn is_plain_http(&self) -> bool {
+ self.plain_http
+ }
+
pub(crate) fn service_key(&self) -> &SecretString {
&self.service_key
}
diff --git a/ai-gateway/src/resolution/tests.rs b/ai-gateway/src/resolution/tests.rs
index 643ceba985..5cb50f061b 100644
--- a/ai-gateway/src/resolution/tests.rs
+++ b/ai-gateway/src/resolution/tests.rs
@@ -972,3 +972,23 @@ fn web_configuration_rejects_unsafe_or_ambiguous_urls() {
assert!(ControlPlaneConfig::new(url, SERVICE_KEY).is_ok());
}
}
+
+#[test]
+fn plain_http_web_configuration_only_relaxes_the_https_requirement() {
+ assert!(ControlPlaneConfig::with_plain_http("http://web.example", SERVICE_KEY).is_ok());
+ for url in [
+ "ftp://web.example",
+ "http://user:[email protected]",
+ "http://web.example?query=true",
+ "http://web.example#fragment",
+ ] {
+ assert!(matches!(
+ ControlPlaneConfig::with_plain_http(url, SERVICE_KEY),
+ Err(ResolutionError::Configuration)
+ ));
+ }
+ assert!(matches!(
+ ControlPlaneConfig::with_plain_http("http://web.example", " "),
+ Err(ResolutionError::Configuration)
+ ));
+}