Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): per-organization request and byte rate limits at /resolve (#19585)

Hassieb Pakzad0f5b1840329e
diff --git a/ai-gateway/src/correlation.rs b/ai-gateway/src/correlation.rs
index 45c442a562..c8de9de077 100644
--- a/ai-gateway/src/correlation.rs
+++ b/ai-gateway/src/correlation.rs
@@ -11,7 +11,9 @@ use axum::{
use crate::{
observability::RequestObservation,
- resolution::{OrganizationId, ProjectId, ProviderConnection, ResolvedRequestContext},
+ resolution::{
+ OrganizationId, ProjectId, ProviderConnection, RateLimit, ResolvedRequestContext,
+ },
};
pub(crate) const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("langfuse-request-id");
@@ -145,6 +147,14 @@ impl RequestCorrelation {
});
}
+ /// Web names the organization it rate limited, since no context was resolved.
+ pub(crate) fn record_rate_limit(&self, limit: &RateLimit) {
+ if let Some(organization_id) = limit.organization_id() {
+ self.server_span
+ .record("langfuse.organization.id", organization_id);
+ }
+ }
+
pub(crate) fn record_upstream(&mut self, target: UpstreamTarget) {
let span = &self.server_span;
span.record("gateway.provider", target.provider);
diff --git a/ai-gateway/src/http.rs b/ai-gateway/src/http.rs
index 58cb9e58b4..e7a0b850f7 100644
--- a/ai-gateway/src/http.rs
+++ b/ai-gateway/src/http.rs
@@ -16,10 +16,13 @@ use tracing::Instrument;
use crate::{
correlation::{RequestCorrelation, assign_request_id},
- inference::{InferenceService, MemoryReservation, provider_origin, resolution_origin},
+ inference::{
+ InferenceService, MemoryReservation, provider_origin, rate_limited_reason,
+ resolution_origin,
+ },
observability::{ErrorOrigin, Wait},
providers::{ProviderError, RequestBody, Route},
- resolution::{ApiFormat, ResolutionError},
+ resolution::{ApiFormat, RateLimit, ResolutionError},
server::GatewayLifecycleState,
};
@@ -220,13 +223,17 @@ async fn relay(
.observation()
.record_wait(Wait::RequestBody, started.elapsed());
let (bytes, reservation) = read?;
- correlation.observation().record_request_bytes(bytes.len());
+ let request_bytes = bytes.len();
+ correlation
+ .observation()
+ .record_request_bytes(request_bytes);
let request_body = request_body_and_model(bytes)?;
let (permit, context) = inference
.resolve_and_admit(
gateway_key,
route.api_format(),
&request_body.model,
+ request_bytes,
correlation,
)
.await
@@ -427,12 +434,12 @@ enum InferenceHttpError {
#[derive(Serialize)]
struct OpenAiErrorResponse<'a> {
- error: OpenAiErrorDetail,
+ error: OpenAiErrorDetail<'a>,
request_id: &'a str,
}
#[derive(Serialize)]
-struct OpenAiErrorDetail {
- message: &'static str,
+struct OpenAiErrorDetail<'a> {
+ message: &'a str,
#[serde(rename = "type")]
kind: &'static str,
param: Option<&'static str>,
@@ -443,14 +450,14 @@ struct OpenAiErrorDetail {
struct AnthropicErrorResponse<'a> {
#[serde(rename = "type")]
kind: &'static str,
- error: AnthropicErrorDetail,
+ error: AnthropicErrorDetail<'a>,
request_id: &'a str,
}
#[derive(Serialize)]
-struct AnthropicErrorDetail {
+struct AnthropicErrorDetail<'a> {
#[serde(rename = "type")]
kind: &'static str,
- message: &'static str,
+ message: &'a str,
}
impl InferenceHttpError {
@@ -472,6 +479,7 @@ impl InferenceHttpError {
R::Forbidden => "forbidden",
R::ModelNotAllowed => "model_not_allowed",
R::NoRoute => "no_route",
+ R::RateLimited(limit) => rate_limited_reason(limit.kind()),
R::Busy => "capacity",
R::Timeout => "timeout",
R::Unavailable => "unavailable",
@@ -519,6 +527,13 @@ impl InferenceHttpError {
"permission_error",
"model_not_allowed",
),
+ // OpenAI's own rate-limit type and code, so SDKs treat it as a 429 to back off.
+ Self::Resolution(R::RateLimited(_)) => (
+ StatusCode::TOO_MANY_REQUESTS,
+ "Langfuse AI Gateway rate limit exceeded",
+ "requests",
+ "rate_limit_exceeded",
+ ),
Self::Resolution(R::NoRoute) => (
StatusCode::NOT_FOUND,
"No eligible provider connection",
@@ -588,7 +603,7 @@ impl InferenceHttpError {
Self::RequestTimeout => (ErrorOrigin::Client, "body_timeout"),
Self::UnsupportedEncoding => (ErrorOrigin::Client, "unsupported_content_encoding"),
Self::InvalidBody => (ErrorOrigin::Client, "invalid_body"),
- Self::Resolution(error) => resolution_origin(*error),
+ Self::Resolution(error) => resolution_origin(error),
Self::Provider(error) => provider_origin(*error),
}
}
@@ -601,8 +616,23 @@ impl InferenceHttpError {
let (phase, reason) = self.category();
let (origin, _) = self.origin();
let (status, message, kind, code) = self.classify();
+ let rate_limit = match &self {
+ Self::Resolution(ResolutionError::RateLimited(limit)) => Some(limit.as_ref()),
+ _ => None,
+ };
// Caller mistakes are routine; only failures that need an operator warn.
- if status.is_server_error() {
+ if let Some(limit) = rate_limit {
+ tracing::warn!(
+ phase,
+ reason,
+ error.origin = origin.as_str(),
+ status = status.as_u16(),
+ rate_limit = limit.kind().as_str(),
+ retry_after_secs = limit.retry_after_secs(),
+ organization_id = limit.organization_id(),
+ "gateway request rate limited"
+ );
+ } else if status.is_server_error() {
tracing::warn!(
phase,
reason,
@@ -620,7 +650,8 @@ impl InferenceHttpError {
);
}
let request_id = correlation.id();
- match api_format {
+ let message = rate_limit.map_or(message, RateLimit::message);
+ let mut response = match api_format {
ApiFormat::OpenAiResponses | ApiFormat::OpenAiChatCompletions => (
status,
Json(OpenAiErrorResponse {
@@ -646,7 +677,13 @@ impl InferenceHttpError {
}),
)
.into_response(),
+ };
+ if let Some(limit) = rate_limit {
+ response
+ .headers_mut()
+ .insert(header::RETRY_AFTER, limit.retry_after_secs().into());
}
+ response
}
}
@@ -655,6 +692,7 @@ fn anthropic_error_type(status: StatusCode) -> &'static str {
StatusCode::UNAUTHORIZED => "authentication_error",
StatusCode::FORBIDDEN => "permission_error",
StatusCode::NOT_FOUND => "not_found_error",
+ StatusCode::TOO_MANY_REQUESTS => "rate_limit_error",
StatusCode::PAYLOAD_TOO_LARGE => "request_too_large",
StatusCode::SERVICE_UNAVAILABLE => "overloaded_error",
status if status.is_client_error() => "invalid_request_error",
diff --git a/ai-gateway/src/http/tests.rs b/ai-gateway/src/http/tests.rs
index 9518b87eaa..c4efcf069c 100644
--- a/ai-gateway/src/http/tests.rs
+++ b/ai-gateway/src/http/tests.rs
@@ -1277,7 +1277,7 @@ async fn anthropic_messages_accept_either_credential_header_and_relay_native_sse
let key = anthropic_gateway_key(&request).unwrap();
assert_eq!(
to_bytes(request.into_body(), 1024).await.unwrap(),
- r#"{"apiFormat":"anthropic.messages","model":"Production key:claude-sonnet-4-5"}"#
+ r#"{"apiFormat":"anthropic.messages","model":"Production key:claude-sonnet-4-5","requestBytes":72}"#
);
if key == "gateway-anthropic" {
resolution_response_for(ApiFormat::AnthropicMessages, "provider-anthropic")
@@ -1469,6 +1469,65 @@ async fn model_policy_rejections_use_provider_native_permission_errors() {
assert_eq!(provider.calls(), 0);
}
+const RATE_LIMIT_MESSAGE: &str = "Langfuse AI Gateway rate limit exceeded: your organization's Hobby plan allows 100 requests per minute. Retry after 23s or upgrade your plan.";
+
+#[tokio::test]
+async fn organization_rate_limits_become_provider_native_429s_with_retry_after() {
+ let web = FakeServer::start(|request| async move {
+ let body: serde_json::Value =
+ serde_json::from_slice(&to_bytes(request.into_body(), 1024).await.unwrap()).unwrap();
+ assert_eq!(body["requestBytes"], 34);
+ Response::builder()
+ .status(StatusCode::TOO_MANY_REQUESTS)
+ .header("retry-after", "23")
+ .header("langfuse-gateway-error-code", "rate_limit_exceeded")
+ .body(Body::from(
+ serde_json::json!({
+ "error": RATE_LIMIT_MESSAGE,
+ "limit": "requests",
+ "organizationId": "org-1",
+ })
+ .to_string(),
+ ))
+ .unwrap()
+ })
+ .await;
+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
+ let gateway = Gateway::start(Some(inference(&web, &provider))).await;
+
+ let openai = gateway
+ .post()
+ .bearer_auth("gateway-key")
+ .body(r#"{"model":"Production key:gpt-4.1"}"#)
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(openai.status(), StatusCode::TOO_MANY_REQUESTS);
+ assert_eq!(openai.headers()[header::RETRY_AFTER], "23");
+ let openai_body: serde_json::Value =
+ serde_json::from_str(&openai.text().await.unwrap()).unwrap();
+ assert_eq!(openai_body["error"]["message"], RATE_LIMIT_MESSAGE);
+ assert_eq!(openai_body["error"]["type"], "requests");
+ assert_eq!(openai_body["error"]["code"], "rate_limit_exceeded");
+
+ let anthropic = reqwest::Client::new()
+ .post(format!("{}/anthropic/v1/messages", gateway.url))
+ .header("x-api-key", "gateway-key")
+ .body(r#"{"model":"Production key:gpt-4.1"}"#)
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(anthropic.status(), StatusCode::TOO_MANY_REQUESTS);
+ assert_eq!(anthropic.headers()[header::RETRY_AFTER], "23");
+ let anthropic_body: serde_json::Value =
+ serde_json::from_str(&anthropic.text().await.unwrap()).unwrap();
+ assert_eq!(anthropic_body["type"], "error");
+ assert_eq!(anthropic_body["error"]["type"], "rate_limit_error");
+ assert_eq!(anthropic_body["error"]["message"], RATE_LIMIT_MESSAGE);
+ assert_eq!(web.calls(), 2);
+ assert_eq!(provider.calls(), 0);
+}
+
#[tokio::test]
async fn anthropic_capacity_rejections_use_the_native_envelope() {
let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
diff --git a/ai-gateway/src/inference.rs b/ai-gateway/src/inference.rs
index dcf088447d..4668c166ba 100644
--- a/ai-gateway/src/inference.rs
+++ b/ai-gateway/src/inference.rs
@@ -14,7 +14,8 @@ use crate::{
observability::ErrorOrigin,
providers::{ProviderError, ProviderTransport, RequestBody, RequestPermit, Route},
resolution::{
- ApiFormat, ControlPlaneClient, ControlPlaneConfig, ResolutionError, ResolvedRequestContext,
+ ApiFormat, ControlPlaneClient, ControlPlaneConfig, RateLimitKind, ResolutionError,
+ ResolvedRequestContext,
},
};
@@ -27,7 +28,7 @@ pub struct InferenceService {
}
/// `error.origin` and `error.reason` for a failed resolution or catalog fetch.
-pub(crate) fn resolution_origin(error: ResolutionError) -> (ErrorOrigin, &'static str) {
+pub(crate) fn resolution_origin(error: &ResolutionError) -> (ErrorOrigin, &'static str) {
use ResolutionError as R;
match error {
R::Configuration => (ErrorOrigin::Gateway, "configuration"),
@@ -37,6 +38,7 @@ pub(crate) fn resolution_origin(error: ResolutionError) -> (ErrorOrigin, &'stati
R::Forbidden => (ErrorOrigin::Client, "forbidden"),
R::ModelNotAllowed => (ErrorOrigin::Client, "model_not_allowed"),
R::NoRoute => (ErrorOrigin::Client, "no_route"),
+ R::RateLimited(limit) => (ErrorOrigin::Client, rate_limited_reason(limit.kind())),
R::Unavailable => (ErrorOrigin::ControlPlane, "unavailable"),
R::Timeout => (ErrorOrigin::ControlPlane, "timeout"),
R::Transport => (ErrorOrigin::ControlPlane, "transport"),
@@ -45,6 +47,14 @@ pub(crate) fn resolution_origin(error: ResolutionError) -> (ErrorOrigin, &'stati
}
}
+/// Distinct from `provider_rate_limited`, which is the provider's own 429.
+pub(crate) fn rate_limited_reason(kind: RateLimitKind) -> &'static str {
+ match kind {
+ RateLimitKind::Requests => "rate_limited_requests",
+ RateLimitKind::Bytes => "rate_limited_bytes",
+ }
+}
+
/// `error.origin` and `error.reason` for a provider request that produced no response.
pub(crate) fn provider_origin(error: ProviderError) -> (ErrorOrigin, &'static str) {
match error {
@@ -205,12 +215,14 @@ impl InferenceService {
}
/// Authenticate, then start the execution. The API format comes from the public route,
- /// so Web selects a compatible connection. Memory is already reserved by the caller.
+ /// so Web selects a compatible connection, and `request_bytes` counts toward the
+ /// organization's byte rate limit. Memory is already reserved by the caller.
pub(crate) async fn resolve_and_admit(
&self,
gateway_key: &str,
api_format: ApiFormat,
model: &str,
+ request_bytes: usize,
correlation: &mut RequestCorrelation,
) -> Result<(RequestPermit, ResolvedRequestContext), ResolutionError> {
let span = tracing::info_span!(
@@ -225,21 +237,30 @@ impl InferenceService {
let resolved = {
let _active = crate::observability::Active::new("resolution");
self.control_plane
- .resolve(gateway_key, api_format, model, correlation.id())
+ .resolve(
+ gateway_key,
+ api_format,
+ model,
+ request_bytes,
+ correlation.id(),
+ )
.await
};
let span = tracing::Span::current();
span.record(
"gateway.outcome",
- if resolved.is_ok() {
- "admitted"
- } else {
- "resolution_failed"
+ match &resolved {
+ Ok(_) => "admitted",
+ Err(ResolutionError::RateLimited(_)) => "rate_limited",
+ Err(_) => "resolution_failed",
},
);
- // Caller-caused failures (bad key, forbidden model) stay unset so
- // Datadog error rates track gateway and control-plane faults only.
- if let Err(error) = resolved
+ // Caller-caused failures (bad key, forbidden model, rate limits) stay unset
+ // so Datadog error rates track gateway and control-plane faults only.
+ if let Err(ResolutionError::RateLimited(limit)) = &resolved {
+ correlation.record_rate_limit(limit);
+ }
+ if let Err(error) = &resolved
&& let (origin, reason) = resolution_origin(error)
&& origin.is_ours()
{
diff --git a/ai-gateway/src/observability/http/tests.rs b/ai-gateway/src/observability/http/tests.rs
index 2a511a2e2d..eb4cb0b265 100644
--- a/ai-gateway/src/observability/http/tests.rs
+++ b/ai-gateway/src/observability/http/tests.rs
@@ -304,6 +304,60 @@ async fn failed_responses_name_their_origin_on_the_server_span() {
}
}
+/// Langfuse's own rate limit is a caller error, kept apart from the provider's 429.
+#[tokio::test]
+async fn organization_rate_limits_are_tagged_on_the_server_and_resolution_spans() {
+ let web = FakeServer::start(|_| async {
+ Response::builder()
+ .status(429)
+ .header("retry-after", "5")
+ .header("langfuse-gateway-error-code", "rate_limit_exceeded")
+ .body(Body::from(
+ r#"{"error":"Langfuse AI Gateway rate limit exceeded","limit":"bytes","organizationId":"org-1"}"#,
+ ))
+ .unwrap()
+ })
+ .await;
+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
+ let service = InferenceService::for_test(
+ web.control_plane(),
+ ProviderTransport::for_test(provider.url.clone(), ProviderLimits::default()),
+ );
+ let recording = Recording::start();
+ let lifecycle = GatewayLifecycleState::default();
+ let app = instrument(crate::http::router(Some(service), lifecycle.clone()));
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let mut serving = Box::pin(crate::server::serve(
+ listener,
+ app.clone(),
+ lifecycle,
+ std::future::pending(),
+ Duration::from_secs(1),
+ ));
+ assert!(futures_util::poll!(serving.as_mut()).is_pending());
+ let response = app.oneshot(generation_request()).await.unwrap();
+ assert_eq!(response.status(), 429);
+ axum::body::to_bytes(response.into_body(), 4096)
+ .await
+ .unwrap();
+
+ let spans = recording.exporter.get_finished_spans().unwrap();
+ let server = spans
+ .iter()
+ .find(|span| span.span_kind == SpanKind::Server)
+ .unwrap();
+ assert_attribute(server, "error.origin", "client");
+ assert_attribute(server, "error.type", "rate_limited_bytes");
+ assert_attribute(server, "langfuse.organization.id", "org-1");
+ assert!(!matches!(
+ server.status,
+ opentelemetry::trace::Status::Error { .. }
+ ));
+ let resolution = spans.iter().find(|span| span.name == "resolution").unwrap();
+ assert_attribute(resolution, "gateway.outcome", "rate_limited");
+ assert_eq!(provider.calls(), 0);
+}
+
/// Token counting captures no generation, yet a body that breaks after the
/// headers must still fail the server span.
#[tokio::test]
diff --git a/ai-gateway/src/providers/tests.rs b/ai-gateway/src/providers/tests.rs
index 67de442f91..51a8a6b073 100644
--- a/ai-gateway/src/providers/tests.rs
+++ b/ai-gateway/src/providers/tests.rs
@@ -112,6 +112,7 @@ async fn missing_provider_model_fails_resolution() {
"gateway-secret",
ApiFormat::OpenAiResponses,
"Production key:gpt-4o",
+ 0,
"test-request-id",
)
.await;
diff --git a/ai-gateway/src/resolution/mod.rs b/ai-gateway/src/resolution/mod.rs
index aea67710ed..4f51f45a83 100644
--- a/ai-gateway/src/resolution/mod.rs
+++ b/ai-gateway/src/resolution/mod.rs
@@ -8,7 +8,7 @@ pub use contracts::{
};
use reqwest::{
Client, Method, Url,
- header::{AUTHORIZATION, CONTENT_TYPE, HeaderValue},
+ header::{AUTHORIZATION, CONTENT_TYPE, HeaderValue, RETRY_AFTER},
};
use reqwest_middleware::{ClientWithMiddleware, RequestBuilder};
use secrecy::{ExposeSecret, SecretString};
@@ -21,6 +21,12 @@ const RESOLVE_PATH: &str = "/api/internal/ai-gateway/v1/resolve";
const MODELS_PATH: &str = "/api/internal/ai-gateway/v1/models";
const REQUEST_ID_HEADER: &str = "langfuse-gateway-request-id";
const ERROR_CODE_HEADER: &str = "langfuse-gateway-error-code";
+const RATE_LIMIT_ERROR_CODE: &str = "rate_limit_exceeded";
+/// Shown when Web's rate-limit body is missing or unreadable.
+const RATE_LIMIT_FALLBACK_MESSAGE: &str =
+ "Langfuse AI Gateway rate limit exceeded for your organization's plan.";
+/// Web's messages are a sentence; a longer one is not relayed to the caller.
+const MAX_RATE_LIMIT_MESSAGE_BYTES: usize = 1024;
// Web sits behind load balancers that drop idle connections after 60s by default
// (AWS ALB). Evicting pooled connections first keeps the resolver from sending on a
// socket the balancer is closing, which would fail without a retry.
@@ -156,24 +162,34 @@ impl ControlPlaneClient {
}
/// Resolve a gateway credential and API format into a validated execution contract.
- /// `request_id` is the gateway request ID, sent so Web logs carry it.
+ /// `request_bytes` is the client body size, counted against the organization's byte
+ /// rate limit. `request_id` is the gateway request ID, sent so Web logs carry it.
///
/// # Errors
/// Returns a sanitized [`ResolutionError`] for invalid credentials, authentication or
- /// authorization failures, missing routes, unavailable Web or system time, request
- /// construction or transport failures, timeouts, or oversized or invalid responses.
+ /// authorization failures, organization rate limits, missing routes, unavailable Web
+ /// or system time, request construction or transport failures, timeouts, or oversized
+ /// or invalid responses.
pub async fn resolve(
&self,
gateway_key: &str,
api_format: ApiFormat,
model: &str,
+ request_bytes: usize,
request_id: &str,
) -> Result<ResolvedRequestContext, ResolutionError> {
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|_| ResolutionError::Unavailable)?;
- self.resolve_at(gateway_key, api_format, model, timestamp, request_id)
- .await
+ self.resolve_at(
+ gateway_key,
+ api_format,
+ model,
+ request_bytes,
+ timestamp,
+ request_id,
+ )
+ .await
}
/// Fetch the caller-visible model catalog from trusted Web. `reserve` is called
@@ -212,6 +228,7 @@ impl ControlPlaneClient {
gateway_key: &str,
api_format: ApiFormat,
model: &str,
+ request_bytes: usize,
timestamp: Duration,
request_id: &str,
) -> Result<ResolvedRequestContext, ResolutionError> {
@@ -219,6 +236,7 @@ impl ControlPlaneClient {
let body = serde_json::to_vec(&serde_json::json!({
"apiFormat": api_format,
"model": model,
+ "requestBytes": request_bytes,
}))
.map_err(|_| ResolutionError::Configuration)?;
let request = self
@@ -288,8 +306,8 @@ impl ControlPlaneClient {
}
/// Like [`Self::execute`], but resends once, immediately, when no usable response
- /// arrived. Resolve is read-only on Web, so a duplicate is harmless. Both attempts
- /// share one deadline, so a timeout is never retried.
+ /// arrived. A duplicate only writes rate-limit counters on Web, so at worst it counts
+ /// one request twice. Both attempts share one deadline, so a timeout is never retried.
async fn execute_retrying_once(
&self,
request: RequestBuilder,
@@ -338,7 +356,7 @@ fn retry_reason(
}
async fn read_response(
- mut response: reqwest::Response,
+ response: reqwest::Response,
limit: usize,
reserve: &mut (dyn FnMut(usize) -> Result<(), ResolutionError> + Send),
) -> Result<Vec<u8>, ResolutionError> {
@@ -363,9 +381,26 @@ async fn read_response(
}
403 => return Err(ResolutionError::Forbidden),
404 => return Err(ResolutionError::NoRoute),
+ 429 if response
+ .headers()
+ .get(ERROR_CODE_HEADER)
+ .is_some_and(|value| value == RATE_LIMIT_ERROR_CODE) =>
+ {
+ return Err(ResolutionError::RateLimited(Box::new(
+ RateLimit::from_response(response, limit).await,
+ )));
+ }
429 | 500..=599 => return Err(ResolutionError::Unavailable),
_ => return Err(ResolutionError::InvalidResponse),
}
+ read_body(response, limit, reserve).await
+}
+
+async fn read_body(
+ mut response: reqwest::Response,
+ limit: usize,
+ reserve: &mut (dyn FnMut(usize) -> Result<(), ResolutionError> + Send),
+) -> Result<Vec<u8>, ResolutionError> {
let declared = response
.content_length()
.map(|size| usize::try_from(size).unwrap_or(usize::MAX));
@@ -396,12 +431,103 @@ async fn read_response(
Ok(bytes)
}
+/// Which of the organization's per-minute limits Web enforced.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum RateLimitKind {
+ Requests,
+ Bytes,
+}
+
+impl RateLimitKind {
+ #[must_use]
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Self::Requests => "requests",
+ Self::Bytes => "bytes",
+ }
+ }
+}
+
+/// Web rejected the request because the organization exceeded a plan rate limit.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct RateLimit {
+ kind: RateLimitKind,
+ retry_after_secs: u64,
+ message: Box<str>,
+ organization_id: Option<Box<str>>,
+}
+
+impl RateLimit {
+ /// Web writes the caller-facing message, so the gateway relays it unchanged.
+ async fn from_response(response: reqwest::Response, limit: usize) -> Self {
+ #[derive(serde::Deserialize)]
+ #[serde(rename_all = "camelCase")]
+ struct Body {
+ error: String,
+ limit: RateLimitKind,
+ organization_id: Option<String>,
+ }
+ let retry_after_secs = response
+ .headers()
+ .get(RETRY_AFTER)
+ .and_then(|value| value.to_str().ok()?.trim().parse::<u64>().ok())
+ .unwrap_or(1)
+ .clamp(1, 60 * 60);
+ let body = read_body(response, limit, &mut |_| Ok(()))
+ .await
+ .ok()
+ .and_then(|bytes| serde_json::from_slice::<Body>(&bytes).ok())
+ .filter(|body| {
+ !body.error.is_empty() && body.error.len() <= MAX_RATE_LIMIT_MESSAGE_BYTES
+ });
+ let Some(body) = body else {
+ return Self {
+ kind: RateLimitKind::Requests,
+ retry_after_secs,
+ message: RATE_LIMIT_FALLBACK_MESSAGE.into(),
+ organization_id: None,
+ };
+ };
+ Self {
+ kind: body.limit,
+ retry_after_secs,
+ message: body.error.into_boxed_str(),
+ organization_id: body
+ .organization_id
+ .filter(|id| valid_token(id) && id.len() <= 256)
+ .map(String::into_boxed_str),
+ }
+ }
+
+ #[must_use]
+ pub fn kind(&self) -> RateLimitKind {
+ self.kind
+ }
+
+ #[must_use]
+ pub fn retry_after_secs(&self) -> u64 {
+ self.retry_after_secs
+ }
+
+ #[must_use]
+ pub fn message(&self) -> &str {
+ &self.message
+ }
+
+ #[must_use]
+ pub fn organization_id(&self) -> Option<&str> {
+ self.organization_id.as_deref()
+ }
+}
+
fn valid_token(value: &str) -> bool {
!value.is_empty() && value.bytes().all(|byte| byte.is_ascii_graphic())
}
-/// Sanitized failure categories. Upstream bodies and transport errors are never retained.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
+/// Sanitized failure categories. Upstream bodies and transport errors are never retained,
+/// except the caller-facing message of a Web rate-limit rejection.
+#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum ResolutionError {
#[error("invalid resolver configuration")]
Configuration,
@@ -417,6 +543,8 @@ pub enum ResolutionError {
ModelNotAllowed,
#[error("no eligible connection")]
NoRoute,
+ #[error("organization rate limit exceeded")]
+ RateLimited(Box<RateLimit>),
/// The gateway's memory budget cannot cover another request.
#[error("gateway at capacity")]
Busy,
diff --git a/ai-gateway/src/resolution/tests.rs b/ai-gateway/src/resolution/tests.rs
index c1d55c6d26..a6552d7872 100644
--- a/ai-gateway/src/resolution/tests.rs
+++ b/ai-gateway/src/resolution/tests.rs
@@ -126,7 +126,7 @@ async fn signs_the_exact_key_and_sends_api_format_and_model() {
assert_eq!(request.headers()["langfuse-gateway-authorization"],
"HMAC timestamp=1800000000,signature=e104b6baa50d9b766b036e27876683aa1f4bff7f31bb6af66dce6b636a53b2bd");
assert_eq!(to_bytes(request.into_body(), 1024).await.unwrap(),
- r#"{"apiFormat":"openai.responses","model":"primary:gpt-4.1"}"#);
+ r#"{"apiFormat":"openai.responses","model":"primary:gpt-4.1","requestBytes":0}"#);
response(success("project-1", "private-provider-token"))
}).await;
@@ -136,6 +136,7 @@ async fn signs_the_exact_key_and_sends_api_format_and_model() {
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -240,6 +241,7 @@ async fn model_catalogs_have_a_larger_response_limit_than_resolution() {
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -274,6 +276,7 @@ async fn denied_and_failed_resolutions_are_classified_without_retries_or_body_le
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -297,6 +300,7 @@ async fn resolve_against(web: &FakeWeb) -> Result<ResolvedRequestContext, Resolu
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -371,6 +375,7 @@ async fn retries_once_when_web_closes_the_connection_without_answering() {
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -394,6 +399,7 @@ async fn timeouts_are_not_retried() {
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -420,6 +426,7 @@ async fn model_policy_rejections_are_distinct_from_other_forbidden_responses() {
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -430,6 +437,83 @@ async fn model_policy_rejections_are_distinct_from_other_forbidden_responses() {
assert!(!format!("{error:?} {error}").contains("private-upstream-error"));
}
+#[tokio::test]
+async fn organization_rate_limits_carry_web_message_and_retry_after() {
+ let web = FakeWeb::start(|_| async {
+ Response::builder()
+ .status(StatusCode::TOO_MANY_REQUESTS)
+ .header("retry-after", "17")
+ .header("langfuse-gateway-error-code", "rate_limit_exceeded")
+ .body(Body::from(
+ r#"{"error":"Langfuse AI Gateway rate limit exceeded: bytes","limit":"bytes","organizationId":"org-1"}"#,
+ ))
+ .unwrap()
+ })
+ .await;
+
+ let error = web
+ .control_plane()
+ .resolve_at(
+ GATEWAY_KEY,
+ ApiFormat::OpenAiResponses,
+ "primary:gpt-4.1",
+ 4096,
+ NOW_TIME,
+ REQUEST_ID,
+ )
+ .await
+ .unwrap_err();
+
+ let ResolutionError::RateLimited(limit) = error else {
+ panic!("expected a rate limit, got {error:?}");
+ };
+ assert_eq!(limit.kind(), RateLimitKind::Bytes);
+ assert_eq!(limit.retry_after_secs(), 17);
+ assert_eq!(
+ limit.message(),
+ "Langfuse AI Gateway rate limit exceeded: bytes"
+ );
+ assert_eq!(limit.organization_id(), Some("org-1"));
+ assert_eq!(web.calls(), 1);
+}
+
+#[tokio::test]
+async fn unreadable_rate_limit_bodies_fall_back_to_a_langfuse_message() {
+ let web = FakeWeb::start(|_| async {
+ Response::builder()
+ .status(StatusCode::TOO_MANY_REQUESTS)
+ .header("langfuse-gateway-error-code", "rate_limit_exceeded")
+ .body(Body::from("private-upstream-error"))
+ .unwrap()
+ })
+ .await;
+
+ let error = web
+ .control_plane()
+ .resolve_at(
+ GATEWAY_KEY,
+ ApiFormat::OpenAiResponses,
+ "primary:gpt-4.1",
+ 0,
+ NOW_TIME,
+ REQUEST_ID,
+ )
+ .await
+ .unwrap_err();
+
+ let ResolutionError::RateLimited(limit) = error else {
+ panic!("expected a rate limit, got {error:?}");
+ };
+ assert_eq!(limit.kind(), RateLimitKind::Requests);
+ assert_eq!(limit.retry_after_secs(), 1);
+ assert!(
+ limit
+ .message()
+ .starts_with("Langfuse AI Gateway rate limit exceeded")
+ );
+ assert!(!limit.message().contains("private-upstream-error"));
+}
+
/// A rejected gateway signature means the service keys disagree, which is the
/// gateway's fault, not the caller's.
#[tokio::test]
@@ -452,6 +536,7 @@ async fn rejected_gateway_signatures_are_configuration_errors() {
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -483,6 +568,7 @@ async fn never_follows_redirects_with_credentials() {
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID
)
@@ -515,6 +601,7 @@ async fn rejects_oversized_declared_and_chunked_responses() {
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -548,6 +635,7 @@ async fn deadline_covers_response_headers_and_body() {
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
),
@@ -569,6 +657,7 @@ async fn malformed_credentials_never_reach_web() {
key,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -593,6 +682,7 @@ async fn rejects_a_grant_that_expires_while_resolving_across_a_second_boundary()
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME + Duration::from_millis(900),
REQUEST_ID,
)
@@ -620,6 +710,7 @@ async fn concurrent_resolutions_keep_credentials_and_contexts_isolated() {
"gw_test_alice",
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID
),
@@ -627,6 +718,7 @@ async fn concurrent_resolutions_keep_credentials_and_contexts_isolated() {
"gw_test_bob",
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID
),
@@ -671,7 +763,7 @@ async fn resolves_anthropic_messages_with_the_native_credential_scheme() {
let web = FakeWeb::start(|request| async move {
assert_eq!(
to_bytes(request.into_body(), 1024).await.unwrap(),
- r#"{"apiFormat":"anthropic.messages","model":"primary:claude-sonnet-4-5"}"#
+ r#"{"apiFormat":"anthropic.messages","model":"primary:claude-sonnet-4-5","requestBytes":0}"#
);
response(anthropic_success("private-anthropic-key"))
})
@@ -683,6 +775,7 @@ async fn resolves_anthropic_messages_with_the_native_credential_scheme() {
GATEWAY_KEY,
ApiFormat::AnthropicMessages,
"primary:claude-sonnet-4-5",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -722,7 +815,7 @@ async fn chat_completions_resolve_only_an_openai_connection_for_that_format() {
async move {
assert_eq!(
to_bytes(request.into_body(), 1024).await.unwrap(),
- r#"{"apiFormat":"openai.chat-completions","model":"primary:gpt-4.1"}"#
+ r#"{"apiFormat":"openai.chat-completions","model":"primary:gpt-4.1","requestBytes":0}"#
);
response(body)
}
@@ -734,6 +827,7 @@ async fn chat_completions_resolve_only_an_openai_connection_for_that_format() {
GATEWAY_KEY,
ApiFormat::OpenAiChatCompletions,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -900,6 +994,7 @@ async fn accepts_unknown_response_fields_outside_provider_auth() {
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -932,6 +1027,7 @@ async fn treats_missing_or_blank_key_snapshots_as_absent() {
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -986,6 +1082,7 @@ async fn rejects_ingestion_tokens_that_authorize_a_different_tenant() {
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID
)
@@ -1011,6 +1108,7 @@ async fn resolves_none_mode_without_an_ingestion_grant_or_project() {
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -1044,6 +1142,7 @@ async fn resolves_keys_that_are_not_bound_to_a_user() {
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -1089,6 +1188,7 @@ async fn assert_invalid_context_for(body: Value, api_format: ApiFormat) {
GATEWAY_KEY,
api_format,
"primary:provider-model",
+ 0,
NOW_TIME,
REQUEST_ID,
)
@@ -1109,6 +1209,7 @@ async fn rejects_malformed_json_without_exposing_response_contents() {
GATEWAY_KEY,
ApiFormat::OpenAiResponses,
"primary:gpt-4.1",
+ 0,
NOW_TIME,
REQUEST_ID,
)
diff --git a/ai-gateway/src/test_support.rs b/ai-gateway/src/test_support.rs
index 1ca1633aea..9be7f39b2f 100644
--- a/ai-gateway/src/test_support.rs
+++ b/ai-gateway/src/test_support.rs
@@ -182,6 +182,7 @@ pub(crate) async fn resolved_request_context_for(
"gateway-secret",
api_format,
"Production key:provider-model",
+ 0,
"test-request-id",
)
.await
diff --git a/packages/shared/src/interfaces/rate-limits.ts b/packages/shared/src/interfaces/rate-limits.ts
index 72a5d3b001..91b78fcc3f 100644
--- a/packages/shared/src/interfaces/rate-limits.ts
+++ b/packages/shared/src/interfaces/rate-limits.ts
@@ -17,6 +17,9 @@ export const RateLimitResource = z.enum([
"score-delete",
"in-app-agent-run",
"feedback",
+ "ai-gateway-requests",
+ // Points are request body bytes, not requests.
+ "ai-gateway-request-bytes",
]);
// result of the rate limit check.
diff --git a/web/src/__e2e__/ai-gateway.gatewaye2e.ts b/web/src/__e2e__/ai-gateway.gatewaye2e.ts
index 078bb6afd4..22b05a66f9 100644
--- a/web/src/__e2e__/ai-gateway.gatewaye2e.ts
+++ b/web/src/__e2e__/ai-gateway.gatewaye2e.ts
@@ -843,7 +843,11 @@ async function gatewayControlPlaneRequest(input: {
"Content-Type": "application/json",
"Langfuse-Gateway-Authorization": gatewayAuthorization,
},
- body: JSON.stringify({ apiFormat: input.apiFormat, model: input.model }),
+ body: JSON.stringify({
+ apiFormat: input.apiFormat,
+ model: input.model,
+ requestBytes: 0,
+ }),
});
return readJsonResponse(response, 200, RESOLVE_PATH);
}
diff --git a/web/src/__tests__/server/ai-gateway-rate-limit.servertest.ts b/web/src/__tests__/server/ai-gateway-rate-limit.servertest.ts
new file mode 100644
index 0000000000..823b0952b1
--- /dev/null
+++ b/web/src/__tests__/server/ai-gateway-rate-limit.servertest.ts
@@ -0,0 +1,191 @@
+import { randomUUID } from "crypto";
+import type { Redis } from "ioredis";
+import type { NextApiRequest, NextApiResponse } from "next";
+import { createMocks } from "node-mocks-http";
+import { logger } from "@langfuse/shared/src/server";
+
+import {
+ clearRedisKeysByPatternSafely,
+ createRedisTestClient,
+ ensureRedisReady,
+ type RedisTestClient,
+} from "@/src/__tests__/server/redis-test-utils";
+import {
+ type GatewayApiKeyAuthContext,
+ GatewayApiKeyAuthenticator,
+} from "@/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator";
+import {
+ enforceGatewayRateLimits,
+ GatewayRateLimitError,
+} from "@/src/features/ai-gateway/server/resolve/gatewayRateLimit";
+import { gatewayResolveApiHandler } from "@/src/features/ai-gateway/server/resolve/gatewayResolveApiHandler";
+import {
+ RATE_LIMIT_REDIS_KEY_PREFIX,
+ RateLimitService,
+} from "@/src/features/public-api/server/RateLimitService";
+
+const MiB = 1024 * 1024;
+
+describe("AI Gateway organization rate limits", () => {
+ let redis: RedisTestClient;
+ let organizationId: string;
+
+ const contextFor = (
+ overrides: Partial<GatewayApiKeyAuthContext> = {},
+ ): GatewayApiKeyAuthContext => ({
+ organizationId,
+ apiKeyId: "gateway-key-1",
+ userId: null,
+ apiKeyName: null,
+ userName: null,
+ userEmail: null,
+ keyMetadata: {},
+ ingestionProjectId: "project-1",
+ ingestionMode: "USAGE",
+ plan: "cloud:hobby",
+ rateLimitOverrides: [],
+ connection: null,
+ providerModel: null,
+ ...overrides,
+ });
+
+ const rejection = async (promise: Promise<unknown>) => {
+ const error = await promise.then(
+ () => null,
+ (error: unknown) => error,
+ );
+ expect(error).toBeInstanceOf(GatewayRateLimitError);
+ return error as GatewayRateLimitError;
+ };
+
+ beforeAll(async () => {
+ RateLimitService.shutdown();
+ redis = createRedisTestClient({
+ maxRetriesPerRequest: null,
+ enableAutoPipelining: false,
+ });
+ await ensureRedisReady(redis);
+ RateLimitService.getInstance(redis as Redis);
+ }, 20_000);
+
+ beforeEach(() => {
+ organizationId = `ai-gateway-rate-limit-org-${randomUUID()}`;
+ });
+
+ afterEach(async () => {
+ vi.restoreAllMocks();
+ await clearRedisKeysByPatternSafely(
+ redis,
+ `${RATE_LIMIT_REDIS_KEY_PREFIX}:*:${organizationId}`,
+ );
+ }, 20_000);
+
+ afterAll(() => {
+ redis.disconnect();
+ RateLimitService.shutdown();
+ });
+
+ it("admits the plan's requests per minute and rejects the next one with a Langfuse message", async () => {
+ const warn = vi.spyOn(logger, "warn");
+ const context = contextFor();
+ for (let i = 0; i < 100; i++) {
+ await enforceGatewayRateLimits({ context, requestBytes: 10 });
+ }
+
+ const error = await rejection(
+ enforceGatewayRateLimits({ context, requestBytes: 10 }),
+ );
+
+ expect(error.status).toBe(429);
+ expect(error.code).toBe("rate_limit_exceeded");
+ expect(error.limit).toBe("requests");
+ expect(error.retryAfterSeconds).toBeGreaterThanOrEqual(1);
+ expect(error.retryAfterSeconds).toBeLessThanOrEqual(60);
+ expect(error.message).toBe(
+ `Langfuse AI Gateway rate limit exceeded: your organization's Hobby plan allows 100 requests per minute. Retry after ${error.retryAfterSeconds}s or upgrade your plan.`,
+ );
+ expect(warn).toHaveBeenCalledWith("AI Gateway rate limit exceeded", {
+ orgId: organizationId,
+ projectId: "project-1",
+ apiKeyId: "gateway-key-1",
+ plan: "cloud:hobby",
+ limit: "requests",
+ current: 101,
+ limitValue: 100,
+ requestBytes: 10,
+ retryAfterSeconds: error.retryAfterSeconds,
+ });
+ });
+
+ it("counts request bodies toward the per-minute bytes limit", async () => {
+ const context = contextFor({ plan: "cloud:core" });
+ await enforceGatewayRateLimits({ context, requestBytes: 256 * MiB });
+
+ const error = await rejection(
+ enforceGatewayRateLimits({ context, requestBytes: 1 }),
+ );
+
+ expect(error.limit).toBe("bytes");
+ expect(error.message).toBe(
+ `Langfuse AI Gateway rate limit exceeded: your organization's Core plan allows 256 MiB of request bodies per minute. Retry after ${error.retryAfterSeconds}s or upgrade your plan.`,
+ );
+ });
+
+ it("names a custom limit, its window, and support when the organization has an override", async () => {
+ const context = contextFor({
+ plan: "cloud:enterprise",
+ rateLimitOverrides: [
+ { resource: "ai-gateway-requests", points: 2, durationInSec: 3600 },
+ ],
+ });
+ await enforceGatewayRateLimits({ context, requestBytes: 0 });
+ await enforceGatewayRateLimits({ context, requestBytes: 0 });
+
+ const error = await rejection(
+ enforceGatewayRateLimits({ context, requestBytes: 0 }),
+ );
+
+ expect(error.message).toBe(
+ `Langfuse AI Gateway rate limit exceeded: your organization's custom limit allows 2 requests per hour. Retry after ${error.retryAfterSeconds}s or contact Langfuse support to raise the limit.`,
+ );
+ });
+
+ it("never limits self-hosted plans", async () => {
+ const context = contextFor({ plan: "oss" });
+
+ await expect(
+ enforceGatewayRateLimits({ context, requestBytes: 1024 * MiB }),
+ ).resolves.toBeUndefined();
+ });
+
+ it("answers /resolve with a 429 the gateway can map to a provider-native error", async () => {
+ vi.spyOn(
+ GatewayApiKeyAuthenticator.prototype,
+ "authenticateRequest",
+ ).mockResolvedValue(contextFor());
+ const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
+ method: "POST",
+ });
+
+ await gatewayResolveApiHandler({
+ req,
+ res,
+ fastHashedSecretKey: "hashed-secret-key",
+ apiFormat: "openai.responses",
+ model: "gpt-4o-mini",
+ requestBytes: 64 * MiB + 1,
+ });
+
+ expect(res.statusCode).toBe(429);
+ const retryAfter = Number(res.getHeader("Retry-After"));
+ expect(retryAfter).toBeGreaterThanOrEqual(1);
+ expect(res.getHeader("langfuse-gateway-error-code")).toBe(
+ "rate_limit_exceeded",
+ );
+ expect(res._getJSONData()).toEqual({
+ error: `Langfuse AI Gateway rate limit exceeded: your organization's Hobby plan allows 64 MiB of request bodies per minute. Retry after ${retryAfter}s or upgrade your plan.`,
+ limit: "bytes",
+ organizationId,
+ });
+ });
+});
diff --git a/web/src/__tests__/server/organizations-api-key-trpc.servertest.ts b/web/src/__tests__/server/organizations-api-key-trpc.servertest.ts
index 0d61efc0c6..54aad9ab47 100644
--- a/web/src/__tests__/server/organizations-api-key-trpc.servertest.ts
+++ b/web/src/__tests__/server/organizations-api-key-trpc.servertest.ts
@@ -468,6 +468,8 @@ describe("organization API keys trpc", () => {
keyMetadata: {},
ingestionProjectId: null,
ingestionMode: null,
+ plan: "cloud:hobby",
+ rateLimitOverrides: [],
connections: [],
},
});
@@ -635,6 +637,8 @@ describe("organization API keys trpc", () => {
keyMetadata: {},
ingestionProjectId: null,
ingestionMode: null,
+ plan: "cloud:hobby",
+ rateLimitOverrides: [],
connections: [],
},
});
diff --git a/web/src/__tests__/server/rate-limit.servertest.ts b/web/src/__tests__/server/rate-limit.servertest.ts
index 481774432c..70816d67d6 100644
--- a/web/src/__tests__/server/rate-limit.servertest.ts
+++ b/web/src/__tests__/server/rate-limit.servertest.ts
@@ -791,6 +791,65 @@ describe("RateLimitService", () => {
}
});
+ it("should apply AI Gateway request and request-byte limits by cloud plan", async () => {
+ const MiB = 1024 * 1024;
+ const GiB = 1024 * MiB;
+ const cases = [
+ { plan: "cloud:hobby" as const, requests: 100, requestBytes: 64 * MiB },
+ { plan: "cloud:core" as const, requests: 500, requestBytes: 256 * MiB },
+ { plan: "cloud:pro" as const, requests: 3_000, requestBytes: 2 * GiB },
+ { plan: "cloud:team" as const, requests: 3_000, requestBytes: 2 * GiB },
+ {
+ plan: "cloud:enterprise" as const,
+ requests: 3_000,
+ requestBytes: 2 * GiB,
+ },
+ ];
+
+ const rateLimitService = RateLimitService.getInstance(redis as Redis);
+
+ for (const testCase of cases) {
+ await redis.del(
+ `${RATE_LIMIT_REDIS_KEY_PREFIX}:ai-gateway-requests:${orgId}`,
+ );
+ await redis.del(
+ `${RATE_LIMIT_REDIS_KEY_PREFIX}:ai-gateway-request-bytes:${orgId}`,
+ );
+ const scope = asScope({
+ orgId,
+ plan: testCase.plan,
+ projectId,
+ accessLevel: "organization",
+ rateLimitOverrides: [],
+ });
+
+ const requests = await rateLimitService.rateLimitRequest(
+ scope,
+ "ai-gateway-requests",
+ );
+ const bytes = await rateLimitService.rateLimitRequest(
+ scope,
+ "ai-gateway-request-bytes",
+ 4096,
+ );
+
+ expect(requests?.res).toMatchObject({
+ points: testCase.requests,
+ consumedPoints: 1,
+ });
+ expect(bytes?.res).toMatchObject({
+ points: testCase.requestBytes,
+ consumedPoints: 4096,
+ remainingPoints: testCase.requestBytes - 4096,
+ });
+ const ttlInSec = await redis.ttl(
+ `${RATE_LIMIT_REDIS_KEY_PREFIX}:ai-gateway-request-bytes:${orgId}`,
+ );
+ expect(ttlInSec).toBeGreaterThan(0);
+ expect(ttlInSec).toBeLessThanOrEqual(60);
+ }
+ });
+
it("should apply media-upload rate limits separately from ingestion", async () => {
const scope = {
orgId: orgId,
diff --git a/web/src/__tests__/server/unit/ai-gateway-resolve-route.servertest.ts b/web/src/__tests__/server/unit/ai-gateway-resolve-route.servertest.ts
index e5ce62da0d..9a6289dc40 100644
--- a/web/src/__tests__/server/unit/ai-gateway-resolve-route.servertest.ts
+++ b/web/src/__tests__/server/unit/ai-gateway-resolve-route.servertest.ts
@@ -51,6 +51,26 @@ describe("POST /api/internal/ai-gateway/v1/resolve", () => {
body: { apiFormat: "openai.responses", limit: 10 },
},
{ case: "not an object", body: "openai.responses" },
+ {
+ case: "missing request bytes",
+ body: { apiFormat: "openai.responses", model: "gpt-4o-mini" },
+ },
+ {
+ case: "negative request bytes",
+ body: {
+ apiFormat: "openai.responses",
+ model: "gpt-4o-mini",
+ requestBytes: -1,
+ },
+ },
+ {
+ case: "fractional request bytes",
+ body: {
+ apiFormat: "openai.responses",
+ model: "gpt-4o-mini",
+ requestBytes: 1.5,
+ },
+ },
])("rejects an invalid body: $case", async ({ body }) => {
expect((await post(body)).statusCode).toBe(400);
});
@@ -67,6 +87,7 @@ describe("POST /api/internal/ai-gateway/v1/resolve", () => {
await post({
apiFormat: format,
model: "production:model-with:separator",
+ requestBytes: 0,
})
).statusCode,
).toBe(401);
@@ -78,7 +99,9 @@ describe("POST /api/internal/ai-gateway/v1/resolve", () => {
])(
"accepts inferred provider model $model and advances to authorization",
async ({ model, apiFormat }) => {
- expect((await post({ apiFormat, model })).statusCode).toBe(401);
+ expect(
+ (await post({ apiFormat, model, requestBytes: 1024 })).statusCode,
+ ).toBe(401);
},
);
@@ -91,15 +114,27 @@ describe("POST /api/internal/ai-gateway/v1/resolve", () => {
it.each([
{
case: "unknown unprefixed model",
- body: { apiFormat: "openai.responses", model: "random-model" },
+ body: {
+ apiFormat: "openai.responses",
+ model: "random-model",
+ requestBytes: 0,
+ },
},
{
case: "empty connection name",
- body: { apiFormat: "openai.responses", model: ":gpt-4o-mini" },
+ body: {
+ apiFormat: "openai.responses",
+ model: ":gpt-4o-mini",
+ requestBytes: 0,
+ },
},
{
case: "empty provider model",
- body: { apiFormat: "openai.responses", model: "production:" },
+ body: {
+ apiFormat: "openai.responses",
+ model: "production:",
+ requestBytes: 0,
+ },
},
])("rejects an invalid model slug: $case", async ({ body }) => {
expect((await post(body)).statusCode).toBe(400);
diff --git a/web/src/__tests__/server/unit/gatewayResolveCache.servertest.ts b/web/src/__tests__/server/unit/gatewayResolveCache.servertest.ts
index 5661d44397..1a4553e2d8 100644
--- a/web/src/__tests__/server/unit/gatewayResolveCache.servertest.ts
+++ b/web/src/__tests__/server/unit/gatewayResolveCache.servertest.ts
@@ -23,6 +23,8 @@ const context: CachedResolveContext = {
keyMetadata: {},
ingestionProjectId: null,
ingestionMode: null,
+ plan: "cloud:hobby",
+ rateLimitOverrides: [],
connections: [],
};
@@ -118,6 +120,8 @@ describe("GatewayResolveCache", () => {
keyMetadata: {},
ingestionProjectId: null,
ingestionMode: null,
+ plan: "cloud:hobby",
+ rateLimitOverrides: [],
connections: [],
};
await cache.set({
@@ -139,6 +143,28 @@ describe("GatewayResolveCache", () => {
});
});
+ it("refetches entries cached before the plan was part of the context", async () => {
+ const redis = createStatefulRedis();
+ const cache = new GatewayResolveCache(redis);
+ const {
+ plan: _plan,
+ rateLimitOverrides: _overrides,
+ ...withoutPlan
+ } = context;
+ await cache.set({
+ fastHashedSecretKey: "hashed-key",
+ apiFormat: "openai.responses",
+ context: withoutPlan as unknown as CachedResolveContext,
+ });
+
+ await expect(
+ cache.get({
+ fastHashedSecretKey: "hashed-key",
+ apiFormat: "openai.responses",
+ }),
+ ).resolves.toBeNull();
+ });
+
it("invalidates cached contexts for an organization", async () => {
const cache = new GatewayResolveCache(createStatefulRedis());
diff --git a/web/src/ee/features/billing/server/chb/chbWebhookHandler.ts b/web/src/ee/features/billing/server/chb/chbWebhookHandler.ts
index cbcfb11e77..fd69f58b47 100644
--- a/web/src/ee/features/billing/server/chb/chbWebhookHandler.ts
+++ b/web/src/ee/features/billing/server/chb/chbWebhookHandler.ts
@@ -5,6 +5,7 @@ import { z } from "zod";
import { env } from "@/src/env.mjs";
import { auditLog } from "@/src/features/audit-logs/server";
+import { invalidateGatewayResolveCacheForOrganization } from "@/src/features/ai-gateway/server/resolve/gatewayResolveCache";
import {
isoDateTime,
type ChbPlanCode,
@@ -437,6 +438,7 @@ async function persistAndPropagate(params: {
// TTL.
try {
await invalidateCachedOrgApiKeys(parsedOrg.id);
+ await invalidateGatewayResolveCacheForOrganization(parsedOrg.id);
} catch (error) {
logger.error(
`[CHB Webhook] Failed to invalidate cached API keys for org ${parsedOrg.id} after applying ${event.type}`,
diff --git a/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.ts b/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.ts
index 6f81e239d5..df6116958f 100644
--- a/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.ts
+++ b/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.ts
@@ -22,6 +22,7 @@ import {
startOfDayUTC,
} from "@langfuse/shared/src/server";
import { auditLog } from "@/src/features/audit-logs/server";
+import { invalidateGatewayResolveCacheForOrganization } from "@/src/features/ai-gateway/server/resolve/gatewayResolveCache";
import { type StripeSubscriptionMetadata } from "@/src/ee/features/billing/utils/stripeSubscriptionMetadata";
import { syncOrgPlanChangeToSfdc } from "@/src/ee/features/sfdc-sync/server";
import { createDefaultSpendAlertsForStripeProduct } from "@/src/ee/features/billing/server/defaultSpendAlerts";
@@ -676,6 +677,7 @@ export async function handleSubscriptionChanged(
// Invalidate API keys in Redis for it to be updated
await invalidateCachedOrgApiKeys(parsedOrg.id);
+ await invalidateGatewayResolveCacheForOrganization(parsedOrg.id);
auditLog({
session: {
@@ -725,6 +727,7 @@ export async function handleSubscriptionChanged(
// Reset billing cycle anchor on downgrade to hobby to start of today
await updateOrgBillingCycleAnchor(parsedOrg.id);
await invalidateCachedOrgApiKeys(parsedOrg.id);
+ await invalidateGatewayResolveCacheForOrganization(parsedOrg.id);
auditLog({
session: {
diff --git a/web/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator.ts b/web/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator.ts
index a235604666..f6497ff3f1 100644
--- a/web/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator.ts
+++ b/web/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator.ts
@@ -1,9 +1,11 @@
+import { CloudConfigSchema } from "@langfuse/shared";
import type { GatewayProvider, PrismaClient } from "@langfuse/shared/src/db";
import { ApiKeyId, OrganizationId } from "@langfuse/shared/rbac";
import { authorize } from "@/src/features/rbac/authorize";
import { getRolesForPrincipal } from "@/src/features/rbac/getRolesForPrincipal";
import { hasOrganizationAccess } from "@/src/features/rbac/utils/checkOrganizationAccess";
+import { getOrganizationPlanServerSide } from "@/src/features/entitlements/server";
import { isGatewayEnabledForOrganization } from "@/src/features/ai-gateway/server/availability";
import { GatewayControlPlaneError } from "@/src/features/ai-gateway/server/gatewayControlPlaneError";
import {
@@ -146,6 +148,9 @@ export class GatewayApiKeyAuthenticator {
Boolean(project) &&
!project?.deletedAt &&
project?.orgId === organizationId;
+ const cloudConfig = organization.cloudConfig
+ ? CloudConfigSchema.safeParse(organization.cloudConfig).data
+ : undefined;
const context: CachedResolveContext = {
organizationId,
apiKeyId: row.id,
@@ -158,6 +163,8 @@ export class GatewayApiKeyAuthenticator {
? (gatewayConfig?.defaultIngestionProjectId ?? null)
: null,
ingestionMode: gatewayConfig?.ingestionMode ?? null,
+ plan: getOrganizationPlanServerSide(cloudConfig),
+ rateLimitOverrides: cloudConfig?.rateLimitOverrides ?? [],
connections: organization.gatewayAiConnections.map((connection) => ({
...connection,
allowedModels: parseGatewayAllowedModels(connection.allowedModels),
diff --git a/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.ts b/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.ts
index 125e3f4e4c..4b5f18f08d 100644
--- a/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.ts
+++ b/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.ts
@@ -110,7 +110,8 @@ describe("withGatewayResolveSignatureVerification", () => {
async (secret) => {
vi.useFakeTimers();
vi.setSystemTime(now);
- const body = '{ "apiFormat": "openai.responses", "model": "gpt-5" }\n';
+ const body =
+ '{ "apiFormat": "openai.responses", "model": "gpt-5", "requestBytes": 512 }\n';
const req = request({
authorization: " Bearer sk-gateway ",
gatewayAuthorization: gatewayAuthorization({ secret }),
@@ -127,6 +128,7 @@ describe("withGatewayResolveSignatureVerification", () => {
fastHashedSecretKey: createShaHash("sk-gateway", "test-salt"),
apiFormat: "openai.responses",
model: "gpt-5",
+ requestBytes: 512,
});
},
);
@@ -143,6 +145,7 @@ describe("withGatewayResolveSignatureVerification", () => {
body: JSON.stringify({
apiFormat: "openai.responses",
model,
+ requestBytes: 0,
}),
}),
res,
@@ -205,7 +208,7 @@ describe("withGatewayResolveSignatureVerification", () => {
secret: "current-service-secret",
key: "sk-other-gateway",
}),
- body: '{"apiFormat":"openai.responses","model":"gpt-5"}',
+ body: '{"apiFormat":"openai.responses","model":"gpt-5","requestBytes":0}',
}),
res,
);
@@ -221,7 +224,8 @@ describe("withGatewayResolveSignatureVerification", () => {
it("rejects stale signatures and headers containing a key id", async () => {
vi.useFakeTimers();
vi.setSystemTime(now);
- const body = '{"apiFormat":"openai.responses","model":"gpt-5"}';
+ const body =
+ '{"apiFormat":"openai.responses","model":"gpt-5","requestBytes":0}';
const staleHeader = gatewayAuthorization({
secret: "current-service-secret",
timestamp: Math.floor(now.getTime() / 1000) - 301,
diff --git a/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.ts b/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.ts
index 854b1740c6..6d9dd84bf5 100644
--- a/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.ts
+++ b/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.ts
@@ -23,6 +23,7 @@ const resolveBodySchema = z
model: z
.string()
.refine((model) => modelRoutingService.resolve(model, []) !== null),
+ requestBytes: z.number().int().nonnegative(),
})
.strict();
const modelsQuerySchema = z
@@ -43,6 +44,7 @@ type GatewayResolveHandler = (
params: GatewayControlPlaneHandlerParams & {
apiFormat: GatewayApiFormat;
model: string;
+ requestBytes: number;
},
) => Promise<unknown>;
@@ -119,6 +121,7 @@ export function withGatewayResolveSignatureVerification(
...params,
apiFormat: body.apiFormat,
model: body.model,
+ requestBytes: body.requestBytes,
}),
resolveBodySchema,
);
diff --git a/web/src/features/ai-gateway/server/gatewayControlPlaneError.ts b/web/src/features/ai-gateway/server/gatewayControlPlaneError.ts
index abf519c54c..6d13f7e5e1 100644
--- a/web/src/features/ai-gateway/server/gatewayControlPlaneError.ts
+++ b/web/src/features/ai-gateway/server/gatewayControlPlaneError.ts
@@ -1,8 +1,11 @@
export class GatewayControlPlaneError extends Error {
constructor(
message: string,
- readonly status: 400 | 401 | 403 | 404 | 503,
- readonly code?: "model_not_allowed" | "invalid_gateway_authorization",
+ readonly status: 400 | 401 | 403 | 404 | 429 | 503,
+ readonly code?:
+ | "model_not_allowed"
+ | "invalid_gateway_authorization"
+ | "rate_limit_exceeded",
) {
super(message);
}
diff --git a/web/src/features/ai-gateway/server/resolve/gatewayRateLimit.ts b/web/src/features/ai-gateway/server/resolve/gatewayRateLimit.ts
new file mode 100644
index 0000000000..5c3e681a25
--- /dev/null
+++ b/web/src/features/ai-gateway/server/resolve/gatewayRateLimit.ts
@@ -0,0 +1,143 @@
+import { planLabels, type Plan, type RateLimitResult } from "@langfuse/shared";
+import { getCurrentSpan, logger } from "@langfuse/shared/src/server";
+
+import type { GatewayApiKeyAuthContext } from "@/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator";
+import { GatewayControlPlaneError } from "@/src/features/ai-gateway/server/gatewayControlPlaneError";
+import { RateLimitService } from "@/src/features/public-api/server/RateLimitService";
+
+type GatewayRateLimitDimension = "requests" | "bytes";
+
+const MiB = 1024 * 1024;
+const GiB = 1024 * MiB;
+
+export class GatewayRateLimitError extends GatewayControlPlaneError {
+ constructor(
+ message: string,
+ readonly limit: GatewayRateLimitDimension,
+ readonly retryAfterSeconds: number,
+ readonly organizationId: string,
+ ) {
+ super(message, 429, "rate_limit_exceeded");
+ }
+}
+
+/**
+ * Counts one request and its body bytes against the organization's per-minute
+ * AI Gateway limits. Fails open like every other RateLimitService caller:
+ * self-hosted, disabled rate limits, or an unavailable Redis never reject.
+ */
+export async function enforceGatewayRateLimits({
+ context,
+ requestBytes,
+}: {
+ context: GatewayApiKeyAuthContext;
+ requestBytes: number;
+}) {
+ const scope = {
+ orgId: context.organizationId,
+ projectId: context.ingestionProjectId,
+ accessLevel: "organization" as const,
+ plan: context.plan,
+ rateLimitOverrides: context.rateLimitOverrides,
+ apiKeyId: context.apiKeyId,
+ publicKey: context.apiKeyId,
+ isIngestionSuspended: false,
+ accessAuditLogEnabled: null,
+ };
+ const service = RateLimitService.getInstance();
+ const [requests, bytes] = await Promise.all([
+ service.rateLimitRequest(scope, "ai-gateway-requests"),
+ service.rateLimitRequest(scope, "ai-gateway-request-bytes", requestBytes),
+ ]);
+
+ const exceeded = (
+ [
+ { limit: "requests", helper: requests },
+ { limit: "bytes", helper: bytes },
+ ] as const
+ )
+ .flatMap(({ limit, helper }) =>
+ helper.isRateLimited() && helper.res ? [{ limit, res: helper.res }] : [],
+ )
+ // Retrying before the later window resets would hit the other limit.
+ .toSorted((a, b) => b.res.msBeforeNext - a.res.msBeforeNext)[0];
+ if (!exceeded) return;
+
+ const { limit, res } = exceeded;
+ const retryAfterSeconds = Math.max(1, Math.ceil(res.msBeforeNext / 1000));
+ logger.warn("AI Gateway rate limit exceeded", {
+ orgId: context.organizationId,
+ projectId: context.ingestionProjectId,
+ apiKeyId: context.apiKeyId,
+ plan: context.plan,
+ limit,
+ current: res.consumedPoints,
+ limitValue: res.points,
+ requestBytes,
+ retryAfterSeconds,
+ });
+ getCurrentSpan()?.setAttributes({
+ "langfuse.ai_gateway.rate_limited": true,
+ "langfuse.ai_gateway.rate_limit.limit": limit,
+ });
+ const override = context.rateLimitOverrides.find(
+ (override) => override.resource === res.resource,
+ );
+ throw new GatewayRateLimitError(
+ gatewayRateLimitMessage({
+ limit,
+ res,
+ plan: context.plan,
+ retryAfterSeconds,
+ hasOverride: Boolean(override),
+ windowSeconds: override?.durationInSec ?? 60,
+ }),
+ limit,
+ retryAfterSeconds,
+ context.organizationId,
+ );
+}
+
+function gatewayRateLimitMessage(params: {
+ limit: GatewayRateLimitDimension;
+ res: RateLimitResult;
+ plan: Plan;
+ retryAfterSeconds: number;
+ hasOverride: boolean;
+ windowSeconds: number;
+}) {
+ const window = formatWindow(params.windowSeconds);
+ const allowance =
+ params.limit === "requests"
+ ? `${params.res.points.toLocaleString("en-US")} requests per ${window}`
+ : `${formatBytes(params.res.points)} of request bodies per ${window}`;
+ const subject = params.hasOverride
+ ? "your organization's custom limit allows"
+ : `your organization's ${planLabels[params.plan]} plan allows`;
+ const remedy =
+ params.plan === "cloud:enterprise" || params.hasOverride
+ ? "contact Langfuse support to raise the limit"
+ : "upgrade your plan";
+ return `Langfuse AI Gateway rate limit exceeded: ${subject} ${allowance}. Retry after ${params.retryAfterSeconds}s or ${remedy}.`;
+}
+
+function formatBytes(bytes: number) {
+ return bytes >= GiB && bytes % GiB === 0
+ ? `${bytes / GiB} GiB`
+ : `${Math.round((bytes / MiB) * 10) / 10} MiB`;
+}
+
+const windowUnits = [
+ { seconds: 86_400, name: "day" },
+ { seconds: 3_600, name: "hour" },
+ { seconds: 60, name: "minute" },
+] as const;
+
+function formatWindow(seconds: number) {
+ for (const unit of windowUnits) {
+ if (seconds % unit.seconds !== 0) continue;
+ const count = seconds / unit.seconds;
+ return count === 1 ? unit.name : `${count} ${unit.name}s`;
+ }
+ return seconds === 1 ? "second" : `${seconds} seconds`;
+}
diff --git a/web/src/features/ai-gateway/server/resolve/gatewayResolveApiHandler.ts b/web/src/features/ai-gateway/server/resolve/gatewayResolveApiHandler.ts
index 1ec9659b27..dcabdc3081 100644
--- a/web/src/features/ai-gateway/server/resolve/gatewayResolveApiHandler.ts
+++ b/web/src/features/ai-gateway/server/resolve/gatewayResolveApiHandler.ts
@@ -7,6 +7,10 @@ import {
type GatewayApiFormat,
GatewayResolveResponseSchema,
} from "@/src/features/ai-gateway/server/provider";
+import {
+ enforceGatewayRateLimits,
+ GatewayRateLimitError,
+} from "./gatewayRateLimit";
import { GatewayResolveError, GatewayResolveService } from "./resolveService";
export async function gatewayResolveApiHandler({
@@ -14,23 +18,35 @@ export async function gatewayResolveApiHandler({
fastHashedSecretKey,
apiFormat,
model,
+ requestBytes,
}: {
req: NextApiRequest;
res: NextApiResponse;
fastHashedSecretKey: string;
apiFormat: GatewayApiFormat;
model: string;
+ requestBytes: number;
}) {
try {
const context = await new GatewayApiKeyAuthenticator(
prisma,
).authenticateRequest({ fastHashedSecretKey, apiFormat, model });
+ await enforceGatewayRateLimits({ context, requestBytes });
const result = await new GatewayResolveService().resolve({
context,
apiFormat,
});
return res.status(200).json(GatewayResolveResponseSchema.parse(result));
} catch (error) {
+ if (error instanceof GatewayRateLimitError) {
+ res.setHeader("Retry-After", String(error.retryAfterSeconds));
+ res.setHeader("langfuse-gateway-error-code", "rate_limit_exceeded");
+ return res.status(error.status).json({
+ error: error.message,
+ limit: error.limit,
+ organizationId: error.organizationId,
+ });
+ }
if (error instanceof GatewayResolveError) {
// The data plane retries a 503, so tell it when to come back rather than
// letting it hammer a database that is already struggling.
diff --git a/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.ts b/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.ts
index 9e01069046..0aec5c7473 100644
--- a/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.ts
+++ b/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.ts
@@ -3,6 +3,7 @@ import {
recordIncrement,
redis as defaultRedis,
} from "@langfuse/shared/src/server";
+import { CloudConfigRateLimit, type Plan, plans } from "@langfuse/shared";
import type { Cluster, Redis } from "ioredis";
import { z } from "zod/v4";
@@ -34,6 +35,8 @@ export const CachedResolveContext = z.object({
keyMetadata: GatewayMetadataSchema,
ingestionProjectId: z.string().nullable(),
ingestionMode: z.enum(["NONE", "USAGE", "FULL"]).nullable(),
+ plan: z.custom<Plan>((value) => plans.includes(value as Plan)),
+ rateLimitOverrides: CloudConfigRateLimit,
connections: z.array(
z.object({
id: z.string(),
diff --git a/web/src/features/ai-gateway/server/resolve/gatewayResolveRepository.ts b/web/src/features/ai-gateway/server/resolve/gatewayResolveRepository.ts
index 45d5f49dce..294064f9b4 100644
--- a/web/src/features/ai-gateway/server/resolve/gatewayResolveRepository.ts
+++ b/web/src/features/ai-gateway/server/resolve/gatewayResolveRepository.ts
@@ -32,6 +32,7 @@ export class GatewayResolveRepository {
gatewayAssociation: { select: { metadata: true } },
organization: {
select: {
+ cloudConfig: true,
gatewayConfig: {
select: {
defaultIngestionProjectId: true,
diff --git a/web/src/features/public-api/server/RateLimitService.ts b/web/src/features/public-api/server/RateLimitService.ts
index 51fd49c32b..595a176fc8 100644
--- a/web/src/features/public-api/server/RateLimitService.ts
+++ b/web/src/features/public-api/server/RateLimitService.ts
@@ -80,9 +80,11 @@ export class RateLimitService {
RateLimitService.instance = null;
}
+ /** `points` is what this request costs, e.g. its byte size for byte-based resources. */
async rateLimitRequest(
scope: ApiAccessScopeWithOptionalApiKeyId,
resource: z.infer<typeof RateLimitResource>,
+ points = 1,
) {
// if cloud config is not present, we don't apply rate limits and just return
if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {
@@ -98,12 +100,15 @@ export class RateLimitService {
return new RateLimitHelper(undefined);
}
- return new RateLimitHelper(await this.checkRateLimit(scope, resource));
+ return new RateLimitHelper(
+ await this.checkRateLimit(scope, resource, points),
+ );
}
async checkRateLimit(
scope: ApiAccessScopeWithOptionalApiKeyId,
resource: z.infer<typeof RateLimitResource>,
+ points = 1,
) {
const effectiveConfig = getRateLimitConfig(scope, resource);
@@ -138,7 +143,7 @@ export class RateLimitService {
let res: RateLimitResult | undefined = undefined;
try {
// orgId used as key for different resources
- const libRes = await rateLimiter.consume(scope.orgId);
+ const libRes = await rateLimiter.consume(scope.orgId, points);
res = {
resource,
scope,
@@ -289,6 +294,9 @@ const getRateLimitConfig = (
return customConfig || planBasedConfig;
};
+const MiB = 1024 * 1024;
+const GiB = 1024 * MiB;
+
const getPlanBasedRateLimitConfig = (
plan: Plan,
resource: z.infer<typeof RateLimitResource>,
@@ -395,6 +403,10 @@ const getPlanBasedRateLimitConfig = (
points: 20,
durationInSec: 86400, // 20 submissions per day
};
+ case "ai-gateway-requests":
+ return { resource, points: 100, durationInSec: 60 };
+ case "ai-gateway-request-bytes":
+ return { resource, points: 64 * MiB, durationInSec: 60 };
default:
const exhaustiveCheck: never = resource;
throw new Error(`Unhandled resource case: ${exhaustiveCheck}`);
@@ -499,6 +511,10 @@ const getPlanBasedRateLimitConfig = (
points: 20,
durationInSec: 86400, // 20 submissions per day
};
+ case "ai-gateway-requests":
+ return { resource, points: 500, durationInSec: 60 };
+ case "ai-gateway-request-bytes":
+ return { resource, points: 256 * MiB, durationInSec: 60 };
default:
const exhaustiveCheck: never = resource;
throw new Error(`Unhandled resource case: ${exhaustiveCheck}`);
@@ -597,6 +613,10 @@ const getPlanBasedRateLimitConfig = (
points: 20,
durationInSec: 86400, // 20 submissions per day
};
+ case "ai-gateway-requests":
+ return { resource, points: 3_000, durationInSec: 60 };
+ case "ai-gateway-request-bytes":
+ return { resource, points: 2 * GiB, durationInSec: 60 };
default:
const exhaustiveCheck: never = resource;
throw new Error(`Unhandled resource case: ${exhaustiveCheck}`);