🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(ai-gateway): per-organization request and byte rate limits at /resolve (#19585)
Hassieb Pakzad0f5b1840329e
diff --git a/ai-gateway/src/correlation.rs b/ai-gateway/src/correlation.rsindex 45c442a562..c8de9de077 100644--- a/ai-gateway/src/correlation.rs+++ b/ai-gateway/src/correlation.rs@@ -11,7 +11,9 @@ use axum::{use crate::{observability::RequestObservation,- resolution::{OrganizationId, ProjectId, ProviderConnection, ResolvedRequestContext},+ resolution::{+ OrganizationId, ProjectId, ProviderConnection, RateLimit, ResolvedRequestContext,+ },};pub(crate) const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("langfuse-request-id");@@ -145,6 +147,14 @@ impl RequestCorrelation {});}+ /// Web names the organization it rate limited, since no context was resolved.+ pub(crate) fn record_rate_limit(&self, limit: &RateLimit) {+ if let Some(organization_id) = limit.organization_id() {+ self.server_span+ .record("langfuse.organization.id", organization_id);+ }+ }+pub(crate) fn record_upstream(&mut self, target: UpstreamTarget) {let span = &self.server_span;span.record("gateway.provider", target.provider);diff --git a/ai-gateway/src/http.rs b/ai-gateway/src/http.rsindex 58cb9e58b4..e7a0b850f7 100644--- a/ai-gateway/src/http.rs+++ b/ai-gateway/src/http.rs@@ -16,10 +16,13 @@ use tracing::Instrument;use crate::{correlation::{RequestCorrelation, assign_request_id},- inference::{InferenceService, MemoryReservation, provider_origin, resolution_origin},+ inference::{+ InferenceService, MemoryReservation, provider_origin, rate_limited_reason,+ resolution_origin,+ },observability::{ErrorOrigin, Wait},providers::{ProviderError, RequestBody, Route},- resolution::{ApiFormat, ResolutionError},+ resolution::{ApiFormat, RateLimit, ResolutionError},server::GatewayLifecycleState,};@@ -220,13 +223,17 @@ async fn relay(.observation().record_wait(Wait::RequestBody, started.elapsed());let (bytes, reservation) = read?;- correlation.observation().record_request_bytes(bytes.len());+ let request_bytes = bytes.len();+ correlation+ .observation()+ .record_request_bytes(request_bytes);let request_body = request_body_and_model(bytes)?;let (permit, context) = inference.resolve_and_admit(gateway_key,route.api_format(),&request_body.model,+ request_bytes,correlation,).await@@ -427,12 +434,12 @@ enum InferenceHttpError {#[derive(Serialize)]struct OpenAiErrorResponse<'a> {- error: OpenAiErrorDetail,+ error: OpenAiErrorDetail<'a>,request_id: &'a str,}#[derive(Serialize)]-struct OpenAiErrorDetail {- message: &'static str,+struct OpenAiErrorDetail<'a> {+ message: &'a str,#[serde(rename = "type")]kind: &'static str,param: Option<&'static str>,@@ -443,14 +450,14 @@ struct OpenAiErrorDetail {struct AnthropicErrorResponse<'a> {#[serde(rename = "type")]kind: &'static str,- error: AnthropicErrorDetail,+ error: AnthropicErrorDetail<'a>,request_id: &'a str,}#[derive(Serialize)]-struct AnthropicErrorDetail {+struct AnthropicErrorDetail<'a> {#[serde(rename = "type")]kind: &'static str,- message: &'static str,+ message: &'a str,}impl InferenceHttpError {@@ -472,6 +479,7 @@ impl InferenceHttpError {R::Forbidden => "forbidden",R::ModelNotAllowed => "model_not_allowed",R::NoRoute => "no_route",+ R::RateLimited(limit) => rate_limited_reason(limit.kind()),R::Busy => "capacity",R::Timeout => "timeout",R::Unavailable => "unavailable",@@ -519,6 +527,13 @@ impl InferenceHttpError {"permission_error","model_not_allowed",),+ // OpenAI's own rate-limit type and code, so SDKs treat it as a 429 to back off.+ Self::Resolution(R::RateLimited(_)) => (+ StatusCode::TOO_MANY_REQUESTS,+ "Langfuse AI Gateway rate limit exceeded",+ "requests",+ "rate_limit_exceeded",+ ),Self::Resolution(R::NoRoute) => (StatusCode::NOT_FOUND,"No eligible provider connection",@@ -588,7 +603,7 @@ impl InferenceHttpError {Self::RequestTimeout => (ErrorOrigin::Client, "body_timeout"),Self::UnsupportedEncoding => (ErrorOrigin::Client, "unsupported_content_encoding"),Self::InvalidBody => (ErrorOrigin::Client, "invalid_body"),- Self::Resolution(error) => resolution_origin(*error),+ Self::Resolution(error) => resolution_origin(error),Self::Provider(error) => provider_origin(*error),}}@@ -601,8 +616,23 @@ impl InferenceHttpError {let (phase, reason) = self.category();let (origin, _) = self.origin();let (status, message, kind, code) = self.classify();+ let rate_limit = match &self {+ Self::Resolution(ResolutionError::RateLimited(limit)) => Some(limit.as_ref()),+ _ => None,+ };// Caller mistakes are routine; only failures that need an operator warn.- if status.is_server_error() {+ if let Some(limit) = rate_limit {+ tracing::warn!(+ phase,+ reason,+ error.origin = origin.as_str(),+ status = status.as_u16(),+ rate_limit = limit.kind().as_str(),+ retry_after_secs = limit.retry_after_secs(),+ organization_id = limit.organization_id(),+ "gateway request rate limited"+ );+ } else if status.is_server_error() {tracing::warn!(phase,reason,@@ -620,7 +650,8 @@ impl InferenceHttpError {);}let request_id = correlation.id();- match api_format {+ let message = rate_limit.map_or(message, RateLimit::message);+ let mut response = match api_format {ApiFormat::OpenAiResponses | ApiFormat::OpenAiChatCompletions => (status,Json(OpenAiErrorResponse {@@ -646,7 +677,13 @@ impl InferenceHttpError {}),).into_response(),+ };+ if let Some(limit) = rate_limit {+ response+ .headers_mut()+ .insert(header::RETRY_AFTER, limit.retry_after_secs().into());}+ response}}@@ -655,6 +692,7 @@ fn anthropic_error_type(status: StatusCode) -> &'static str {StatusCode::UNAUTHORIZED => "authentication_error",StatusCode::FORBIDDEN => "permission_error",StatusCode::NOT_FOUND => "not_found_error",+ StatusCode::TOO_MANY_REQUESTS => "rate_limit_error",StatusCode::PAYLOAD_TOO_LARGE => "request_too_large",StatusCode::SERVICE_UNAVAILABLE => "overloaded_error",status if status.is_client_error() => "invalid_request_error",diff --git a/ai-gateway/src/http/tests.rs b/ai-gateway/src/http/tests.rsindex 9518b87eaa..c4efcf069c 100644--- a/ai-gateway/src/http/tests.rs+++ b/ai-gateway/src/http/tests.rs@@ -1277,7 +1277,7 @@ async fn anthropic_messages_accept_either_credential_header_and_relay_native_sselet key = anthropic_gateway_key(&request).unwrap();assert_eq!(to_bytes(request.into_body(), 1024).await.unwrap(),- r#"{"apiFormat":"anthropic.messages","model":"Production key:claude-sonnet-4-5"}"#+ r#"{"apiFormat":"anthropic.messages","model":"Production key:claude-sonnet-4-5","requestBytes":72}"#);if key == "gateway-anthropic" {resolution_response_for(ApiFormat::AnthropicMessages, "provider-anthropic")@@ -1469,6 +1469,65 @@ async fn model_policy_rejections_use_provider_native_permission_errors() {assert_eq!(provider.calls(), 0);}+const RATE_LIMIT_MESSAGE: &str = "Langfuse AI Gateway rate limit exceeded: your organization's Hobby plan allows 100 requests per minute. Retry after 23s or upgrade your plan.";++#[tokio::test]+async fn organization_rate_limits_become_provider_native_429s_with_retry_after() {+ let web = FakeServer::start(|request| async move {+ let body: serde_json::Value =+ serde_json::from_slice(&to_bytes(request.into_body(), 1024).await.unwrap()).unwrap();+ assert_eq!(body["requestBytes"], 34);+ Response::builder()+ .status(StatusCode::TOO_MANY_REQUESTS)+ .header("retry-after", "23")+ .header("langfuse-gateway-error-code", "rate_limit_exceeded")+ .body(Body::from(+ serde_json::json!({+ "error": RATE_LIMIT_MESSAGE,+ "limit": "requests",+ "organizationId": "org-1",+ })+ .to_string(),+ ))+ .unwrap()+ })+ .await;+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;+ let gateway = Gateway::start(Some(inference(&web, &provider))).await;++ let openai = gateway+ .post()+ .bearer_auth("gateway-key")+ .body(r#"{"model":"Production key:gpt-4.1"}"#)+ .send()+ .await+ .unwrap();+ assert_eq!(openai.status(), StatusCode::TOO_MANY_REQUESTS);+ assert_eq!(openai.headers()[header::RETRY_AFTER], "23");+ let openai_body: serde_json::Value =+ serde_json::from_str(&openai.text().await.unwrap()).unwrap();+ assert_eq!(openai_body["error"]["message"], RATE_LIMIT_MESSAGE);+ assert_eq!(openai_body["error"]["type"], "requests");+ assert_eq!(openai_body["error"]["code"], "rate_limit_exceeded");++ let anthropic = reqwest::Client::new()+ .post(format!("{}/anthropic/v1/messages", gateway.url))+ .header("x-api-key", "gateway-key")+ .body(r#"{"model":"Production key:gpt-4.1"}"#)+ .send()+ .await+ .unwrap();+ assert_eq!(anthropic.status(), StatusCode::TOO_MANY_REQUESTS);+ assert_eq!(anthropic.headers()[header::RETRY_AFTER], "23");+ let anthropic_body: serde_json::Value =+ serde_json::from_str(&anthropic.text().await.unwrap()).unwrap();+ assert_eq!(anthropic_body["type"], "error");+ assert_eq!(anthropic_body["error"]["type"], "rate_limit_error");+ assert_eq!(anthropic_body["error"]["message"], RATE_LIMIT_MESSAGE);+ assert_eq!(web.calls(), 2);+ assert_eq!(provider.calls(), 0);+}+#[tokio::test]async fn anthropic_capacity_rejections_use_the_native_envelope() {let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;diff --git a/ai-gateway/src/inference.rs b/ai-gateway/src/inference.rsindex dcf088447d..4668c166ba 100644--- a/ai-gateway/src/inference.rs+++ b/ai-gateway/src/inference.rs@@ -14,7 +14,8 @@ use crate::{observability::ErrorOrigin,providers::{ProviderError, ProviderTransport, RequestBody, RequestPermit, Route},resolution::{- ApiFormat, ControlPlaneClient, ControlPlaneConfig, ResolutionError, ResolvedRequestContext,+ ApiFormat, ControlPlaneClient, ControlPlaneConfig, RateLimitKind, ResolutionError,+ ResolvedRequestContext,},};@@ -27,7 +28,7 @@ pub struct InferenceService {}/// `error.origin` and `error.reason` for a failed resolution or catalog fetch.-pub(crate) fn resolution_origin(error: ResolutionError) -> (ErrorOrigin, &'static str) {+pub(crate) fn resolution_origin(error: &ResolutionError) -> (ErrorOrigin, &'static str) {use ResolutionError as R;match error {R::Configuration => (ErrorOrigin::Gateway, "configuration"),@@ -37,6 +38,7 @@ pub(crate) fn resolution_origin(error: ResolutionError) -> (ErrorOrigin, &'statiR::Forbidden => (ErrorOrigin::Client, "forbidden"),R::ModelNotAllowed => (ErrorOrigin::Client, "model_not_allowed"),R::NoRoute => (ErrorOrigin::Client, "no_route"),+ R::RateLimited(limit) => (ErrorOrigin::Client, rate_limited_reason(limit.kind())),R::Unavailable => (ErrorOrigin::ControlPlane, "unavailable"),R::Timeout => (ErrorOrigin::ControlPlane, "timeout"),R::Transport => (ErrorOrigin::ControlPlane, "transport"),@@ -45,6 +47,14 @@ pub(crate) fn resolution_origin(error: ResolutionError) -> (ErrorOrigin, &'stati}}+/// Distinct from `provider_rate_limited`, which is the provider's own 429.+pub(crate) fn rate_limited_reason(kind: RateLimitKind) -> &'static str {+ match kind {+ RateLimitKind::Requests => "rate_limited_requests",+ RateLimitKind::Bytes => "rate_limited_bytes",+ }+}+/// `error.origin` and `error.reason` for a provider request that produced no response.pub(crate) fn provider_origin(error: ProviderError) -> (ErrorOrigin, &'static str) {match error {@@ -205,12 +215,14 @@ impl InferenceService {}/// Authenticate, then start the execution. The API format comes from the public route,- /// so Web selects a compatible connection. Memory is already reserved by the caller.+ /// so Web selects a compatible connection, and `request_bytes` counts toward the+ /// organization's byte rate limit. Memory is already reserved by the caller.pub(crate) async fn resolve_and_admit(&self,gateway_key: &str,api_format: ApiFormat,model: &str,+ request_bytes: usize,correlation: &mut RequestCorrelation,) -> Result<(RequestPermit, ResolvedRequestContext), ResolutionError> {let span = tracing::info_span!(@@ -225,21 +237,30 @@ impl InferenceService {let resolved = {let _active = crate::observability::Active::new("resolution");self.control_plane- .resolve(gateway_key, api_format, model, correlation.id())+ .resolve(+ gateway_key,+ api_format,+ model,+ request_bytes,+ correlation.id(),+ ).await};let span = tracing::Span::current();span.record("gateway.outcome",- if resolved.is_ok() {- "admitted"- } else {- "resolution_failed"+ match &resolved {+ Ok(_) => "admitted",+ Err(ResolutionError::RateLimited(_)) => "rate_limited",+ Err(_) => "resolution_failed",},);- // Caller-caused failures (bad key, forbidden model) stay unset so- // Datadog error rates track gateway and control-plane faults only.- if let Err(error) = resolved+ // Caller-caused failures (bad key, forbidden model, rate limits) stay unset+ // so Datadog error rates track gateway and control-plane faults only.+ if let Err(ResolutionError::RateLimited(limit)) = &resolved {+ correlation.record_rate_limit(limit);+ }+ if let Err(error) = &resolved&& let (origin, reason) = resolution_origin(error)&& origin.is_ours(){diff --git a/ai-gateway/src/observability/http/tests.rs b/ai-gateway/src/observability/http/tests.rsindex 2a511a2e2d..eb4cb0b265 100644--- a/ai-gateway/src/observability/http/tests.rs+++ b/ai-gateway/src/observability/http/tests.rs@@ -304,6 +304,60 @@ async fn failed_responses_name_their_origin_on_the_server_span() {}}+/// Langfuse's own rate limit is a caller error, kept apart from the provider's 429.+#[tokio::test]+async fn organization_rate_limits_are_tagged_on_the_server_and_resolution_spans() {+ let web = FakeServer::start(|_| async {+ Response::builder()+ .status(429)+ .header("retry-after", "5")+ .header("langfuse-gateway-error-code", "rate_limit_exceeded")+ .body(Body::from(+ r#"{"error":"Langfuse AI Gateway rate limit exceeded","limit":"bytes","organizationId":"org-1"}"#,+ ))+ .unwrap()+ })+ .await;+ let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;+ let service = InferenceService::for_test(+ web.control_plane(),+ ProviderTransport::for_test(provider.url.clone(), ProviderLimits::default()),+ );+ let recording = Recording::start();+ let lifecycle = GatewayLifecycleState::default();+ let app = instrument(crate::http::router(Some(service), lifecycle.clone()));+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let mut serving = Box::pin(crate::server::serve(+ listener,+ app.clone(),+ lifecycle,+ std::future::pending(),+ Duration::from_secs(1),+ ));+ assert!(futures_util::poll!(serving.as_mut()).is_pending());+ let response = app.oneshot(generation_request()).await.unwrap();+ assert_eq!(response.status(), 429);+ axum::body::to_bytes(response.into_body(), 4096)+ .await+ .unwrap();++ let spans = recording.exporter.get_finished_spans().unwrap();+ let server = spans+ .iter()+ .find(|span| span.span_kind == SpanKind::Server)+ .unwrap();+ assert_attribute(server, "error.origin", "client");+ assert_attribute(server, "error.type", "rate_limited_bytes");+ assert_attribute(server, "langfuse.organization.id", "org-1");+ assert!(!matches!(+ server.status,+ opentelemetry::trace::Status::Error { .. }+ ));+ let resolution = spans.iter().find(|span| span.name == "resolution").unwrap();+ assert_attribute(resolution, "gateway.outcome", "rate_limited");+ assert_eq!(provider.calls(), 0);+}+/// Token counting captures no generation, yet a body that breaks after the/// headers must still fail the server span.#[tokio::test]diff --git a/ai-gateway/src/providers/tests.rs b/ai-gateway/src/providers/tests.rsindex 67de442f91..51a8a6b073 100644--- a/ai-gateway/src/providers/tests.rs+++ b/ai-gateway/src/providers/tests.rs@@ -112,6 +112,7 @@ async fn missing_provider_model_fails_resolution() {"gateway-secret",ApiFormat::OpenAiResponses,"Production key:gpt-4o",+ 0,"test-request-id",).await;diff --git a/ai-gateway/src/resolution/mod.rs b/ai-gateway/src/resolution/mod.rsindex aea67710ed..4f51f45a83 100644--- a/ai-gateway/src/resolution/mod.rs+++ b/ai-gateway/src/resolution/mod.rs@@ -8,7 +8,7 @@ pub use contracts::{};use reqwest::{Client, Method, Url,- header::{AUTHORIZATION, CONTENT_TYPE, HeaderValue},+ header::{AUTHORIZATION, CONTENT_TYPE, HeaderValue, RETRY_AFTER},};use reqwest_middleware::{ClientWithMiddleware, RequestBuilder};use secrecy::{ExposeSecret, SecretString};@@ -21,6 +21,12 @@ const RESOLVE_PATH: &str = "/api/internal/ai-gateway/v1/resolve";const MODELS_PATH: &str = "/api/internal/ai-gateway/v1/models";const REQUEST_ID_HEADER: &str = "langfuse-gateway-request-id";const ERROR_CODE_HEADER: &str = "langfuse-gateway-error-code";+const RATE_LIMIT_ERROR_CODE: &str = "rate_limit_exceeded";+/// Shown when Web's rate-limit body is missing or unreadable.+const RATE_LIMIT_FALLBACK_MESSAGE: &str =+ "Langfuse AI Gateway rate limit exceeded for your organization's plan.";+/// Web's messages are a sentence; a longer one is not relayed to the caller.+const MAX_RATE_LIMIT_MESSAGE_BYTES: usize = 1024;// Web sits behind load balancers that drop idle connections after 60s by default// (AWS ALB). Evicting pooled connections first keeps the resolver from sending on a// socket the balancer is closing, which would fail without a retry.@@ -156,24 +162,34 @@ impl ControlPlaneClient {}/// Resolve a gateway credential and API format into a validated execution contract.- /// `request_id` is the gateway request ID, sent so Web logs carry it.+ /// `request_bytes` is the client body size, counted against the organization's byte+ /// rate limit. `request_id` is the gateway request ID, sent so Web logs carry it.////// # Errors/// Returns a sanitized [`ResolutionError`] for invalid credentials, authentication or- /// authorization failures, missing routes, unavailable Web or system time, request- /// construction or transport failures, timeouts, or oversized or invalid responses.+ /// authorization failures, organization rate limits, missing routes, unavailable Web+ /// or system time, request construction or transport failures, timeouts, or oversized+ /// or invalid responses.pub async fn resolve(&self,gateway_key: &str,api_format: ApiFormat,model: &str,+ request_bytes: usize,request_id: &str,) -> Result<ResolvedRequestContext, ResolutionError> {let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).map_err(|_| ResolutionError::Unavailable)?;- self.resolve_at(gateway_key, api_format, model, timestamp, request_id)- .await+ self.resolve_at(+ gateway_key,+ api_format,+ model,+ request_bytes,+ timestamp,+ request_id,+ )+ .await}/// Fetch the caller-visible model catalog from trusted Web. `reserve` is called@@ -212,6 +228,7 @@ impl ControlPlaneClient {gateway_key: &str,api_format: ApiFormat,model: &str,+ request_bytes: usize,timestamp: Duration,request_id: &str,) -> Result<ResolvedRequestContext, ResolutionError> {@@ -219,6 +236,7 @@ impl ControlPlaneClient {let body = serde_json::to_vec(&serde_json::json!({"apiFormat": api_format,"model": model,+ "requestBytes": request_bytes,})).map_err(|_| ResolutionError::Configuration)?;let request = self@@ -288,8 +306,8 @@ impl ControlPlaneClient {}/// Like [`Self::execute`], but resends once, immediately, when no usable response- /// arrived. Resolve is read-only on Web, so a duplicate is harmless. Both attempts- /// share one deadline, so a timeout is never retried.+ /// arrived. A duplicate only writes rate-limit counters on Web, so at worst it counts+ /// one request twice. Both attempts share one deadline, so a timeout is never retried.async fn execute_retrying_once(&self,request: RequestBuilder,@@ -338,7 +356,7 @@ fn retry_reason(}async fn read_response(- mut response: reqwest::Response,+ response: reqwest::Response,limit: usize,reserve: &mut (dyn FnMut(usize) -> Result<(), ResolutionError> + Send),) -> Result<Vec<u8>, ResolutionError> {@@ -363,9 +381,26 @@ async fn read_response(}403 => return Err(ResolutionError::Forbidden),404 => return Err(ResolutionError::NoRoute),+ 429 if response+ .headers()+ .get(ERROR_CODE_HEADER)+ .is_some_and(|value| value == RATE_LIMIT_ERROR_CODE) =>+ {+ return Err(ResolutionError::RateLimited(Box::new(+ RateLimit::from_response(response, limit).await,+ )));+ }429 | 500..=599 => return Err(ResolutionError::Unavailable),_ => return Err(ResolutionError::InvalidResponse),}+ read_body(response, limit, reserve).await+}++async fn read_body(+ mut response: reqwest::Response,+ limit: usize,+ reserve: &mut (dyn FnMut(usize) -> Result<(), ResolutionError> + Send),+) -> Result<Vec<u8>, ResolutionError> {let declared = response.content_length().map(|size| usize::try_from(size).unwrap_or(usize::MAX));@@ -396,12 +431,103 @@ async fn read_response(Ok(bytes)}+/// Which of the organization's per-minute limits Web enforced.+#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize)]+#[serde(rename_all = "lowercase")]+pub enum RateLimitKind {+ Requests,+ Bytes,+}++impl RateLimitKind {+ #[must_use]+ pub fn as_str(self) -> &'static str {+ match self {+ Self::Requests => "requests",+ Self::Bytes => "bytes",+ }+ }+}++/// Web rejected the request because the organization exceeded a plan rate limit.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct RateLimit {+ kind: RateLimitKind,+ retry_after_secs: u64,+ message: Box<str>,+ organization_id: Option<Box<str>>,+}++impl RateLimit {+ /// Web writes the caller-facing message, so the gateway relays it unchanged.+ async fn from_response(response: reqwest::Response, limit: usize) -> Self {+ #[derive(serde::Deserialize)]+ #[serde(rename_all = "camelCase")]+ struct Body {+ error: String,+ limit: RateLimitKind,+ organization_id: Option<String>,+ }+ let retry_after_secs = response+ .headers()+ .get(RETRY_AFTER)+ .and_then(|value| value.to_str().ok()?.trim().parse::<u64>().ok())+ .unwrap_or(1)+ .clamp(1, 60 * 60);+ let body = read_body(response, limit, &mut |_| Ok(()))+ .await+ .ok()+ .and_then(|bytes| serde_json::from_slice::<Body>(&bytes).ok())+ .filter(|body| {+ !body.error.is_empty() && body.error.len() <= MAX_RATE_LIMIT_MESSAGE_BYTES+ });+ let Some(body) = body else {+ return Self {+ kind: RateLimitKind::Requests,+ retry_after_secs,+ message: RATE_LIMIT_FALLBACK_MESSAGE.into(),+ organization_id: None,+ };+ };+ Self {+ kind: body.limit,+ retry_after_secs,+ message: body.error.into_boxed_str(),+ organization_id: body+ .organization_id+ .filter(|id| valid_token(id) && id.len() <= 256)+ .map(String::into_boxed_str),+ }+ }++ #[must_use]+ pub fn kind(&self) -> RateLimitKind {+ self.kind+ }++ #[must_use]+ pub fn retry_after_secs(&self) -> u64 {+ self.retry_after_secs+ }++ #[must_use]+ pub fn message(&self) -> &str {+ &self.message+ }++ #[must_use]+ pub fn organization_id(&self) -> Option<&str> {+ self.organization_id.as_deref()+ }+}+fn valid_token(value: &str) -> bool {!value.is_empty() && value.bytes().all(|byte| byte.is_ascii_graphic())}-/// Sanitized failure categories. Upstream bodies and transport errors are never retained.-#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]+/// Sanitized failure categories. Upstream bodies and transport errors are never retained,+/// except the caller-facing message of a Web rate-limit rejection.+#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]pub enum ResolutionError {#[error("invalid resolver configuration")]Configuration,@@ -417,6 +543,8 @@ pub enum ResolutionError {ModelNotAllowed,#[error("no eligible connection")]NoRoute,+ #[error("organization rate limit exceeded")]+ RateLimited(Box<RateLimit>),/// The gateway's memory budget cannot cover another request.#[error("gateway at capacity")]Busy,diff --git a/ai-gateway/src/resolution/tests.rs b/ai-gateway/src/resolution/tests.rsindex c1d55c6d26..a6552d7872 100644--- a/ai-gateway/src/resolution/tests.rs+++ b/ai-gateway/src/resolution/tests.rs@@ -126,7 +126,7 @@ async fn signs_the_exact_key_and_sends_api_format_and_model() {assert_eq!(request.headers()["langfuse-gateway-authorization"],"HMAC timestamp=1800000000,signature=e104b6baa50d9b766b036e27876683aa1f4bff7f31bb6af66dce6b636a53b2bd");assert_eq!(to_bytes(request.into_body(), 1024).await.unwrap(),- r#"{"apiFormat":"openai.responses","model":"primary:gpt-4.1"}"#);+ r#"{"apiFormat":"openai.responses","model":"primary:gpt-4.1","requestBytes":0}"#);response(success("project-1", "private-provider-token"))}).await;@@ -136,6 +136,7 @@ async fn signs_the_exact_key_and_sends_api_format_and_model() {GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)@@ -240,6 +241,7 @@ async fn model_catalogs_have_a_larger_response_limit_than_resolution() {GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)@@ -274,6 +276,7 @@ async fn denied_and_failed_resolutions_are_classified_without_retries_or_body_leGATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)@@ -297,6 +300,7 @@ async fn resolve_against(web: &FakeWeb) -> Result<ResolvedRequestContext, ResoluGATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)@@ -371,6 +375,7 @@ async fn retries_once_when_web_closes_the_connection_without_answering() {GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)@@ -394,6 +399,7 @@ async fn timeouts_are_not_retried() {GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)@@ -420,6 +426,7 @@ async fn model_policy_rejections_are_distinct_from_other_forbidden_responses() {GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)@@ -430,6 +437,83 @@ async fn model_policy_rejections_are_distinct_from_other_forbidden_responses() {assert!(!format!("{error:?} {error}").contains("private-upstream-error"));}+#[tokio::test]+async fn organization_rate_limits_carry_web_message_and_retry_after() {+ let web = FakeWeb::start(|_| async {+ Response::builder()+ .status(StatusCode::TOO_MANY_REQUESTS)+ .header("retry-after", "17")+ .header("langfuse-gateway-error-code", "rate_limit_exceeded")+ .body(Body::from(+ r#"{"error":"Langfuse AI Gateway rate limit exceeded: bytes","limit":"bytes","organizationId":"org-1"}"#,+ ))+ .unwrap()+ })+ .await;++ let error = web+ .control_plane()+ .resolve_at(+ GATEWAY_KEY,+ ApiFormat::OpenAiResponses,+ "primary:gpt-4.1",+ 4096,+ NOW_TIME,+ REQUEST_ID,+ )+ .await+ .unwrap_err();++ let ResolutionError::RateLimited(limit) = error else {+ panic!("expected a rate limit, got {error:?}");+ };+ assert_eq!(limit.kind(), RateLimitKind::Bytes);+ assert_eq!(limit.retry_after_secs(), 17);+ assert_eq!(+ limit.message(),+ "Langfuse AI Gateway rate limit exceeded: bytes"+ );+ assert_eq!(limit.organization_id(), Some("org-1"));+ assert_eq!(web.calls(), 1);+}++#[tokio::test]+async fn unreadable_rate_limit_bodies_fall_back_to_a_langfuse_message() {+ let web = FakeWeb::start(|_| async {+ Response::builder()+ .status(StatusCode::TOO_MANY_REQUESTS)+ .header("langfuse-gateway-error-code", "rate_limit_exceeded")+ .body(Body::from("private-upstream-error"))+ .unwrap()+ })+ .await;++ let error = web+ .control_plane()+ .resolve_at(+ GATEWAY_KEY,+ ApiFormat::OpenAiResponses,+ "primary:gpt-4.1",+ 0,+ NOW_TIME,+ REQUEST_ID,+ )+ .await+ .unwrap_err();++ let ResolutionError::RateLimited(limit) = error else {+ panic!("expected a rate limit, got {error:?}");+ };+ assert_eq!(limit.kind(), RateLimitKind::Requests);+ assert_eq!(limit.retry_after_secs(), 1);+ assert!(+ limit+ .message()+ .starts_with("Langfuse AI Gateway rate limit exceeded")+ );+ assert!(!limit.message().contains("private-upstream-error"));+}+/// A rejected gateway signature means the service keys disagree, which is the/// gateway's fault, not the caller's.#[tokio::test]@@ -452,6 +536,7 @@ async fn rejected_gateway_signatures_are_configuration_errors() {GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)@@ -483,6 +568,7 @@ async fn never_follows_redirects_with_credentials() {GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID)@@ -515,6 +601,7 @@ async fn rejects_oversized_declared_and_chunked_responses() {GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)@@ -548,6 +635,7 @@ async fn deadline_covers_response_headers_and_body() {GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,),@@ -569,6 +657,7 @@ async fn malformed_credentials_never_reach_web() {key,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)@@ -593,6 +682,7 @@ async fn rejects_a_grant_that_expires_while_resolving_across_a_second_boundary()GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME + Duration::from_millis(900),REQUEST_ID,)@@ -620,6 +710,7 @@ async fn concurrent_resolutions_keep_credentials_and_contexts_isolated() {"gw_test_alice",ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID),@@ -627,6 +718,7 @@ async fn concurrent_resolutions_keep_credentials_and_contexts_isolated() {"gw_test_bob",ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID),@@ -671,7 +763,7 @@ async fn resolves_anthropic_messages_with_the_native_credential_scheme() {let web = FakeWeb::start(|request| async move {assert_eq!(to_bytes(request.into_body(), 1024).await.unwrap(),- r#"{"apiFormat":"anthropic.messages","model":"primary:claude-sonnet-4-5"}"#+ r#"{"apiFormat":"anthropic.messages","model":"primary:claude-sonnet-4-5","requestBytes":0}"#);response(anthropic_success("private-anthropic-key"))})@@ -683,6 +775,7 @@ async fn resolves_anthropic_messages_with_the_native_credential_scheme() {GATEWAY_KEY,ApiFormat::AnthropicMessages,"primary:claude-sonnet-4-5",+ 0,NOW_TIME,REQUEST_ID,)@@ -722,7 +815,7 @@ async fn chat_completions_resolve_only_an_openai_connection_for_that_format() {async move {assert_eq!(to_bytes(request.into_body(), 1024).await.unwrap(),- r#"{"apiFormat":"openai.chat-completions","model":"primary:gpt-4.1"}"#+ r#"{"apiFormat":"openai.chat-completions","model":"primary:gpt-4.1","requestBytes":0}"#);response(body)}@@ -734,6 +827,7 @@ async fn chat_completions_resolve_only_an_openai_connection_for_that_format() {GATEWAY_KEY,ApiFormat::OpenAiChatCompletions,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)@@ -900,6 +994,7 @@ async fn accepts_unknown_response_fields_outside_provider_auth() {GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)@@ -932,6 +1027,7 @@ async fn treats_missing_or_blank_key_snapshots_as_absent() {GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)@@ -986,6 +1082,7 @@ async fn rejects_ingestion_tokens_that_authorize_a_different_tenant() {GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID)@@ -1011,6 +1108,7 @@ async fn resolves_none_mode_without_an_ingestion_grant_or_project() {GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)@@ -1044,6 +1142,7 @@ async fn resolves_keys_that_are_not_bound_to_a_user() {GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)@@ -1089,6 +1188,7 @@ async fn assert_invalid_context_for(body: Value, api_format: ApiFormat) {GATEWAY_KEY,api_format,"primary:provider-model",+ 0,NOW_TIME,REQUEST_ID,)@@ -1109,6 +1209,7 @@ async fn rejects_malformed_json_without_exposing_response_contents() {GATEWAY_KEY,ApiFormat::OpenAiResponses,"primary:gpt-4.1",+ 0,NOW_TIME,REQUEST_ID,)diff --git a/ai-gateway/src/test_support.rs b/ai-gateway/src/test_support.rsindex 1ca1633aea..9be7f39b2f 100644--- a/ai-gateway/src/test_support.rs+++ b/ai-gateway/src/test_support.rs@@ -182,6 +182,7 @@ pub(crate) async fn resolved_request_context_for("gateway-secret",api_format,"Production key:provider-model",+ 0,"test-request-id",).awaitdiff --git a/packages/shared/src/interfaces/rate-limits.ts b/packages/shared/src/interfaces/rate-limits.tsindex 72a5d3b001..91b78fcc3f 100644--- a/packages/shared/src/interfaces/rate-limits.ts+++ b/packages/shared/src/interfaces/rate-limits.ts@@ -17,6 +17,9 @@ export const RateLimitResource = z.enum(["score-delete","in-app-agent-run","feedback",+ "ai-gateway-requests",+ // Points are request body bytes, not requests.+ "ai-gateway-request-bytes",]);// result of the rate limit check.diff --git a/web/src/__e2e__/ai-gateway.gatewaye2e.ts b/web/src/__e2e__/ai-gateway.gatewaye2e.tsindex 078bb6afd4..22b05a66f9 100644--- a/web/src/__e2e__/ai-gateway.gatewaye2e.ts+++ b/web/src/__e2e__/ai-gateway.gatewaye2e.ts@@ -843,7 +843,11 @@ async function gatewayControlPlaneRequest(input: {"Content-Type": "application/json","Langfuse-Gateway-Authorization": gatewayAuthorization,},- body: JSON.stringify({ apiFormat: input.apiFormat, model: input.model }),+ body: JSON.stringify({+ apiFormat: input.apiFormat,+ model: input.model,+ requestBytes: 0,+ }),});return readJsonResponse(response, 200, RESOLVE_PATH);}diff --git a/web/src/__tests__/server/ai-gateway-rate-limit.servertest.ts b/web/src/__tests__/server/ai-gateway-rate-limit.servertest.tsnew file mode 100644index 0000000000..823b0952b1--- /dev/null+++ b/web/src/__tests__/server/ai-gateway-rate-limit.servertest.ts@@ -0,0 +1,191 @@+import { randomUUID } from "crypto";+import type { Redis } from "ioredis";+import type { NextApiRequest, NextApiResponse } from "next";+import { createMocks } from "node-mocks-http";+import { logger } from "@langfuse/shared/src/server";++import {+ clearRedisKeysByPatternSafely,+ createRedisTestClient,+ ensureRedisReady,+ type RedisTestClient,+} from "@/src/__tests__/server/redis-test-utils";+import {+ type GatewayApiKeyAuthContext,+ GatewayApiKeyAuthenticator,+} from "@/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator";+import {+ enforceGatewayRateLimits,+ GatewayRateLimitError,+} from "@/src/features/ai-gateway/server/resolve/gatewayRateLimit";+import { gatewayResolveApiHandler } from "@/src/features/ai-gateway/server/resolve/gatewayResolveApiHandler";+import {+ RATE_LIMIT_REDIS_KEY_PREFIX,+ RateLimitService,+} from "@/src/features/public-api/server/RateLimitService";++const MiB = 1024 * 1024;++describe("AI Gateway organization rate limits", () => {+ let redis: RedisTestClient;+ let organizationId: string;++ const contextFor = (+ overrides: Partial<GatewayApiKeyAuthContext> = {},+ ): GatewayApiKeyAuthContext => ({+ organizationId,+ apiKeyId: "gateway-key-1",+ userId: null,+ apiKeyName: null,+ userName: null,+ userEmail: null,+ keyMetadata: {},+ ingestionProjectId: "project-1",+ ingestionMode: "USAGE",+ plan: "cloud:hobby",+ rateLimitOverrides: [],+ connection: null,+ providerModel: null,+ ...overrides,+ });++ const rejection = async (promise: Promise<unknown>) => {+ const error = await promise.then(+ () => null,+ (error: unknown) => error,+ );+ expect(error).toBeInstanceOf(GatewayRateLimitError);+ return error as GatewayRateLimitError;+ };++ beforeAll(async () => {+ RateLimitService.shutdown();+ redis = createRedisTestClient({+ maxRetriesPerRequest: null,+ enableAutoPipelining: false,+ });+ await ensureRedisReady(redis);+ RateLimitService.getInstance(redis as Redis);+ }, 20_000);++ beforeEach(() => {+ organizationId = `ai-gateway-rate-limit-org-${randomUUID()}`;+ });++ afterEach(async () => {+ vi.restoreAllMocks();+ await clearRedisKeysByPatternSafely(+ redis,+ `${RATE_LIMIT_REDIS_KEY_PREFIX}:*:${organizationId}`,+ );+ }, 20_000);++ afterAll(() => {+ redis.disconnect();+ RateLimitService.shutdown();+ });++ it("admits the plan's requests per minute and rejects the next one with a Langfuse message", async () => {+ const warn = vi.spyOn(logger, "warn");+ const context = contextFor();+ for (let i = 0; i < 100; i++) {+ await enforceGatewayRateLimits({ context, requestBytes: 10 });+ }++ const error = await rejection(+ enforceGatewayRateLimits({ context, requestBytes: 10 }),+ );++ expect(error.status).toBe(429);+ expect(error.code).toBe("rate_limit_exceeded");+ expect(error.limit).toBe("requests");+ expect(error.retryAfterSeconds).toBeGreaterThanOrEqual(1);+ expect(error.retryAfterSeconds).toBeLessThanOrEqual(60);+ expect(error.message).toBe(+ `Langfuse AI Gateway rate limit exceeded: your organization's Hobby plan allows 100 requests per minute. Retry after ${error.retryAfterSeconds}s or upgrade your plan.`,+ );+ expect(warn).toHaveBeenCalledWith("AI Gateway rate limit exceeded", {+ orgId: organizationId,+ projectId: "project-1",+ apiKeyId: "gateway-key-1",+ plan: "cloud:hobby",+ limit: "requests",+ current: 101,+ limitValue: 100,+ requestBytes: 10,+ retryAfterSeconds: error.retryAfterSeconds,+ });+ });++ it("counts request bodies toward the per-minute bytes limit", async () => {+ const context = contextFor({ plan: "cloud:core" });+ await enforceGatewayRateLimits({ context, requestBytes: 256 * MiB });++ const error = await rejection(+ enforceGatewayRateLimits({ context, requestBytes: 1 }),+ );++ expect(error.limit).toBe("bytes");+ expect(error.message).toBe(+ `Langfuse AI Gateway rate limit exceeded: your organization's Core plan allows 256 MiB of request bodies per minute. Retry after ${error.retryAfterSeconds}s or upgrade your plan.`,+ );+ });++ it("names a custom limit, its window, and support when the organization has an override", async () => {+ const context = contextFor({+ plan: "cloud:enterprise",+ rateLimitOverrides: [+ { resource: "ai-gateway-requests", points: 2, durationInSec: 3600 },+ ],+ });+ await enforceGatewayRateLimits({ context, requestBytes: 0 });+ await enforceGatewayRateLimits({ context, requestBytes: 0 });++ const error = await rejection(+ enforceGatewayRateLimits({ context, requestBytes: 0 }),+ );++ expect(error.message).toBe(+ `Langfuse AI Gateway rate limit exceeded: your organization's custom limit allows 2 requests per hour. Retry after ${error.retryAfterSeconds}s or contact Langfuse support to raise the limit.`,+ );+ });++ it("never limits self-hosted plans", async () => {+ const context = contextFor({ plan: "oss" });++ await expect(+ enforceGatewayRateLimits({ context, requestBytes: 1024 * MiB }),+ ).resolves.toBeUndefined();+ });++ it("answers /resolve with a 429 the gateway can map to a provider-native error", async () => {+ vi.spyOn(+ GatewayApiKeyAuthenticator.prototype,+ "authenticateRequest",+ ).mockResolvedValue(contextFor());+ const { req, res } = createMocks<NextApiRequest, NextApiResponse>({+ method: "POST",+ });++ await gatewayResolveApiHandler({+ req,+ res,+ fastHashedSecretKey: "hashed-secret-key",+ apiFormat: "openai.responses",+ model: "gpt-4o-mini",+ requestBytes: 64 * MiB + 1,+ });++ expect(res.statusCode).toBe(429);+ const retryAfter = Number(res.getHeader("Retry-After"));+ expect(retryAfter).toBeGreaterThanOrEqual(1);+ expect(res.getHeader("langfuse-gateway-error-code")).toBe(+ "rate_limit_exceeded",+ );+ expect(res._getJSONData()).toEqual({+ error: `Langfuse AI Gateway rate limit exceeded: your organization's Hobby plan allows 64 MiB of request bodies per minute. Retry after ${retryAfter}s or upgrade your plan.`,+ limit: "bytes",+ organizationId,+ });+ });+});diff --git a/web/src/__tests__/server/organizations-api-key-trpc.servertest.ts b/web/src/__tests__/server/organizations-api-key-trpc.servertest.tsindex 0d61efc0c6..54aad9ab47 100644--- a/web/src/__tests__/server/organizations-api-key-trpc.servertest.ts+++ b/web/src/__tests__/server/organizations-api-key-trpc.servertest.ts@@ -468,6 +468,8 @@ describe("organization API keys trpc", () => {keyMetadata: {},ingestionProjectId: null,ingestionMode: null,+ plan: "cloud:hobby",+ rateLimitOverrides: [],connections: [],},});@@ -635,6 +637,8 @@ describe("organization API keys trpc", () => {keyMetadata: {},ingestionProjectId: null,ingestionMode: null,+ plan: "cloud:hobby",+ rateLimitOverrides: [],connections: [],},});diff --git a/web/src/__tests__/server/rate-limit.servertest.ts b/web/src/__tests__/server/rate-limit.servertest.tsindex 481774432c..70816d67d6 100644--- a/web/src/__tests__/server/rate-limit.servertest.ts+++ b/web/src/__tests__/server/rate-limit.servertest.ts@@ -791,6 +791,65 @@ describe("RateLimitService", () => {}});+ it("should apply AI Gateway request and request-byte limits by cloud plan", async () => {+ const MiB = 1024 * 1024;+ const GiB = 1024 * MiB;+ const cases = [+ { plan: "cloud:hobby" as const, requests: 100, requestBytes: 64 * MiB },+ { plan: "cloud:core" as const, requests: 500, requestBytes: 256 * MiB },+ { plan: "cloud:pro" as const, requests: 3_000, requestBytes: 2 * GiB },+ { plan: "cloud:team" as const, requests: 3_000, requestBytes: 2 * GiB },+ {+ plan: "cloud:enterprise" as const,+ requests: 3_000,+ requestBytes: 2 * GiB,+ },+ ];++ const rateLimitService = RateLimitService.getInstance(redis as Redis);++ for (const testCase of cases) {+ await redis.del(+ `${RATE_LIMIT_REDIS_KEY_PREFIX}:ai-gateway-requests:${orgId}`,+ );+ await redis.del(+ `${RATE_LIMIT_REDIS_KEY_PREFIX}:ai-gateway-request-bytes:${orgId}`,+ );+ const scope = asScope({+ orgId,+ plan: testCase.plan,+ projectId,+ accessLevel: "organization",+ rateLimitOverrides: [],+ });++ const requests = await rateLimitService.rateLimitRequest(+ scope,+ "ai-gateway-requests",+ );+ const bytes = await rateLimitService.rateLimitRequest(+ scope,+ "ai-gateway-request-bytes",+ 4096,+ );++ expect(requests?.res).toMatchObject({+ points: testCase.requests,+ consumedPoints: 1,+ });+ expect(bytes?.res).toMatchObject({+ points: testCase.requestBytes,+ consumedPoints: 4096,+ remainingPoints: testCase.requestBytes - 4096,+ });+ const ttlInSec = await redis.ttl(+ `${RATE_LIMIT_REDIS_KEY_PREFIX}:ai-gateway-request-bytes:${orgId}`,+ );+ expect(ttlInSec).toBeGreaterThan(0);+ expect(ttlInSec).toBeLessThanOrEqual(60);+ }+ });+it("should apply media-upload rate limits separately from ingestion", async () => {const scope = {orgId: orgId,diff --git a/web/src/__tests__/server/unit/ai-gateway-resolve-route.servertest.ts b/web/src/__tests__/server/unit/ai-gateway-resolve-route.servertest.tsindex e5ce62da0d..9a6289dc40 100644--- a/web/src/__tests__/server/unit/ai-gateway-resolve-route.servertest.ts+++ b/web/src/__tests__/server/unit/ai-gateway-resolve-route.servertest.ts@@ -51,6 +51,26 @@ describe("POST /api/internal/ai-gateway/v1/resolve", () => {body: { apiFormat: "openai.responses", limit: 10 },},{ case: "not an object", body: "openai.responses" },+ {+ case: "missing request bytes",+ body: { apiFormat: "openai.responses", model: "gpt-4o-mini" },+ },+ {+ case: "negative request bytes",+ body: {+ apiFormat: "openai.responses",+ model: "gpt-4o-mini",+ requestBytes: -1,+ },+ },+ {+ case: "fractional request bytes",+ body: {+ apiFormat: "openai.responses",+ model: "gpt-4o-mini",+ requestBytes: 1.5,+ },+ },])("rejects an invalid body: $case", async ({ body }) => {expect((await post(body)).statusCode).toBe(400);});@@ -67,6 +87,7 @@ describe("POST /api/internal/ai-gateway/v1/resolve", () => {await post({apiFormat: format,model: "production:model-with:separator",+ requestBytes: 0,})).statusCode,).toBe(401);@@ -78,7 +99,9 @@ describe("POST /api/internal/ai-gateway/v1/resolve", () => {])("accepts inferred provider model $model and advances to authorization",async ({ model, apiFormat }) => {- expect((await post({ apiFormat, model })).statusCode).toBe(401);+ expect(+ (await post({ apiFormat, model, requestBytes: 1024 })).statusCode,+ ).toBe(401);},);@@ -91,15 +114,27 @@ describe("POST /api/internal/ai-gateway/v1/resolve", () => {it.each([{case: "unknown unprefixed model",- body: { apiFormat: "openai.responses", model: "random-model" },+ body: {+ apiFormat: "openai.responses",+ model: "random-model",+ requestBytes: 0,+ },},{case: "empty connection name",- body: { apiFormat: "openai.responses", model: ":gpt-4o-mini" },+ body: {+ apiFormat: "openai.responses",+ model: ":gpt-4o-mini",+ requestBytes: 0,+ },},{case: "empty provider model",- body: { apiFormat: "openai.responses", model: "production:" },+ body: {+ apiFormat: "openai.responses",+ model: "production:",+ requestBytes: 0,+ },},])("rejects an invalid model slug: $case", async ({ body }) => {expect((await post(body)).statusCode).toBe(400);diff --git a/web/src/__tests__/server/unit/gatewayResolveCache.servertest.ts b/web/src/__tests__/server/unit/gatewayResolveCache.servertest.tsindex 5661d44397..1a4553e2d8 100644--- a/web/src/__tests__/server/unit/gatewayResolveCache.servertest.ts+++ b/web/src/__tests__/server/unit/gatewayResolveCache.servertest.ts@@ -23,6 +23,8 @@ const context: CachedResolveContext = {keyMetadata: {},ingestionProjectId: null,ingestionMode: null,+ plan: "cloud:hobby",+ rateLimitOverrides: [],connections: [],};@@ -118,6 +120,8 @@ describe("GatewayResolveCache", () => {keyMetadata: {},ingestionProjectId: null,ingestionMode: null,+ plan: "cloud:hobby",+ rateLimitOverrides: [],connections: [],};await cache.set({@@ -139,6 +143,28 @@ describe("GatewayResolveCache", () => {});});+ it("refetches entries cached before the plan was part of the context", async () => {+ const redis = createStatefulRedis();+ const cache = new GatewayResolveCache(redis);+ const {+ plan: _plan,+ rateLimitOverrides: _overrides,+ ...withoutPlan+ } = context;+ await cache.set({+ fastHashedSecretKey: "hashed-key",+ apiFormat: "openai.responses",+ context: withoutPlan as unknown as CachedResolveContext,+ });++ await expect(+ cache.get({+ fastHashedSecretKey: "hashed-key",+ apiFormat: "openai.responses",+ }),+ ).resolves.toBeNull();+ });+it("invalidates cached contexts for an organization", async () => {const cache = new GatewayResolveCache(createStatefulRedis());diff --git a/web/src/ee/features/billing/server/chb/chbWebhookHandler.ts b/web/src/ee/features/billing/server/chb/chbWebhookHandler.tsindex cbcfb11e77..fd69f58b47 100644--- a/web/src/ee/features/billing/server/chb/chbWebhookHandler.ts+++ b/web/src/ee/features/billing/server/chb/chbWebhookHandler.ts@@ -5,6 +5,7 @@ import { z } from "zod";import { env } from "@/src/env.mjs";import { auditLog } from "@/src/features/audit-logs/server";+import { invalidateGatewayResolveCacheForOrganization } from "@/src/features/ai-gateway/server/resolve/gatewayResolveCache";import {isoDateTime,type ChbPlanCode,@@ -437,6 +438,7 @@ async function persistAndPropagate(params: {// TTL.try {await invalidateCachedOrgApiKeys(parsedOrg.id);+ await invalidateGatewayResolveCacheForOrganization(parsedOrg.id);} catch (error) {logger.error(`[CHB Webhook] Failed to invalidate cached API keys for org ${parsedOrg.id} after applying ${event.type}`,diff --git a/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.ts b/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.tsindex 6f81e239d5..df6116958f 100644--- a/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.ts+++ b/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.ts@@ -22,6 +22,7 @@ import {startOfDayUTC,} from "@langfuse/shared/src/server";import { auditLog } from "@/src/features/audit-logs/server";+import { invalidateGatewayResolveCacheForOrganization } from "@/src/features/ai-gateway/server/resolve/gatewayResolveCache";import { type StripeSubscriptionMetadata } from "@/src/ee/features/billing/utils/stripeSubscriptionMetadata";import { syncOrgPlanChangeToSfdc } from "@/src/ee/features/sfdc-sync/server";import { createDefaultSpendAlertsForStripeProduct } from "@/src/ee/features/billing/server/defaultSpendAlerts";@@ -676,6 +677,7 @@ export async function handleSubscriptionChanged(// Invalidate API keys in Redis for it to be updatedawait invalidateCachedOrgApiKeys(parsedOrg.id);+ await invalidateGatewayResolveCacheForOrganization(parsedOrg.id);auditLog({session: {@@ -725,6 +727,7 @@ export async function handleSubscriptionChanged(// Reset billing cycle anchor on downgrade to hobby to start of todayawait updateOrgBillingCycleAnchor(parsedOrg.id);await invalidateCachedOrgApiKeys(parsedOrg.id);+ await invalidateGatewayResolveCacheForOrganization(parsedOrg.id);auditLog({session: {diff --git a/web/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator.ts b/web/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator.tsindex a235604666..f6497ff3f1 100644--- a/web/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator.ts+++ b/web/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator.ts@@ -1,9 +1,11 @@+import { CloudConfigSchema } from "@langfuse/shared";import type { GatewayProvider, PrismaClient } from "@langfuse/shared/src/db";import { ApiKeyId, OrganizationId } from "@langfuse/shared/rbac";import { authorize } from "@/src/features/rbac/authorize";import { getRolesForPrincipal } from "@/src/features/rbac/getRolesForPrincipal";import { hasOrganizationAccess } from "@/src/features/rbac/utils/checkOrganizationAccess";+import { getOrganizationPlanServerSide } from "@/src/features/entitlements/server";import { isGatewayEnabledForOrganization } from "@/src/features/ai-gateway/server/availability";import { GatewayControlPlaneError } from "@/src/features/ai-gateway/server/gatewayControlPlaneError";import {@@ -146,6 +148,9 @@ export class GatewayApiKeyAuthenticator {Boolean(project) &&!project?.deletedAt &&project?.orgId === organizationId;+ const cloudConfig = organization.cloudConfig+ ? CloudConfigSchema.safeParse(organization.cloudConfig).data+ : undefined;const context: CachedResolveContext = {organizationId,apiKeyId: row.id,@@ -158,6 +163,8 @@ export class GatewayApiKeyAuthenticator {? (gatewayConfig?.defaultIngestionProjectId ?? null): null,ingestionMode: gatewayConfig?.ingestionMode ?? null,+ plan: getOrganizationPlanServerSide(cloudConfig),+ rateLimitOverrides: cloudConfig?.rateLimitOverrides ?? [],connections: organization.gatewayAiConnections.map((connection) => ({...connection,allowedModels: parseGatewayAllowedModels(connection.allowedModels),diff --git a/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.ts b/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.tsindex 125e3f4e4c..4b5f18f08d 100644--- a/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.ts+++ b/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.ts@@ -110,7 +110,8 @@ describe("withGatewayResolveSignatureVerification", () => {async (secret) => {vi.useFakeTimers();vi.setSystemTime(now);- const body = '{ "apiFormat": "openai.responses", "model": "gpt-5" }\n';+ const body =+ '{ "apiFormat": "openai.responses", "model": "gpt-5", "requestBytes": 512 }\n';const req = request({authorization: " Bearer sk-gateway ",gatewayAuthorization: gatewayAuthorization({ secret }),@@ -127,6 +128,7 @@ describe("withGatewayResolveSignatureVerification", () => {fastHashedSecretKey: createShaHash("sk-gateway", "test-salt"),apiFormat: "openai.responses",model: "gpt-5",+ requestBytes: 512,});},);@@ -143,6 +145,7 @@ describe("withGatewayResolveSignatureVerification", () => {body: JSON.stringify({apiFormat: "openai.responses",model,+ requestBytes: 0,}),}),res,@@ -205,7 +208,7 @@ describe("withGatewayResolveSignatureVerification", () => {secret: "current-service-secret",key: "sk-other-gateway",}),- body: '{"apiFormat":"openai.responses","model":"gpt-5"}',+ body: '{"apiFormat":"openai.responses","model":"gpt-5","requestBytes":0}',}),res,);@@ -221,7 +224,8 @@ describe("withGatewayResolveSignatureVerification", () => {it("rejects stale signatures and headers containing a key id", async () => {vi.useFakeTimers();vi.setSystemTime(now);- const body = '{"apiFormat":"openai.responses","model":"gpt-5"}';+ const body =+ '{"apiFormat":"openai.responses","model":"gpt-5","requestBytes":0}';const staleHeader = gatewayAuthorization({secret: "current-service-secret",timestamp: Math.floor(now.getTime() / 1000) - 301,diff --git a/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.ts b/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.tsindex 854b1740c6..6d9dd84bf5 100644--- a/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.ts+++ b/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.ts@@ -23,6 +23,7 @@ const resolveBodySchema = zmodel: z.string().refine((model) => modelRoutingService.resolve(model, []) !== null),+ requestBytes: z.number().int().nonnegative(),}).strict();const modelsQuerySchema = z@@ -43,6 +44,7 @@ type GatewayResolveHandler = (params: GatewayControlPlaneHandlerParams & {apiFormat: GatewayApiFormat;model: string;+ requestBytes: number;},) => Promise<unknown>;@@ -119,6 +121,7 @@ export function withGatewayResolveSignatureVerification(...params,apiFormat: body.apiFormat,model: body.model,+ requestBytes: body.requestBytes,}),resolveBodySchema,);diff --git a/web/src/features/ai-gateway/server/gatewayControlPlaneError.ts b/web/src/features/ai-gateway/server/gatewayControlPlaneError.tsindex abf519c54c..6d13f7e5e1 100644--- a/web/src/features/ai-gateway/server/gatewayControlPlaneError.ts+++ b/web/src/features/ai-gateway/server/gatewayControlPlaneError.ts@@ -1,8 +1,11 @@export class GatewayControlPlaneError extends Error {constructor(message: string,- readonly status: 400 | 401 | 403 | 404 | 503,- readonly code?: "model_not_allowed" | "invalid_gateway_authorization",+ readonly status: 400 | 401 | 403 | 404 | 429 | 503,+ readonly code?:+ | "model_not_allowed"+ | "invalid_gateway_authorization"+ | "rate_limit_exceeded",) {super(message);}diff --git a/web/src/features/ai-gateway/server/resolve/gatewayRateLimit.ts b/web/src/features/ai-gateway/server/resolve/gatewayRateLimit.tsnew file mode 100644index 0000000000..5c3e681a25--- /dev/null+++ b/web/src/features/ai-gateway/server/resolve/gatewayRateLimit.ts@@ -0,0 +1,143 @@+import { planLabels, type Plan, type RateLimitResult } from "@langfuse/shared";+import { getCurrentSpan, logger } from "@langfuse/shared/src/server";++import type { GatewayApiKeyAuthContext } from "@/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator";+import { GatewayControlPlaneError } from "@/src/features/ai-gateway/server/gatewayControlPlaneError";+import { RateLimitService } from "@/src/features/public-api/server/RateLimitService";++type GatewayRateLimitDimension = "requests" | "bytes";++const MiB = 1024 * 1024;+const GiB = 1024 * MiB;++export class GatewayRateLimitError extends GatewayControlPlaneError {+ constructor(+ message: string,+ readonly limit: GatewayRateLimitDimension,+ readonly retryAfterSeconds: number,+ readonly organizationId: string,+ ) {+ super(message, 429, "rate_limit_exceeded");+ }+}++/**+ * Counts one request and its body bytes against the organization's per-minute+ * AI Gateway limits. Fails open like every other RateLimitService caller:+ * self-hosted, disabled rate limits, or an unavailable Redis never reject.+ */+export async function enforceGatewayRateLimits({+ context,+ requestBytes,+}: {+ context: GatewayApiKeyAuthContext;+ requestBytes: number;+}) {+ const scope = {+ orgId: context.organizationId,+ projectId: context.ingestionProjectId,+ accessLevel: "organization" as const,+ plan: context.plan,+ rateLimitOverrides: context.rateLimitOverrides,+ apiKeyId: context.apiKeyId,+ publicKey: context.apiKeyId,+ isIngestionSuspended: false,+ accessAuditLogEnabled: null,+ };+ const service = RateLimitService.getInstance();+ const [requests, bytes] = await Promise.all([+ service.rateLimitRequest(scope, "ai-gateway-requests"),+ service.rateLimitRequest(scope, "ai-gateway-request-bytes", requestBytes),+ ]);++ const exceeded = (+ [+ { limit: "requests", helper: requests },+ { limit: "bytes", helper: bytes },+ ] as const+ )+ .flatMap(({ limit, helper }) =>+ helper.isRateLimited() && helper.res ? [{ limit, res: helper.res }] : [],+ )+ // Retrying before the later window resets would hit the other limit.+ .toSorted((a, b) => b.res.msBeforeNext - a.res.msBeforeNext)[0];+ if (!exceeded) return;++ const { limit, res } = exceeded;+ const retryAfterSeconds = Math.max(1, Math.ceil(res.msBeforeNext / 1000));+ logger.warn("AI Gateway rate limit exceeded", {+ orgId: context.organizationId,+ projectId: context.ingestionProjectId,+ apiKeyId: context.apiKeyId,+ plan: context.plan,+ limit,+ current: res.consumedPoints,+ limitValue: res.points,+ requestBytes,+ retryAfterSeconds,+ });+ getCurrentSpan()?.setAttributes({+ "langfuse.ai_gateway.rate_limited": true,+ "langfuse.ai_gateway.rate_limit.limit": limit,+ });+ const override = context.rateLimitOverrides.find(+ (override) => override.resource === res.resource,+ );+ throw new GatewayRateLimitError(+ gatewayRateLimitMessage({+ limit,+ res,+ plan: context.plan,+ retryAfterSeconds,+ hasOverride: Boolean(override),+ windowSeconds: override?.durationInSec ?? 60,+ }),+ limit,+ retryAfterSeconds,+ context.organizationId,+ );+}++function gatewayRateLimitMessage(params: {+ limit: GatewayRateLimitDimension;+ res: RateLimitResult;+ plan: Plan;+ retryAfterSeconds: number;+ hasOverride: boolean;+ windowSeconds: number;+}) {+ const window = formatWindow(params.windowSeconds);+ const allowance =+ params.limit === "requests"+ ? `${params.res.points.toLocaleString("en-US")} requests per ${window}`+ : `${formatBytes(params.res.points)} of request bodies per ${window}`;+ const subject = params.hasOverride+ ? "your organization's custom limit allows"+ : `your organization's ${planLabels[params.plan]} plan allows`;+ const remedy =+ params.plan === "cloud:enterprise" || params.hasOverride+ ? "contact Langfuse support to raise the limit"+ : "upgrade your plan";+ return `Langfuse AI Gateway rate limit exceeded: ${subject} ${allowance}. Retry after ${params.retryAfterSeconds}s or ${remedy}.`;+}++function formatBytes(bytes: number) {+ return bytes >= GiB && bytes % GiB === 0+ ? `${bytes / GiB} GiB`+ : `${Math.round((bytes / MiB) * 10) / 10} MiB`;+}++const windowUnits = [+ { seconds: 86_400, name: "day" },+ { seconds: 3_600, name: "hour" },+ { seconds: 60, name: "minute" },+] as const;++function formatWindow(seconds: number) {+ for (const unit of windowUnits) {+ if (seconds % unit.seconds !== 0) continue;+ const count = seconds / unit.seconds;+ return count === 1 ? unit.name : `${count} ${unit.name}s`;+ }+ return seconds === 1 ? "second" : `${seconds} seconds`;+}diff --git a/web/src/features/ai-gateway/server/resolve/gatewayResolveApiHandler.ts b/web/src/features/ai-gateway/server/resolve/gatewayResolveApiHandler.tsindex 1ec9659b27..dcabdc3081 100644--- a/web/src/features/ai-gateway/server/resolve/gatewayResolveApiHandler.ts+++ b/web/src/features/ai-gateway/server/resolve/gatewayResolveApiHandler.ts@@ -7,6 +7,10 @@ import {type GatewayApiFormat,GatewayResolveResponseSchema,} from "@/src/features/ai-gateway/server/provider";+import {+ enforceGatewayRateLimits,+ GatewayRateLimitError,+} from "./gatewayRateLimit";import { GatewayResolveError, GatewayResolveService } from "./resolveService";export async function gatewayResolveApiHandler({@@ -14,23 +18,35 @@ export async function gatewayResolveApiHandler({fastHashedSecretKey,apiFormat,model,+ requestBytes,}: {req: NextApiRequest;res: NextApiResponse;fastHashedSecretKey: string;apiFormat: GatewayApiFormat;model: string;+ requestBytes: number;}) {try {const context = await new GatewayApiKeyAuthenticator(prisma,).authenticateRequest({ fastHashedSecretKey, apiFormat, model });+ await enforceGatewayRateLimits({ context, requestBytes });const result = await new GatewayResolveService().resolve({context,apiFormat,});return res.status(200).json(GatewayResolveResponseSchema.parse(result));} catch (error) {+ if (error instanceof GatewayRateLimitError) {+ res.setHeader("Retry-After", String(error.retryAfterSeconds));+ res.setHeader("langfuse-gateway-error-code", "rate_limit_exceeded");+ return res.status(error.status).json({+ error: error.message,+ limit: error.limit,+ organizationId: error.organizationId,+ });+ }if (error instanceof GatewayResolveError) {// The data plane retries a 503, so tell it when to come back rather than// letting it hammer a database that is already struggling.diff --git a/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.ts b/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.tsindex 9e01069046..0aec5c7473 100644--- a/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.ts+++ b/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.ts@@ -3,6 +3,7 @@ import {recordIncrement,redis as defaultRedis,} from "@langfuse/shared/src/server";+import { CloudConfigRateLimit, type Plan, plans } from "@langfuse/shared";import type { Cluster, Redis } from "ioredis";import { z } from "zod/v4";@@ -34,6 +35,8 @@ export const CachedResolveContext = z.object({keyMetadata: GatewayMetadataSchema,ingestionProjectId: z.string().nullable(),ingestionMode: z.enum(["NONE", "USAGE", "FULL"]).nullable(),+ plan: z.custom<Plan>((value) => plans.includes(value as Plan)),+ rateLimitOverrides: CloudConfigRateLimit,connections: z.array(z.object({id: z.string(),diff --git a/web/src/features/ai-gateway/server/resolve/gatewayResolveRepository.ts b/web/src/features/ai-gateway/server/resolve/gatewayResolveRepository.tsindex 45d5f49dce..294064f9b4 100644--- a/web/src/features/ai-gateway/server/resolve/gatewayResolveRepository.ts+++ b/web/src/features/ai-gateway/server/resolve/gatewayResolveRepository.ts@@ -32,6 +32,7 @@ export class GatewayResolveRepository {gatewayAssociation: { select: { metadata: true } },organization: {select: {+ cloudConfig: true,gatewayConfig: {select: {defaultIngestionProjectId: true,diff --git a/web/src/features/public-api/server/RateLimitService.ts b/web/src/features/public-api/server/RateLimitService.tsindex 51fd49c32b..595a176fc8 100644--- a/web/src/features/public-api/server/RateLimitService.ts+++ b/web/src/features/public-api/server/RateLimitService.ts@@ -80,9 +80,11 @@ export class RateLimitService {RateLimitService.instance = null;}+ /** `points` is what this request costs, e.g. its byte size for byte-based resources. */async rateLimitRequest(scope: ApiAccessScopeWithOptionalApiKeyId,resource: z.infer<typeof RateLimitResource>,+ points = 1,) {// if cloud config is not present, we don't apply rate limits and just returnif (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {@@ -98,12 +100,15 @@ export class RateLimitService {return new RateLimitHelper(undefined);}- return new RateLimitHelper(await this.checkRateLimit(scope, resource));+ return new RateLimitHelper(+ await this.checkRateLimit(scope, resource, points),+ );}async checkRateLimit(scope: ApiAccessScopeWithOptionalApiKeyId,resource: z.infer<typeof RateLimitResource>,+ points = 1,) {const effectiveConfig = getRateLimitConfig(scope, resource);@@ -138,7 +143,7 @@ export class RateLimitService {let res: RateLimitResult | undefined = undefined;try {// orgId used as key for different resources- const libRes = await rateLimiter.consume(scope.orgId);+ const libRes = await rateLimiter.consume(scope.orgId, points);res = {resource,scope,@@ -289,6 +294,9 @@ const getRateLimitConfig = (return customConfig || planBasedConfig;};+const MiB = 1024 * 1024;+const GiB = 1024 * MiB;+const getPlanBasedRateLimitConfig = (plan: Plan,resource: z.infer<typeof RateLimitResource>,@@ -395,6 +403,10 @@ const getPlanBasedRateLimitConfig = (points: 20,durationInSec: 86400, // 20 submissions per day};+ case "ai-gateway-requests":+ return { resource, points: 100, durationInSec: 60 };+ case "ai-gateway-request-bytes":+ return { resource, points: 64 * MiB, durationInSec: 60 };default:const exhaustiveCheck: never = resource;throw new Error(`Unhandled resource case: ${exhaustiveCheck}`);@@ -499,6 +511,10 @@ const getPlanBasedRateLimitConfig = (points: 20,durationInSec: 86400, // 20 submissions per day};+ case "ai-gateway-requests":+ return { resource, points: 500, durationInSec: 60 };+ case "ai-gateway-request-bytes":+ return { resource, points: 256 * MiB, durationInSec: 60 };default:const exhaustiveCheck: never = resource;throw new Error(`Unhandled resource case: ${exhaustiveCheck}`);@@ -597,6 +613,10 @@ const getPlanBasedRateLimitConfig = (points: 20,durationInSec: 86400, // 20 submissions per day};+ case "ai-gateway-requests":+ return { resource, points: 3_000, durationInSec: 60 };+ case "ai-gateway-request-bytes":+ return { resource, points: 2 * GiB, durationInSec: 60 };default:const exhaustiveCheck: never = resource;throw new Error(`Unhandled resource case: ${exhaustiveCheck}`);