🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(ai-gateway): resolve NONE ingestion mode without an ingestion project or signing keys (#19074)
Tobias Wochinger6b4f9d9b6800
diff --git a/ai-gateway/src/capture/mod.rs b/ai-gateway/src/capture/mod.rsindex eef54ba7eb..d375dce16c 100644--- a/ai-gateway/src/capture/mod.rs+++ b/ai-gateway/src/capture/mod.rs@@ -13,7 +13,7 @@ use tokio::time::Instant;use crate::{correlation::RequestCorrelation,- resolution::{ApiFormat, IngestionMode, MetadataValue, ResolvedRequestContext},+ resolution::{ApiFormat, IngestionMode, MetadataValue, ProjectId, ResolvedRequestContext},telemetry,};use anthropic_messages::AnthropicMessagesCapture;@@ -168,7 +168,7 @@ impl ExecutionCapture {http_status: None,metadata: json!({"organization_id": attribution.organization_id().as_str(),- "project_id": attribution.project_id().as_str(),+ "project_id": attribution.project_id().map(ProjectId::as_str),"key_id": attribution.key_id(),"provider_connection_id": attribution.provider_connection_id(),"key_metadata": key_metadata,@@ -190,7 +190,11 @@ impl ExecutionCapture {.protocol.as_ref().and_then(ProtocolCapture::client_metadata);- let delivery = telemetry::DeliveryContext::from_resolved(context, headers, client_metadata);+ let Some(delivery) =+ telemetry::DeliveryContext::from_resolved(context, headers, client_metadata)+ else {+ return;+ };correlation.record_generation(delivery.generation_ids());self.metadata["request_id"] = json!(correlation.id());if let Some(client_id) = correlation.client_id() {diff --git a/ai-gateway/src/correlation.rs b/ai-gateway/src/correlation.rsindex 956c54115e..45c442a562 100644--- a/ai-gateway/src/correlation.rs+++ b/ai-gateway/src/correlation.rs@@ -36,7 +36,7 @@ pub(crate) struct GenerationIds {#[derive(Clone, Debug)]struct Tenant {organization_id: OrganizationId,- project_id: ProjectId,+ project_id: Option<ProjectId>,}/// The provider connection and model of an upstream attempt. With several@@ -135,11 +135,13 @@ impl RequestCorrelation {"langfuse.organization.id",attribution.organization_id().as_str(),);- self.server_span- .record("langfuse.project.id", attribution.project_id().as_str());+ if let Some(project_id) = attribution.project_id() {+ self.server_span+ .record("langfuse.project.id", project_id.as_str());+ }self.tenant = Some(Tenant {organization_id: attribution.organization_id().clone(),- project_id: attribution.project_id().clone(),+ project_id: attribution.project_id().cloned(),});}@@ -165,10 +167,10 @@ impl RequestCorrelation {let Some(tenant) = &self.tenant else {return;};- let mut values = vec![- (ORGANIZATION_ID_HEADER, tenant.organization_id.as_str()),- (PROJECT_ID_HEADER, tenant.project_id.as_str()),- ];+ let mut values = vec![(ORGANIZATION_ID_HEADER, tenant.organization_id.as_str())];+ if let Some(project_id) = &tenant.project_id {+ values.push((PROJECT_ID_HEADER, project_id.as_str()));+ }if let Some(upstream) = &self.upstream {values.push((PROVIDER_HEADER, upstream.provider));values.push((PROVIDER_CONNECTION_ID_HEADER, &upstream.connection_id));diff --git a/ai-gateway/src/resolution/contracts.rs b/ai-gateway/src/resolution/contracts.rsindex 2f50af47ff..4557eac60e 100644--- a/ai-gateway/src/resolution/contracts.rs+++ b/ai-gateway/src/resolution/contracts.rs@@ -241,7 +241,8 @@ impl From<&str> for ProjectId {#[serde(deny_unknown_fields)]pub struct RequestAttribution {organization_id: OrganizationId,- project_id: ProjectId,+ /// Absent only in [`IngestionMode::None`], which needs no ingestion project.+ project_id: Option<ProjectId>,key_id: String,key_metadata: BTreeMap<String, MetadataValue>,provider_connection_id: String,@@ -251,8 +252,8 @@ impl RequestAttribution {pub fn organization_id(&self) -> &OrganizationId {&self.organization_id}- pub fn project_id(&self) -> &ProjectId {- &self.project_id+ pub fn project_id(&self) -> Option<&ProjectId> {+ self.project_id.as_ref()}pub fn key_id(&self) -> &str {&self.key_id@@ -291,7 +292,8 @@ struct ResolutionResponse {attribution: RequestAttribution,#[serde(deserialize_with = "string_enum")]ingestion_mode: IngestionMode,- ingestion: IngestionGrant,+ /// Absent only in [`IngestionMode::None`]; nothing is delivered without it.+ ingestion: Option<IngestionGrant>,}/// A validated v1 execution. Construction is restricted to successful resolution.@@ -314,8 +316,19 @@ impl ResolvedRequestContext {pub fn ingestion_mode(&self) -> IngestionMode {self.0.ingestion_mode}- pub fn ingestion(&self) -> &IngestionGrant {- &self.0.ingestion+ pub fn ingestion(&self) -> Option<&IngestionGrant> {+ self.0.ingestion.as_ref()+ }+ /// The project and grant telemetry is delivered with. Resolution guarantees both+ /// in every mode but [`IngestionMode::None`], which never delivers.+ pub fn delivery_target(&self) -> Option<(&ProjectId, &IngestionGrant)> {+ if self.0.ingestion_mode == IngestionMode::None {+ return None;+ }+ Some((+ self.0.attribution.project_id.as_ref()?,+ self.0.ingestion.as_ref()?,+ ))}}@@ -348,10 +361,24 @@ fn ingestion_claims_match(token: &str, attribution: &RequestAttribution) -> bool.and_then(|bytes| serde_json::from_slice::<IngestionClaims>(&bytes).ok()).is_some_and(|claims| {claims.organization_id == attribution.organization_id- && claims.project_id == attribution.project_id+ && Some(&claims.project_id) == attribution.project_id.as_ref()})}+/// A grant must be valid and bound to the attributed tenant whenever it is sent;+/// only [`IngestionMode::None`] may omit it.+fn valid_ingestion(response: &ResolutionResponse, now: u64) -> bool {+ match &response.ingestion {+ None => response.ingestion_mode == IngestionMode::None,+ Some(grant) => {+ let token = grant.access_token.expose_secret();+ valid_token(token)+ && ingestion_claims_match(token, &response.attribution)+ && grant.expires_at > now+ }+ }+}+pub(super) fn decode(bytes: &[u8],expected_format: ApiFormat,@@ -368,17 +395,15 @@ pub(super) fn decode(|| connection.base_url != provider.official_origin()|| !provider.accepts(connection.credential())|| !valid_token(connection.credential().secret())- || !valid_token(response.ingestion.access_token.expose_secret())- || !ingestion_claims_match(response.ingestion.access_token.expose_secret(), attribution)- || response.ingestion.expires_at <= now+ || !valid_ingestion(&response, now)|| [connection.id.as_str(),attribution.organization_id.as_str(),- attribution.project_id.as_str(),attribution.key_id.as_str(),attribution.provider_connection_id.as_str(),]- .iter()+ .into_iter()+ .chain(attribution.project_id.as_ref().map(ProjectId::as_str)).any(|value| value.trim().is_empty())|| connection.provider_model.trim().is_empty(){diff --git a/ai-gateway/src/resolution/tests.rs b/ai-gateway/src/resolution/tests.rsindex a19f831ea6..994cf62fae 100644--- a/ai-gateway/src/resolution/tests.rs+++ b/ai-gateway/src/resolution/tests.rs@@ -149,13 +149,16 @@ async fn signs_the_exact_key_and_sends_api_format_and_model() {context.connection().credential(),ProviderCredential::Bearer("private-provider-token")));- assert_eq!(context.attribution().project_id().as_str(), "project-1");+ assert_eq!(+ context.attribution().project_id().map(ProjectId::as_str),+ Some("project-1")+ );assert_eq!(context.attribution().provider_connection_id(),"connection-1");assert_eq!(- context.ingestion().access_token().expose_secret(),+ context.ingestion().unwrap().access_token().expose_secret(),ingestion_token("org-1", "project-1"));let debug = format!("{context:?}");@@ -516,12 +519,18 @@ async fn concurrent_resolutions_keep_credentials_and_contexts_isolated() {);let alice = alice.unwrap();let bob = bob.unwrap();- assert_eq!(alice.attribution().project_id().as_str(), "project-alice");+ assert_eq!(+ alice.attribution().project_id().map(ProjectId::as_str),+ Some("project-alice")+ );assert!(matches!(alice.connection().credential(),ProviderCredential::Bearer("token-alice")));- assert_eq!(bob.attribution().project_id().as_str(), "project-bob");+ assert_eq!(+ bob.attribution().project_id().map(ProjectId::as_str),+ Some("project-bob")+ );assert!(matches!(bob.connection().credential(),ProviderCredential::Bearer("token-bob")@@ -807,6 +816,53 @@ async fn rejects_ingestion_tokens_that_authorize_a_different_tenant() {);}+#[tokio::test]+async fn resolves_none_mode_without_an_ingestion_grant_or_project() {+ let mut body = success("project-1", "provider-token");+ body["ingestion_mode"] = json!("none");+ body["ingestion"] = Value::Null;+ body["attribution"]["project_id"] = Value::Null;+ let web = FakeWeb::start(move |_| {+ let body = body.clone();+ async move { response(body) }+ })+ .await;+ let context = web+ .control_plane()+ .resolve_at(+ GATEWAY_KEY,+ ApiFormat::OpenAiResponses,+ "primary:gpt-4.1",+ NOW_TIME,+ REQUEST_ID,+ )+ .await+ .unwrap();+ assert_eq!(context.ingestion_mode(), IngestionMode::None);+ assert!(context.ingestion().is_none());+ assert!(context.attribution().project_id().is_none());+}++#[tokio::test]+async fn delivering_modes_require_an_ingestion_grant_and_project() {+ for mode in ["usage", "full"] {+ let mut body = success("project-1", "provider-token");+ body["ingestion_mode"] = json!(mode);+ body["ingestion"] = Value::Null;+ assert_invalid_context(body).await;++ let mut body = success("project-1", "provider-token");+ body["ingestion_mode"] = json!(mode);+ body["attribution"]["project_id"] = Value::Null;+ assert_invalid_context(body).await;+ }+ // A grant sent alongside NONE must still be bound to the attributed tenant.+ let mut body = success("project-1", "provider-token");+ body["ingestion_mode"] = json!("none");+ body["attribution"]["project_id"] = json!("project-2");+ assert_invalid_context(body).await;+}+async fn assert_invalid_context(body: Value) {assert_invalid_context_for(body, ApiFormat::OpenAiResponses).await;}diff --git a/ai-gateway/src/telemetry/mod.rs b/ai-gateway/src/telemetry/mod.rsindex 6de1aca06e..8e3b75a4c0 100644--- a/ai-gateway/src/telemetry/mod.rs+++ b/ai-gateway/src/telemetry/mod.rs@@ -68,19 +68,21 @@ pub(crate) struct DeliveryContext {}impl DeliveryContext {+ /// `None` when the request has nothing to deliver to, as in ingestion mode `none`.pub(crate) fn from_resolved(context: &ResolvedRequestContext,headers: &HeaderMap,client_metadata: Option<&Map<String, Value>>,- ) -> Self {- Self {+ ) -> Option<Self> {+ let (project_id, ingestion) = context.delivery_target()?;+ Some(Self {grant: Grant {- project_id: context.attribution().project_id().clone(),- access_token: context.ingestion().access_token().clone(),- expires_at: context.ingestion().expires_at(),+ project_id: project_id.clone(),+ access_token: ingestion.access_token().clone(),+ expires_at: ingestion.expires_at(),},generation: GenerationContext::from_request(headers, client_metadata),- }+ })}pub(crate) fn generation_ids(&self) -> crate::correlation::GenerationIds {diff --git a/ai-gateway/src/telemetry/tests.rs b/ai-gateway/src/telemetry/tests.rsindex 77f0fd79ea..c03509d622 100644--- a/ai-gateway/src/telemetry/tests.rs+++ b/ai-gateway/src/telemetry/tests.rs@@ -39,6 +39,7 @@ async fn grant() -> DeliveryContext {&HeaderMap::new(),None,)+ .unwrap()}fn uploader(url: &str) -> Uploader {@@ -642,7 +643,8 @@ async fn retained_byte_budget_is_released_after_successful_drain() {&resolved_request_context("provider-secret").await,&HeaderMap::new(),None,- );+ )+ .unwrap();telemetry.record(second, facts("project-1"));assert_eq!(telemetry.0.stats.dropped.load(Ordering::Relaxed), 1);release.notify_one();diff --git a/ai-gateway/src/test_support.rs b/ai-gateway/src/test_support.rsindex c210f8ca65..1ca1633aea 100644--- a/ai-gateway/src/test_support.rs+++ b/ai-gateway/src/test_support.rs@@ -165,6 +165,11 @@ pub(crate) async fn resolved_request_context_for(.unwrap();let mut body: serde_json::Value = serde_json::from_slice(&bytes).unwrap();body["ingestion_mode"] = mode.into();+ if mode == "none" {+ // Web mints no grant and may have no ingestion project in this mode.+ body["ingestion"] = serde_json::Value::Null;+ body["attribution"]["project_id"] = serde_json::Value::Null;+ }Response::builder().header("content-type", "application/json").body(Body::from(body.to_string()))diff --git a/web/src/__e2e__/ai-gateway.gatewaye2e.ts b/web/src/__e2e__/ai-gateway.gatewaye2e.tsindex 8fa3d7e356..8bc63ebe76 100644--- a/web/src/__e2e__/ai-gateway.gatewaye2e.ts+++ b/web/src/__e2e__/ai-gateway.gatewaye2e.ts@@ -620,7 +620,7 @@ describe("AI gateway live end-to-end", () => {expires_at: expect.any(Number),});- const ingestionToken = resolved.ingestion.access_token;+ const ingestionToken = resolved.ingestion!.access_token;await ingestTrace(ingestionToken, { provider, apiFormat });},);diff --git a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts b/web/src/__tests__/server/ai-gateway-control-plane.servertest.tsindex 2d91bf2710..947ca90b54 100644--- a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts+++ b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts@@ -860,6 +860,8 @@ describe("AI gateway control plane", () => {model: "gpt-4o",});expect(result.ingestion_mode).toBe("none");+ expect(result.ingestion).toBeNull();+ expect(result.attribution.project_id).toBe(project.id);Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU" });await expect(diff --git a/web/src/features/ai-gateway/server/provider/registry.ts b/web/src/features/ai-gateway/server/provider/registry.tsindex fa08daa629..cd124c63ae 100644--- a/web/src/features/ai-gateway/server/provider/registry.ts+++ b/web/src/features/ai-gateway/server/provider/registry.ts@@ -45,20 +45,23 @@ export const GatewayResolveResponseSchema = zattribution: z.object({organization_id: z.string(),- project_id: z.string(),+ // Null only in "none" mode, which needs no ingestion project.+ project_id: z.string().nullable(),key_id: z.string(),key_metadata: GatewayMetadataSchema,provider_connection_id: z.string(),}).strict(),ingestion_mode: z.enum(["none", "usage", "full"]),+ // Null exactly in "none" mode: nothing is delivered, so no grant is minted.ingestion: z.object({access_token: z.string(),token_type: z.literal("Bearer"),expires_at: z.number().int().positive(),})- .strict(),+ .strict()+ .nullable(),}).strict();export type GatewayResolveResponse = z.infer<diff --git a/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts b/web/src/features/ai-gateway/server/resolve/resolveService.servertest.tsindex 47486b0f89..fc5b442fcb 100644--- a/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts+++ b/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts@@ -5,20 +5,20 @@ import { GatewayApiKeyAuthenticator } from "@/src/features/ai-gateway/server/autimport type { GatewayApiFormat } from "@/src/features/ai-gateway/server/provider";import { GatewayResolveService } from "./resolveService";-vi.mock("@/src/env.mjs", () => ({- env: {- LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],- // Caching is covered against real Redis in the control-plane servertest;- // these cases are about what the service does with a database row.- LANGFUSE_AI_GATEWAY_CACHE_RESOLVE_ENABLED: "false",- LANGFUSE_AI_GATEWAY_JWT_PRIVATE_KEY: "private-key",- LANGFUSE_AI_GATEWAY_JWT_PUBLIC_KEY: "public-key",- LANGFUSE_AI_GATEWAY_JWT_KEY_ID: "key-id",- LANGFUSE_AI_GATEWAY_JWT_ISSUER: "test-issuer",- LANGFUSE_AI_GATEWAY_JWT_AUDIENCE: "test-audience",- },+const mockEnv = vi.hoisted(() => ({+ LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],+ // Caching is covered against real Redis in the control-plane servertest;+ // these cases are about what the service does with a database row.+ LANGFUSE_AI_GATEWAY_CACHE_RESOLVE_ENABLED: "false",+ LANGFUSE_AI_GATEWAY_JWT_PRIVATE_KEY: "private-key" as string | undefined,+ LANGFUSE_AI_GATEWAY_JWT_PUBLIC_KEY: "public-key" as string | undefined,+ LANGFUSE_AI_GATEWAY_JWT_KEY_ID: "key-id" as string | undefined,+ LANGFUSE_AI_GATEWAY_JWT_ISSUER: "test-issuer",+ LANGFUSE_AI_GATEWAY_JWT_AUDIENCE: "test-audience",}));+vi.mock("@/src/env.mjs", () => ({ env: mockEnv }));+vi.mock("@/src/server/utils/jwt", () => ({createEs256JwtSigner: vi.fn(() => ({sign: vi.fn(() => "ingestion-token"),@@ -60,6 +60,7 @@ const serviceWith = (findFirst: () => Promise<unknown>, keyOrgId = "org-1") => {const row = (overrides: {ingestionMode?: string;projectOrgId?: string;+ withoutProject?: boolean;metadata?: unknown;}) => ({id: "key-1",@@ -69,13 +70,15 @@ const row = (overrides: {overrides.metadata === undefined ? null : { metadata: overrides.metadata },organization: {gatewayConfig: {- defaultIngestionProjectId: "project-1",+ defaultIngestionProjectId: overrides.withoutProject ? null : "project-1",ingestionMode: overrides.ingestionMode ?? "USAGE",- defaultIngestionProject: {- id: "project-1",- orgId: overrides.projectOrgId ?? "org-1",- deletedAt: null,- },+ defaultIngestionProject: overrides.withoutProject+ ? null+ : {+ id: "project-1",+ orgId: overrides.projectOrgId ?? "org-1",+ deletedAt: null,+ },},gatewayAiConnections: [{@@ -178,6 +181,53 @@ describe("GatewayResolveService", () => {).rejects.toMatchObject({ status: 403 });});+ it("refuses to resolve USAGE mode without an ingestion project", async () => {+ const service = serviceWith(() =>+ Promise.resolve(row({ withoutProject: true })),+ );++ await expect(+ service.resolve({+ fastHashedSecretKey: "hashed-secret-key",+ apiFormat: "openai.responses",+ }),+ ).rejects.toMatchObject({ status: 403 });+ });++ it("resolves NONE mode without an ingestion project or signing keys", async () => {+ const signingKeys = {+ LANGFUSE_AI_GATEWAY_JWT_PRIVATE_KEY:+ mockEnv.LANGFUSE_AI_GATEWAY_JWT_PRIVATE_KEY,+ LANGFUSE_AI_GATEWAY_JWT_PUBLIC_KEY:+ mockEnv.LANGFUSE_AI_GATEWAY_JWT_PUBLIC_KEY,+ LANGFUSE_AI_GATEWAY_JWT_KEY_ID: mockEnv.LANGFUSE_AI_GATEWAY_JWT_KEY_ID,+ };+ Object.assign(mockEnv, {+ LANGFUSE_AI_GATEWAY_JWT_PRIVATE_KEY: undefined,+ LANGFUSE_AI_GATEWAY_JWT_PUBLIC_KEY: undefined,+ LANGFUSE_AI_GATEWAY_JWT_KEY_ID: undefined,+ });+ try {+ const service = serviceWith(() =>+ Promise.resolve(row({ ingestionMode: "NONE", withoutProject: true })),+ );++ await expect(+ service.resolve({+ fastHashedSecretKey: "hashed-secret-key",+ apiFormat: "openai.responses",+ }),+ ).resolves.toMatchObject({+ connection: { id: "connection-1", provider_model: "gpt-4o" },+ attribution: { organization_id: "org-1", project_id: null },+ ingestion_mode: "none",+ ingestion: null,+ });+ } finally {+ Object.assign(mockEnv, signingKeys);+ }+ });+it("rejects an unknown or expired gateway key", async () => {const service = serviceWith(() => Promise.resolve(null));diff --git a/web/src/features/ai-gateway/server/resolve/resolveService.ts b/web/src/features/ai-gateway/server/resolve/resolveService.tsindex 642b476178..15e86e75e9 100644--- a/web/src/features/ai-gateway/server/resolve/resolveService.ts+++ b/web/src/features/ai-gateway/server/resolve/resolveService.ts@@ -74,8 +74,12 @@ export class GatewayResolveService {params.context.organizationId,);const { context } = params;+ const { ingestionMode, ingestionProjectId } = context;+ // NONE mode delivers nothing to Langfuse, so it needs neither an+ // ingestion project nor ingestion signing keys.+ const deliversToLangfuse = ingestionMode !== "NONE";- if (!context.ingestionProjectId || !context.ingestionMode) {+ if (!ingestionMode || (deliversToLangfuse && !ingestionProjectId)) {throw new GatewayResolveError("Gateway ingestion project is unavailable",403,@@ -112,18 +116,21 @@ export class GatewayResolveService {},attribution: {organization_id: context.organizationId,- project_id: context.ingestionProjectId,+ project_id: ingestionProjectId,key_id: context.apiKeyId,key_metadata: context.keyMetadata,provider_connection_id: context.connection.id,},ingestion_mode:- context.ingestionMode.toLowerCase() as GatewayResolveResponse["ingestion_mode"],- ingestion: this.createIngestionResponse({- mode: context.ingestionMode,- organizationId: context.organizationId,- projectId: context.ingestionProjectId,- }),+ ingestionMode.toLowerCase() as GatewayResolveResponse["ingestion_mode"],+ ingestion:+ deliversToLangfuse && ingestionProjectId+ ? this.createIngestionResponse({+ mode: ingestionMode,+ organizationId: context.organizationId,+ projectId: ingestionProjectId,+ })+ : null,};});}