Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(ai-gateway): resolve NONE ingestion mode without an ingestion project or signing keys (#19074)

Tobias Wochinger6b4f9d9b6800
diff --git a/ai-gateway/src/capture/mod.rs b/ai-gateway/src/capture/mod.rs
index eef54ba7eb..d375dce16c 100644
--- a/ai-gateway/src/capture/mod.rs
+++ b/ai-gateway/src/capture/mod.rs
@@ -13,7 +13,7 @@ use tokio::time::Instant;
use crate::{
correlation::RequestCorrelation,
- resolution::{ApiFormat, IngestionMode, MetadataValue, ResolvedRequestContext},
+ resolution::{ApiFormat, IngestionMode, MetadataValue, ProjectId, ResolvedRequestContext},
telemetry,
};
use anthropic_messages::AnthropicMessagesCapture;
@@ -168,7 +168,7 @@ impl ExecutionCapture {
http_status: None,
metadata: json!({
"organization_id": attribution.organization_id().as_str(),
- "project_id": attribution.project_id().as_str(),
+ "project_id": attribution.project_id().map(ProjectId::as_str),
"key_id": attribution.key_id(),
"provider_connection_id": attribution.provider_connection_id(),
"key_metadata": key_metadata,
@@ -190,7 +190,11 @@ impl ExecutionCapture {
.protocol
.as_ref()
.and_then(ProtocolCapture::client_metadata);
- let delivery = telemetry::DeliveryContext::from_resolved(context, headers, client_metadata);
+ let Some(delivery) =
+ telemetry::DeliveryContext::from_resolved(context, headers, client_metadata)
+ else {
+ return;
+ };
correlation.record_generation(delivery.generation_ids());
self.metadata["request_id"] = json!(correlation.id());
if let Some(client_id) = correlation.client_id() {
diff --git a/ai-gateway/src/correlation.rs b/ai-gateway/src/correlation.rs
index 956c54115e..45c442a562 100644
--- a/ai-gateway/src/correlation.rs
+++ b/ai-gateway/src/correlation.rs
@@ -36,7 +36,7 @@ pub(crate) struct GenerationIds {
#[derive(Clone, Debug)]
struct Tenant {
organization_id: OrganizationId,
- project_id: ProjectId,
+ project_id: Option<ProjectId>,
}
/// The provider connection and model of an upstream attempt. With several
@@ -135,11 +135,13 @@ impl RequestCorrelation {
"langfuse.organization.id",
attribution.organization_id().as_str(),
);
- self.server_span
- .record("langfuse.project.id", attribution.project_id().as_str());
+ if let Some(project_id) = attribution.project_id() {
+ self.server_span
+ .record("langfuse.project.id", project_id.as_str());
+ }
self.tenant = Some(Tenant {
organization_id: attribution.organization_id().clone(),
- project_id: attribution.project_id().clone(),
+ project_id: attribution.project_id().cloned(),
});
}
@@ -165,10 +167,10 @@ impl RequestCorrelation {
let Some(tenant) = &self.tenant else {
return;
};
- let mut values = vec![
- (ORGANIZATION_ID_HEADER, tenant.organization_id.as_str()),
- (PROJECT_ID_HEADER, tenant.project_id.as_str()),
- ];
+ let mut values = vec![(ORGANIZATION_ID_HEADER, tenant.organization_id.as_str())];
+ if let Some(project_id) = &tenant.project_id {
+ values.push((PROJECT_ID_HEADER, project_id.as_str()));
+ }
if let Some(upstream) = &self.upstream {
values.push((PROVIDER_HEADER, upstream.provider));
values.push((PROVIDER_CONNECTION_ID_HEADER, &upstream.connection_id));
diff --git a/ai-gateway/src/resolution/contracts.rs b/ai-gateway/src/resolution/contracts.rs
index 2f50af47ff..4557eac60e 100644
--- a/ai-gateway/src/resolution/contracts.rs
+++ b/ai-gateway/src/resolution/contracts.rs
@@ -241,7 +241,8 @@ impl From<&str> for ProjectId {
#[serde(deny_unknown_fields)]
pub struct RequestAttribution {
organization_id: OrganizationId,
- project_id: ProjectId,
+ /// Absent only in [`IngestionMode::None`], which needs no ingestion project.
+ project_id: Option<ProjectId>,
key_id: String,
key_metadata: BTreeMap<String, MetadataValue>,
provider_connection_id: String,
@@ -251,8 +252,8 @@ impl RequestAttribution {
pub fn organization_id(&self) -> &OrganizationId {
&self.organization_id
}
- pub fn project_id(&self) -> &ProjectId {
- &self.project_id
+ pub fn project_id(&self) -> Option<&ProjectId> {
+ self.project_id.as_ref()
}
pub fn key_id(&self) -> &str {
&self.key_id
@@ -291,7 +292,8 @@ struct ResolutionResponse {
attribution: RequestAttribution,
#[serde(deserialize_with = "string_enum")]
ingestion_mode: IngestionMode,
- ingestion: IngestionGrant,
+ /// Absent only in [`IngestionMode::None`]; nothing is delivered without it.
+ ingestion: Option<IngestionGrant>,
}
/// A validated v1 execution. Construction is restricted to successful resolution.
@@ -314,8 +316,19 @@ impl ResolvedRequestContext {
pub fn ingestion_mode(&self) -> IngestionMode {
self.0.ingestion_mode
}
- pub fn ingestion(&self) -> &IngestionGrant {
- &self.0.ingestion
+ pub fn ingestion(&self) -> Option<&IngestionGrant> {
+ self.0.ingestion.as_ref()
+ }
+ /// The project and grant telemetry is delivered with. Resolution guarantees both
+ /// in every mode but [`IngestionMode::None`], which never delivers.
+ pub fn delivery_target(&self) -> Option<(&ProjectId, &IngestionGrant)> {
+ if self.0.ingestion_mode == IngestionMode::None {
+ return None;
+ }
+ Some((
+ self.0.attribution.project_id.as_ref()?,
+ self.0.ingestion.as_ref()?,
+ ))
}
}
@@ -348,10 +361,24 @@ fn ingestion_claims_match(token: &str, attribution: &RequestAttribution) -> bool
.and_then(|bytes| serde_json::from_slice::<IngestionClaims>(&bytes).ok())
.is_some_and(|claims| {
claims.organization_id == attribution.organization_id
- && claims.project_id == attribution.project_id
+ && Some(&claims.project_id) == attribution.project_id.as_ref()
})
}
+/// A grant must be valid and bound to the attributed tenant whenever it is sent;
+/// only [`IngestionMode::None`] may omit it.
+fn valid_ingestion(response: &ResolutionResponse, now: u64) -> bool {
+ match &response.ingestion {
+ None => response.ingestion_mode == IngestionMode::None,
+ Some(grant) => {
+ let token = grant.access_token.expose_secret();
+ valid_token(token)
+ && ingestion_claims_match(token, &response.attribution)
+ && grant.expires_at > now
+ }
+ }
+}
+
pub(super) fn decode(
bytes: &[u8],
expected_format: ApiFormat,
@@ -368,17 +395,15 @@ pub(super) fn decode(
|| connection.base_url != provider.official_origin()
|| !provider.accepts(connection.credential())
|| !valid_token(connection.credential().secret())
- || !valid_token(response.ingestion.access_token.expose_secret())
- || !ingestion_claims_match(response.ingestion.access_token.expose_secret(), attribution)
- || response.ingestion.expires_at <= now
+ || !valid_ingestion(&response, now)
|| [
connection.id.as_str(),
attribution.organization_id.as_str(),
- attribution.project_id.as_str(),
attribution.key_id.as_str(),
attribution.provider_connection_id.as_str(),
]
- .iter()
+ .into_iter()
+ .chain(attribution.project_id.as_ref().map(ProjectId::as_str))
.any(|value| value.trim().is_empty())
|| connection.provider_model.trim().is_empty()
{
diff --git a/ai-gateway/src/resolution/tests.rs b/ai-gateway/src/resolution/tests.rs
index a19f831ea6..994cf62fae 100644
--- a/ai-gateway/src/resolution/tests.rs
+++ b/ai-gateway/src/resolution/tests.rs
@@ -149,13 +149,16 @@ async fn signs_the_exact_key_and_sends_api_format_and_model() {
context.connection().credential(),
ProviderCredential::Bearer("private-provider-token")
));
- assert_eq!(context.attribution().project_id().as_str(), "project-1");
+ assert_eq!(
+ context.attribution().project_id().map(ProjectId::as_str),
+ Some("project-1")
+ );
assert_eq!(
context.attribution().provider_connection_id(),
"connection-1"
);
assert_eq!(
- context.ingestion().access_token().expose_secret(),
+ context.ingestion().unwrap().access_token().expose_secret(),
ingestion_token("org-1", "project-1")
);
let debug = format!("{context:?}");
@@ -516,12 +519,18 @@ async fn concurrent_resolutions_keep_credentials_and_contexts_isolated() {
);
let alice = alice.unwrap();
let bob = bob.unwrap();
- assert_eq!(alice.attribution().project_id().as_str(), "project-alice");
+ assert_eq!(
+ alice.attribution().project_id().map(ProjectId::as_str),
+ Some("project-alice")
+ );
assert!(matches!(
alice.connection().credential(),
ProviderCredential::Bearer("token-alice")
));
- assert_eq!(bob.attribution().project_id().as_str(), "project-bob");
+ assert_eq!(
+ bob.attribution().project_id().map(ProjectId::as_str),
+ Some("project-bob")
+ );
assert!(matches!(
bob.connection().credential(),
ProviderCredential::Bearer("token-bob")
@@ -807,6 +816,53 @@ async fn rejects_ingestion_tokens_that_authorize_a_different_tenant() {
);
}
+#[tokio::test]
+async fn resolves_none_mode_without_an_ingestion_grant_or_project() {
+ let mut body = success("project-1", "provider-token");
+ body["ingestion_mode"] = json!("none");
+ body["ingestion"] = Value::Null;
+ body["attribution"]["project_id"] = Value::Null;
+ let web = FakeWeb::start(move |_| {
+ let body = body.clone();
+ async move { response(body) }
+ })
+ .await;
+ let context = web
+ .control_plane()
+ .resolve_at(
+ GATEWAY_KEY,
+ ApiFormat::OpenAiResponses,
+ "primary:gpt-4.1",
+ NOW_TIME,
+ REQUEST_ID,
+ )
+ .await
+ .unwrap();
+ assert_eq!(context.ingestion_mode(), IngestionMode::None);
+ assert!(context.ingestion().is_none());
+ assert!(context.attribution().project_id().is_none());
+}
+
+#[tokio::test]
+async fn delivering_modes_require_an_ingestion_grant_and_project() {
+ for mode in ["usage", "full"] {
+ let mut body = success("project-1", "provider-token");
+ body["ingestion_mode"] = json!(mode);
+ body["ingestion"] = Value::Null;
+ assert_invalid_context(body).await;
+
+ let mut body = success("project-1", "provider-token");
+ body["ingestion_mode"] = json!(mode);
+ body["attribution"]["project_id"] = Value::Null;
+ assert_invalid_context(body).await;
+ }
+ // A grant sent alongside NONE must still be bound to the attributed tenant.
+ let mut body = success("project-1", "provider-token");
+ body["ingestion_mode"] = json!("none");
+ body["attribution"]["project_id"] = json!("project-2");
+ assert_invalid_context(body).await;
+}
+
async fn assert_invalid_context(body: Value) {
assert_invalid_context_for(body, ApiFormat::OpenAiResponses).await;
}
diff --git a/ai-gateway/src/telemetry/mod.rs b/ai-gateway/src/telemetry/mod.rs
index 6de1aca06e..8e3b75a4c0 100644
--- a/ai-gateway/src/telemetry/mod.rs
+++ b/ai-gateway/src/telemetry/mod.rs
@@ -68,19 +68,21 @@ pub(crate) struct DeliveryContext {
}
impl DeliveryContext {
+ /// `None` when the request has nothing to deliver to, as in ingestion mode `none`.
pub(crate) fn from_resolved(
context: &ResolvedRequestContext,
headers: &HeaderMap,
client_metadata: Option<&Map<String, Value>>,
- ) -> Self {
- Self {
+ ) -> Option<Self> {
+ let (project_id, ingestion) = context.delivery_target()?;
+ Some(Self {
grant: Grant {
- project_id: context.attribution().project_id().clone(),
- access_token: context.ingestion().access_token().clone(),
- expires_at: context.ingestion().expires_at(),
+ project_id: project_id.clone(),
+ access_token: ingestion.access_token().clone(),
+ expires_at: ingestion.expires_at(),
},
generation: GenerationContext::from_request(headers, client_metadata),
- }
+ })
}
pub(crate) fn generation_ids(&self) -> crate::correlation::GenerationIds {
diff --git a/ai-gateway/src/telemetry/tests.rs b/ai-gateway/src/telemetry/tests.rs
index 77f0fd79ea..c03509d622 100644
--- a/ai-gateway/src/telemetry/tests.rs
+++ b/ai-gateway/src/telemetry/tests.rs
@@ -39,6 +39,7 @@ async fn grant() -> DeliveryContext {
&HeaderMap::new(),
None,
)
+ .unwrap()
}
fn uploader(url: &str) -> Uploader {
@@ -642,7 +643,8 @@ async fn retained_byte_budget_is_released_after_successful_drain() {
&resolved_request_context("provider-secret").await,
&HeaderMap::new(),
None,
- );
+ )
+ .unwrap();
telemetry.record(second, facts("project-1"));
assert_eq!(telemetry.0.stats.dropped.load(Ordering::Relaxed), 1);
release.notify_one();
diff --git a/ai-gateway/src/test_support.rs b/ai-gateway/src/test_support.rs
index c210f8ca65..1ca1633aea 100644
--- a/ai-gateway/src/test_support.rs
+++ b/ai-gateway/src/test_support.rs
@@ -165,6 +165,11 @@ pub(crate) async fn resolved_request_context_for(
.unwrap();
let mut body: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
body["ingestion_mode"] = mode.into();
+ if mode == "none" {
+ // Web mints no grant and may have no ingestion project in this mode.
+ body["ingestion"] = serde_json::Value::Null;
+ body["attribution"]["project_id"] = serde_json::Value::Null;
+ }
Response::builder()
.header("content-type", "application/json")
.body(Body::from(body.to_string()))
diff --git a/web/src/__e2e__/ai-gateway.gatewaye2e.ts b/web/src/__e2e__/ai-gateway.gatewaye2e.ts
index 8fa3d7e356..8bc63ebe76 100644
--- a/web/src/__e2e__/ai-gateway.gatewaye2e.ts
+++ b/web/src/__e2e__/ai-gateway.gatewaye2e.ts
@@ -620,7 +620,7 @@ describe("AI gateway live end-to-end", () => {
expires_at: expect.any(Number),
});
- const ingestionToken = resolved.ingestion.access_token;
+ const ingestionToken = resolved.ingestion!.access_token;
await ingestTrace(ingestionToken, { provider, apiFormat });
},
);
diff --git a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
index 2d91bf2710..947ca90b54 100644
--- a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
+++ b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
@@ -860,6 +860,8 @@ describe("AI gateway control plane", () => {
model: "gpt-4o",
});
expect(result.ingestion_mode).toBe("none");
+ expect(result.ingestion).toBeNull();
+ expect(result.attribution.project_id).toBe(project.id);
Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU" });
await expect(
diff --git a/web/src/features/ai-gateway/server/provider/registry.ts b/web/src/features/ai-gateway/server/provider/registry.ts
index fa08daa629..cd124c63ae 100644
--- a/web/src/features/ai-gateway/server/provider/registry.ts
+++ b/web/src/features/ai-gateway/server/provider/registry.ts
@@ -45,20 +45,23 @@ export const GatewayResolveResponseSchema = z
attribution: z
.object({
organization_id: z.string(),
- project_id: z.string(),
+ // Null only in "none" mode, which needs no ingestion project.
+ project_id: z.string().nullable(),
key_id: z.string(),
key_metadata: GatewayMetadataSchema,
provider_connection_id: z.string(),
})
.strict(),
ingestion_mode: z.enum(["none", "usage", "full"]),
+ // Null exactly in "none" mode: nothing is delivered, so no grant is minted.
ingestion: z
.object({
access_token: z.string(),
token_type: z.literal("Bearer"),
expires_at: z.number().int().positive(),
})
- .strict(),
+ .strict()
+ .nullable(),
})
.strict();
export type GatewayResolveResponse = z.infer<
diff --git a/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts b/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts
index 47486b0f89..fc5b442fcb 100644
--- a/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts
+++ b/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts
@@ -5,20 +5,20 @@ import { GatewayApiKeyAuthenticator } from "@/src/features/ai-gateway/server/aut
import type { GatewayApiFormat } from "@/src/features/ai-gateway/server/provider";
import { GatewayResolveService } from "./resolveService";
-vi.mock("@/src/env.mjs", () => ({
- env: {
- LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],
- // Caching is covered against real Redis in the control-plane servertest;
- // these cases are about what the service does with a database row.
- LANGFUSE_AI_GATEWAY_CACHE_RESOLVE_ENABLED: "false",
- LANGFUSE_AI_GATEWAY_JWT_PRIVATE_KEY: "private-key",
- LANGFUSE_AI_GATEWAY_JWT_PUBLIC_KEY: "public-key",
- LANGFUSE_AI_GATEWAY_JWT_KEY_ID: "key-id",
- LANGFUSE_AI_GATEWAY_JWT_ISSUER: "test-issuer",
- LANGFUSE_AI_GATEWAY_JWT_AUDIENCE: "test-audience",
- },
+const mockEnv = vi.hoisted(() => ({
+ LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],
+ // Caching is covered against real Redis in the control-plane servertest;
+ // these cases are about what the service does with a database row.
+ LANGFUSE_AI_GATEWAY_CACHE_RESOLVE_ENABLED: "false",
+ LANGFUSE_AI_GATEWAY_JWT_PRIVATE_KEY: "private-key" as string | undefined,
+ LANGFUSE_AI_GATEWAY_JWT_PUBLIC_KEY: "public-key" as string | undefined,
+ LANGFUSE_AI_GATEWAY_JWT_KEY_ID: "key-id" as string | undefined,
+ LANGFUSE_AI_GATEWAY_JWT_ISSUER: "test-issuer",
+ LANGFUSE_AI_GATEWAY_JWT_AUDIENCE: "test-audience",
}));
+vi.mock("@/src/env.mjs", () => ({ env: mockEnv }));
+
vi.mock("@/src/server/utils/jwt", () => ({
createEs256JwtSigner: vi.fn(() => ({
sign: vi.fn(() => "ingestion-token"),
@@ -60,6 +60,7 @@ const serviceWith = (findFirst: () => Promise<unknown>, keyOrgId = "org-1") => {
const row = (overrides: {
ingestionMode?: string;
projectOrgId?: string;
+ withoutProject?: boolean;
metadata?: unknown;
}) => ({
id: "key-1",
@@ -69,13 +70,15 @@ const row = (overrides: {
overrides.metadata === undefined ? null : { metadata: overrides.metadata },
organization: {
gatewayConfig: {
- defaultIngestionProjectId: "project-1",
+ defaultIngestionProjectId: overrides.withoutProject ? null : "project-1",
ingestionMode: overrides.ingestionMode ?? "USAGE",
- defaultIngestionProject: {
- id: "project-1",
- orgId: overrides.projectOrgId ?? "org-1",
- deletedAt: null,
- },
+ defaultIngestionProject: overrides.withoutProject
+ ? null
+ : {
+ id: "project-1",
+ orgId: overrides.projectOrgId ?? "org-1",
+ deletedAt: null,
+ },
},
gatewayAiConnections: [
{
@@ -178,6 +181,53 @@ describe("GatewayResolveService", () => {
).rejects.toMatchObject({ status: 403 });
});
+ it("refuses to resolve USAGE mode without an ingestion project", async () => {
+ const service = serviceWith(() =>
+ Promise.resolve(row({ withoutProject: true })),
+ );
+
+ await expect(
+ service.resolve({
+ fastHashedSecretKey: "hashed-secret-key",
+ apiFormat: "openai.responses",
+ }),
+ ).rejects.toMatchObject({ status: 403 });
+ });
+
+ it("resolves NONE mode without an ingestion project or signing keys", async () => {
+ const signingKeys = {
+ LANGFUSE_AI_GATEWAY_JWT_PRIVATE_KEY:
+ mockEnv.LANGFUSE_AI_GATEWAY_JWT_PRIVATE_KEY,
+ LANGFUSE_AI_GATEWAY_JWT_PUBLIC_KEY:
+ mockEnv.LANGFUSE_AI_GATEWAY_JWT_PUBLIC_KEY,
+ LANGFUSE_AI_GATEWAY_JWT_KEY_ID: mockEnv.LANGFUSE_AI_GATEWAY_JWT_KEY_ID,
+ };
+ Object.assign(mockEnv, {
+ LANGFUSE_AI_GATEWAY_JWT_PRIVATE_KEY: undefined,
+ LANGFUSE_AI_GATEWAY_JWT_PUBLIC_KEY: undefined,
+ LANGFUSE_AI_GATEWAY_JWT_KEY_ID: undefined,
+ });
+ try {
+ const service = serviceWith(() =>
+ Promise.resolve(row({ ingestionMode: "NONE", withoutProject: true })),
+ );
+
+ await expect(
+ service.resolve({
+ fastHashedSecretKey: "hashed-secret-key",
+ apiFormat: "openai.responses",
+ }),
+ ).resolves.toMatchObject({
+ connection: { id: "connection-1", provider_model: "gpt-4o" },
+ attribution: { organization_id: "org-1", project_id: null },
+ ingestion_mode: "none",
+ ingestion: null,
+ });
+ } finally {
+ Object.assign(mockEnv, signingKeys);
+ }
+ });
+
it("rejects an unknown or expired gateway key", async () => {
const service = serviceWith(() => Promise.resolve(null));
diff --git a/web/src/features/ai-gateway/server/resolve/resolveService.ts b/web/src/features/ai-gateway/server/resolve/resolveService.ts
index 642b476178..15e86e75e9 100644
--- a/web/src/features/ai-gateway/server/resolve/resolveService.ts
+++ b/web/src/features/ai-gateway/server/resolve/resolveService.ts
@@ -74,8 +74,12 @@ export class GatewayResolveService {
params.context.organizationId,
);
const { context } = params;
+ const { ingestionMode, ingestionProjectId } = context;
+ // NONE mode delivers nothing to Langfuse, so it needs neither an
+ // ingestion project nor ingestion signing keys.
+ const deliversToLangfuse = ingestionMode !== "NONE";
- if (!context.ingestionProjectId || !context.ingestionMode) {
+ if (!ingestionMode || (deliversToLangfuse && !ingestionProjectId)) {
throw new GatewayResolveError(
"Gateway ingestion project is unavailable",
403,
@@ -112,18 +116,21 @@ export class GatewayResolveService {
},
attribution: {
organization_id: context.organizationId,
- project_id: context.ingestionProjectId,
+ project_id: ingestionProjectId,
key_id: context.apiKeyId,
key_metadata: context.keyMetadata,
provider_connection_id: context.connection.id,
},
ingestion_mode:
- context.ingestionMode.toLowerCase() as GatewayResolveResponse["ingestion_mode"],
- ingestion: this.createIngestionResponse({
- mode: context.ingestionMode,
- organizationId: context.organizationId,
- projectId: context.ingestionProjectId,
- }),
+ ingestionMode.toLowerCase() as GatewayResolveResponse["ingestion_mode"],
+ ingestion:
+ deliversToLangfuse && ingestionProjectId
+ ? this.createIngestionResponse({
+ mode: ingestionMode,
+ organizationId: context.organizationId,
+ projectId: ingestionProjectId,
+ })
+ : null,
};
});
}