Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): add self-hosted "None" ingestion mode (#19032)

Tobias Wochingerecbb58d25eb8
diff --git a/ai-gateway/src/capture/mod.rs b/ai-gateway/src/capture/mod.rs
index 612eaead07..eef54ba7eb 100644
--- a/ai-gateway/src/capture/mod.rs
+++ b/ai-gateway/src/capture/mod.rs
@@ -150,7 +150,6 @@ impl ExecutionCapture {
(key.clone(), value)
})
.collect();
- let full = context.ingestion_mode() == IngestionMode::Full;
let span = tracing::Span::current();
// Parsing the whole request is CPU-bound and scales with the body.
let protocol = tracing::info_span!(
@@ -173,7 +172,7 @@ impl ExecutionCapture {
"key_id": attribution.key_id(),
"provider_connection_id": attribution.provider_connection_id(),
"key_metadata": key_metadata,
- "ingestion_mode": if full { "full" } else { "usage" },
+ "ingestion_mode": context.ingestion_mode().as_str(),
}),
provider_model: None,
delivery: None,
diff --git a/ai-gateway/src/providers/mod.rs b/ai-gateway/src/providers/mod.rs
index 426ff66ede..4dc340db7a 100644
--- a/ai-gateway/src/providers/mod.rs
+++ b/ai-gateway/src/providers/mod.rs
@@ -18,7 +18,7 @@ use crate::{
config::ExecutionLimits,
correlation::{RequestCorrelation, UpstreamTarget},
observability::{ErrorOrigin, RequestObservation, Wait, provider_status_reason},
- resolution::{ApiFormat, Provider, ProviderCredential, ResolvedRequestContext},
+ resolution::{ApiFormat, IngestionMode, Provider, ProviderCredential, ResolvedRequestContext},
transport,
};
@@ -196,7 +196,9 @@ impl ProviderTransport {
let provider_model = context.connection().provider_model();
let mut capture = if route.captures_generation() {
let mut capture = ExecutionCapture::for_request(api_format, &context, headers, &body);
- if let Some(telemetry) = &self.telemetry {
+ if let Some(telemetry) = &self.telemetry
+ && context.ingestion_mode() != IngestionMode::None
+ {
capture.deliver_to(telemetry.clone(), &context, headers, correlation);
}
capture
diff --git a/ai-gateway/src/providers/tests.rs b/ai-gateway/src/providers/tests.rs
index f7c52d00dc..acb2cc8c1f 100644
--- a/ai-gateway/src/providers/tests.rs
+++ b/ai-gateway/src/providers/tests.rs
@@ -556,6 +556,42 @@ async fn completed_json_and_sse_upload_once_without_waiting_for_ingestion() {
}
}
+#[tokio::test]
+async fn ingestion_mode_none_relays_without_uploading() {
+ use crate::{
+ resolution::ControlPlaneConfig, telemetry::Telemetry,
+ test_support::resolved_request_context_with_mode,
+ };
+
+ const JSON: &str = r#"{"id":"resp-1","model":"actual","status":"completed","output":[],"usage":{"input_tokens":2,"output_tokens":1,"total_tokens":3}}"#;
+ let sink = FakeServer::start(|_| async { Response::new(Body::from("{}")) }).await;
+ let telemetry =
+ Telemetry::for_test(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap());
+ let upstream = FakeServer::start(|_| async {
+ Response::builder()
+ .header("content-type", "application/json")
+ .body(Body::from(JSON))
+ .unwrap()
+ })
+ .await;
+ let relay = provider(&upstream).with_telemetry(telemetry.clone());
+ let response = relay
+ .forward(
+ relay.admit(),
+ resolved_request_context_with_mode("provider-secret", "none").await,
+ &HeaderMap::new(),
+ Bytes::from_static(br#"{"model":"requested","input":"prompt-canary"}"#),
+ )
+ .await
+ .unwrap();
+ assert_eq!(to_bytes(response.into_body(), 4096).await.unwrap(), JSON);
+ telemetry
+ .shutdown(Instant::now() + Duration::from_secs(1))
+ .await;
+ assert_eq!(upstream.calls(), 1);
+ assert_eq!(sink.calls(), 0);
+}
+
#[tokio::test]
async fn cancelled_and_timed_out_executions_upload_after_provider_context_is_released() {
use crate::{resolution::ControlPlaneConfig, telemetry::Telemetry};
diff --git a/ai-gateway/src/resolution/contracts.rs b/ai-gateway/src/resolution/contracts.rs
index 9b16879d8d..2f50af47ff 100644
--- a/ai-gateway/src/resolution/contracts.rs
+++ b/ai-gateway/src/resolution/contracts.rs
@@ -79,10 +79,22 @@ impl Provider {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum IngestionMode {
+ /// Route only; nothing is delivered to Langfuse.
+ None,
Usage,
Full,
}
+impl IngestionMode {
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Self::None => "none",
+ Self::Usage => "usage",
+ Self::Full => "full",
+ }
+ }
+}
+
#[derive(Clone, Deserialize)]
#[serde(untagged)]
pub enum MetadataValue {
diff --git a/ai-gateway/src/resolution/tests.rs b/ai-gateway/src/resolution/tests.rs
index 5e77af0eea..a19f831ea6 100644
--- a/ai-gateway/src/resolution/tests.rs
+++ b/ai-gateway/src/resolution/tests.rs
@@ -731,7 +731,7 @@ async fn rejects_incompatible_or_incomplete_execution_contexts() {
json!({"nested": {"key": "value"}}),
),
("/attribution/key_metadata", json!({"null": null})),
- ("/ingestion_mode", json!("none")),
+ ("/ingestion_mode", json!("off")),
("/ingestion/access_token", json!("")),
("/ingestion/token_type", json!("Basic")),
("/ingestion/expires_at", json!(NOW)),
diff --git a/packages/shared/prisma/migrations/20261019141500_add_gateway_ingestion_mode_none/migration.sql b/packages/shared/prisma/migrations/20261019141500_add_gateway_ingestion_mode_none/migration.sql
new file mode 100644
index 0000000000..1db6bf3220
--- /dev/null
+++ b/packages/shared/prisma/migrations/20261019141500_add_gateway_ingestion_mode_none/migration.sql
@@ -0,0 +1 @@
+ALTER TYPE "GatewayIngestionMode" ADD VALUE IF NOT EXISTS 'none' BEFORE 'usage';
diff --git a/packages/shared/prisma/schema.prisma b/packages/shared/prisma/schema.prisma
index daa5af6e13..5e3a21a337 100644
--- a/packages/shared/prisma/schema.prisma
+++ b/packages/shared/prisma/schema.prisma
@@ -320,6 +320,7 @@ model RoleAssignment {
}
enum GatewayIngestionMode {
+ NONE @map("none")
USAGE @map("usage")
FULL @map("full")
}
diff --git a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
index 60a368a992..2d91bf2710 100644
--- a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
+++ b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
@@ -832,6 +832,57 @@ describe("AI gateway control plane", () => {
expect(result.ingestion?.expires_at).toBe(ingestionClaims.exp);
});
+ it("resolves NONE ingestion mode on self-hosted and rejects it on cloud", async () => {
+ configureGatewayJwtSigning();
+ const { caller, org, project } = await prepare();
+ const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
+ try {
+ Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined });
+ await caller.aiGateway.updateConfig({
+ orgId: org.id,
+ defaultIngestionProjectId: project.id,
+ ingestionMode: "NONE",
+ });
+ await caller.aiGateway.createConnection({
+ orgId: org.id,
+ name: "OpenAI",
+ provider: "OPENAI",
+ credential: "sk-test-openai",
+ });
+ const gatewayKey = await caller.aiGateway.createApiKey({
+ orgId: org.id,
+ metadata: {},
+ });
+
+ const result = await authenticateRequest({
+ fastHashedSecretKey: createShaHash(gatewayKey.secretKey, env.SALT),
+ apiFormat: "openai.responses",
+ model: "gpt-4o",
+ });
+ expect(result.ingestion_mode).toBe("none");
+
+ Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU" });
+ await expect(
+ caller.aiGateway.updateConfig({
+ orgId: org.id,
+ defaultIngestionProjectId: project.id,
+ ingestionMode: "NONE",
+ }),
+ ).rejects.toThrow("only available on self-hosted deployments");
+ await expect(
+ caller.aiGateway.updateConfig({
+ orgId: org.id,
+ defaultIngestionProjectId: project.id,
+ ingestionMode: "USAGE",
+ }),
+ ).resolves.toMatchObject({ ingestionMode: "USAGE" });
+ } finally {
+ Object.assign(env, {
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: originalCloudRegion,
+ });
+ }
+ });
+
it("routes unprefixed models by provider and rejects unknown models", async () => {
configureGatewayJwtSigning();
const { caller, org, project } = await prepare();
diff --git a/web/src/features/ai-gateway/components/GatewayConfigurationPage/GatewayConfigurationPage.tsx b/web/src/features/ai-gateway/components/GatewayConfigurationPage/GatewayConfigurationPage.tsx
index d47793e5ca..aeb244502e 100644
--- a/web/src/features/ai-gateway/components/GatewayConfigurationPage/GatewayConfigurationPage.tsx
+++ b/web/src/features/ai-gateway/components/GatewayConfigurationPage/GatewayConfigurationPage.tsx
@@ -4,10 +4,17 @@ import Header from "@/src/components/layouts/header";
import { Alert } from "@/src/components/design-system/Alert/Alert";
import { Button } from "@/src/components/ui/button";
import { Skeleton } from "@/src/components/ui/skeleton";
-import { GatewayConfigurationView } from "@/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView";
+import {
+ GatewayConfigurationView,
+ type IngestionMode,
+} from "@/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView";
import { showSuccessToast } from "@/src/features/notifications";
+import { useLangfuseCloudRegion } from "@/src/features/organizations";
import { api, reportNonTrpcError } from "@/src/utils/api";
+const CLOUD_INGESTION_MODES = ["USAGE", "FULL"] as const;
+const SELF_HOSTED_INGESTION_MODES = ["NONE", "USAGE", "FULL"] as const;
+
type Project = {
id: string;
name: string;
@@ -22,6 +29,7 @@ export function GatewayConfigurationPage({
projects: Project[];
}) {
const session = useSession();
+ const { isLangfuseCloud } = useLangfuseCloudRegion();
const configQuery = api.aiGateway.getConfig.useQuery({
orgId: organizationId,
});
@@ -31,7 +39,7 @@ export function GatewayConfigurationPage({
async function saveGatewayConfig(params: {
projectId: string | null;
- ingestionMode: "USAGE" | "FULL";
+ ingestionMode: IngestionMode;
}) {
await updateConfig.mutateAsync({
orgId: organizationId,
@@ -79,6 +87,9 @@ export function GatewayConfigurationPage({
gatewayBaseUrl={gatewayBaseUrl}
initialProjectId={config?.defaultIngestionProjectId ?? null}
initialIngestionMode={config?.ingestionMode ?? "USAGE"}
+ availableIngestionModes={
+ isLangfuseCloud ? CLOUD_INGESTION_MODES : SELF_HOSTED_INGESTION_MODES
+ }
isSaving={createProject.isPending || updateConfig.isPending}
saveError={createProject.isError || updateConfig.isError}
onSave={async ({ projectId, ingestionMode }) => {
diff --git a/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.stories.tsx b/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.stories.tsx
index e3face3286..c9de3a77a6 100644
--- a/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.stories.tsx
+++ b/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.stories.tsx
@@ -17,6 +17,21 @@ export const Enabled = meta.story({
gatewayBaseUrl: "https://gateway.staging.langfuse.com/v1",
initialProjectId: "project-production",
initialIngestionMode: "USAGE",
+ availableIngestionModes: ["USAGE", "FULL"],
+ isSaving: false,
+ saveError: false,
+ onSave: fn(),
+ onCreateProject: fn(),
+ },
+});
+
+export const SelfHostedNoCapture = meta.story({
+ args: {
+ projects,
+ gatewayBaseUrl: "https://gateway.example.com/v1",
+ initialProjectId: "project-production",
+ initialIngestionMode: "NONE",
+ availableIngestionModes: ["NONE", "USAGE", "FULL"],
isSaving: false,
saveError: false,
onSave: fn(),
@@ -30,6 +45,7 @@ export const MissingProjectWarning = meta.story({
gatewayBaseUrl: "https://gateway.staging.langfuse.com/v1",
initialProjectId: "deleted-project",
initialIngestionMode: "FULL",
+ availableIngestionModes: ["USAGE", "FULL"],
isSaving: false,
saveError: false,
onSave: fn(),
diff --git a/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.tsx b/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.tsx
index c17f5f8a90..c7c3d528fe 100644
--- a/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.tsx
+++ b/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.tsx
@@ -2,6 +2,7 @@ import { useState } from "react";
import {
Check,
Copy,
+ EyeOff,
Gauge,
Plus,
ScanText,
@@ -32,7 +33,7 @@ import { DEFAULT_GATEWAY_INGESTION_PROJECT_NAME } from "@/src/features/ai-gatewa
import { useCopyToClipboard } from "@/src/hooks/useCopyToClipboard";
import { cn } from "@/src/utils/tailwind";
-type IngestionMode = "USAGE" | "FULL";
+export type IngestionMode = "NONE" | "USAGE" | "FULL";
type Project = {
id: string;
name: string;
@@ -45,6 +46,12 @@ const ingestionModes: Array<{
description: string;
icon: IconComponent;
}> = [
+ {
+ value: "NONE",
+ title: "None",
+ description: "Route requests without capturing anything in Langfuse.",
+ icon: EyeOff,
+ },
{
value: "USAGE",
title: "Usage",
@@ -64,6 +71,7 @@ export function GatewayConfigurationView({
gatewayBaseUrl,
initialProjectId,
initialIngestionMode,
+ availableIngestionModes,
isSaving,
saveError,
onSave,
@@ -73,6 +81,7 @@ export function GatewayConfigurationView({
gatewayBaseUrl: string;
initialProjectId: string | null;
initialIngestionMode: IngestionMode;
+ availableIngestionModes: readonly IngestionMode[];
isSaving: boolean;
saveError: boolean;
onSave: (values: {
@@ -96,6 +105,9 @@ export function GatewayConfigurationView({
const [ingestionMode, setIngestionMode] =
useState<IngestionMode>(initialIngestionMode);
const projectId = projectSelection ?? null;
+ const visibleIngestionModes = ingestionModes.filter((option) =>
+ availableIngestionModes.includes(option.value),
+ );
const isDirty =
projectId !== initialProjectId || ingestionMode !== initialIngestionMode;
@@ -173,8 +185,15 @@ export function GatewayConfigurationView({
<div>
<CardTitle className="mb-4 text-base">Ingestion mode</CardTitle>
- <div className="grid gap-2 md:grid-cols-2">
- {ingestionModes.map((option) => {
+ <div
+ className={cn(
+ "grid gap-2",
+ visibleIngestionModes.length > 2
+ ? "md:grid-cols-3"
+ : "md:grid-cols-2",
+ )}
+ >
+ {visibleIngestionModes.map((option) => {
const Icon = option.icon;
const selected = ingestionMode === option.value;
return (
diff --git a/web/src/features/ai-gateway/server/provider/registry.ts b/web/src/features/ai-gateway/server/provider/registry.ts
index 8cebca0319..fa08daa629 100644
--- a/web/src/features/ai-gateway/server/provider/registry.ts
+++ b/web/src/features/ai-gateway/server/provider/registry.ts
@@ -51,7 +51,7 @@ export const GatewayResolveResponseSchema = z
provider_connection_id: z.string(),
})
.strict(),
- ingestion_mode: z.enum(["usage", "full"]),
+ ingestion_mode: z.enum(["none", "usage", "full"]),
ingestion: z
.object({
access_token: z.string(),
diff --git a/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.ts b/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.ts
index fc97c44e90..6c81b3e4a7 100644
--- a/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.ts
+++ b/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.ts
@@ -28,7 +28,7 @@ export const CachedResolveContext = z.object({
apiKeyId: z.string(),
keyMetadata: GatewayMetadataSchema,
ingestionProjectId: z.string().nullable(),
- ingestionMode: z.enum(["USAGE", "FULL"]).nullable(),
+ ingestionMode: z.enum(["NONE", "USAGE", "FULL"]).nullable(),
connections: z.array(
z.object({
id: z.string(),
diff --git a/web/src/features/ai-gateway/server/resolve/resolveService.ts b/web/src/features/ai-gateway/server/resolve/resolveService.ts
index b59f0b784f..642b476178 100644
--- a/web/src/features/ai-gateway/server/resolve/resolveService.ts
+++ b/web/src/features/ai-gateway/server/resolve/resolveService.ts
@@ -117,7 +117,8 @@ export class GatewayResolveService {
key_metadata: context.keyMetadata,
provider_connection_id: context.connection.id,
},
- ingestion_mode: context.ingestionMode.toLowerCase() as "usage" | "full",
+ ingestion_mode:
+ context.ingestionMode.toLowerCase() as GatewayResolveResponse["ingestion_mode"],
ingestion: this.createIngestionResponse({
mode: context.ingestionMode,
organizationId: context.organizationId,
@@ -146,7 +147,8 @@ export class GatewayResolveService {
version: 1,
organization_id: params.organizationId,
project_id: params.projectId,
- ingestion_mode: params.mode.toLowerCase() as "usage" | "full",
+ ingestion_mode:
+ params.mode.toLowerCase() as GatewayResolveResponse["ingestion_mode"],
scope: "gateway-ingest",
},
}),
diff --git a/web/src/features/ai-gateway/server/router.ts b/web/src/features/ai-gateway/server/router.ts
index ef941974e4..8871935b8e 100644
--- a/web/src/features/ai-gateway/server/router.ts
+++ b/web/src/features/ai-gateway/server/router.ts
@@ -108,7 +108,14 @@ export const aiGatewayRouter = createTRPCRouter({
.input(
organizationInput.extend({
defaultIngestionProjectId: z.string().nullable(),
- ingestionMode: z.enum(GatewayIngestionMode),
+ ingestionMode: z
+ .enum(GatewayIngestionMode)
+ .refine(
+ (mode) =>
+ mode !== GatewayIngestionMode.NONE ||
+ !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
+ "Ingestion mode NONE is only available on self-hosted deployments",
+ ),
}),
)
.mutation(async ({ input, ctx }) => {