🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(ai-gateway): add self-hosted "None" ingestion mode (#19032)
Tobias Wochingerecbb58d25eb8
diff --git a/ai-gateway/src/capture/mod.rs b/ai-gateway/src/capture/mod.rsindex 612eaead07..eef54ba7eb 100644--- a/ai-gateway/src/capture/mod.rs+++ b/ai-gateway/src/capture/mod.rs@@ -150,7 +150,6 @@ impl ExecutionCapture {(key.clone(), value)}).collect();- let full = context.ingestion_mode() == IngestionMode::Full;let span = tracing::Span::current();// Parsing the whole request is CPU-bound and scales with the body.let protocol = tracing::info_span!(@@ -173,7 +172,7 @@ impl ExecutionCapture {"key_id": attribution.key_id(),"provider_connection_id": attribution.provider_connection_id(),"key_metadata": key_metadata,- "ingestion_mode": if full { "full" } else { "usage" },+ "ingestion_mode": context.ingestion_mode().as_str(),}),provider_model: None,delivery: None,diff --git a/ai-gateway/src/providers/mod.rs b/ai-gateway/src/providers/mod.rsindex 426ff66ede..4dc340db7a 100644--- a/ai-gateway/src/providers/mod.rs+++ b/ai-gateway/src/providers/mod.rs@@ -18,7 +18,7 @@ use crate::{config::ExecutionLimits,correlation::{RequestCorrelation, UpstreamTarget},observability::{ErrorOrigin, RequestObservation, Wait, provider_status_reason},- resolution::{ApiFormat, Provider, ProviderCredential, ResolvedRequestContext},+ resolution::{ApiFormat, IngestionMode, Provider, ProviderCredential, ResolvedRequestContext},transport,};@@ -196,7 +196,9 @@ impl ProviderTransport {let provider_model = context.connection().provider_model();let mut capture = if route.captures_generation() {let mut capture = ExecutionCapture::for_request(api_format, &context, headers, &body);- if let Some(telemetry) = &self.telemetry {+ if let Some(telemetry) = &self.telemetry+ && context.ingestion_mode() != IngestionMode::None+ {capture.deliver_to(telemetry.clone(), &context, headers, correlation);}capturediff --git a/ai-gateway/src/providers/tests.rs b/ai-gateway/src/providers/tests.rsindex f7c52d00dc..acb2cc8c1f 100644--- a/ai-gateway/src/providers/tests.rs+++ b/ai-gateway/src/providers/tests.rs@@ -556,6 +556,42 @@ async fn completed_json_and_sse_upload_once_without_waiting_for_ingestion() {}}+#[tokio::test]+async fn ingestion_mode_none_relays_without_uploading() {+ use crate::{+ resolution::ControlPlaneConfig, telemetry::Telemetry,+ test_support::resolved_request_context_with_mode,+ };++ const JSON: &str = r#"{"id":"resp-1","model":"actual","status":"completed","output":[],"usage":{"input_tokens":2,"output_tokens":1,"total_tokens":3}}"#;+ let sink = FakeServer::start(|_| async { Response::new(Body::from("{}")) }).await;+ let telemetry =+ Telemetry::for_test(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap());+ let upstream = FakeServer::start(|_| async {+ Response::builder()+ .header("content-type", "application/json")+ .body(Body::from(JSON))+ .unwrap()+ })+ .await;+ let relay = provider(&upstream).with_telemetry(telemetry.clone());+ let response = relay+ .forward(+ relay.admit(),+ resolved_request_context_with_mode("provider-secret", "none").await,+ &HeaderMap::new(),+ Bytes::from_static(br#"{"model":"requested","input":"prompt-canary"}"#),+ )+ .await+ .unwrap();+ assert_eq!(to_bytes(response.into_body(), 4096).await.unwrap(), JSON);+ telemetry+ .shutdown(Instant::now() + Duration::from_secs(1))+ .await;+ assert_eq!(upstream.calls(), 1);+ assert_eq!(sink.calls(), 0);+}+#[tokio::test]async fn cancelled_and_timed_out_executions_upload_after_provider_context_is_released() {use crate::{resolution::ControlPlaneConfig, telemetry::Telemetry};diff --git a/ai-gateway/src/resolution/contracts.rs b/ai-gateway/src/resolution/contracts.rsindex 9b16879d8d..2f50af47ff 100644--- a/ai-gateway/src/resolution/contracts.rs+++ b/ai-gateway/src/resolution/contracts.rs@@ -79,10 +79,22 @@ impl Provider {#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]#[serde(rename_all = "lowercase")]pub enum IngestionMode {+ /// Route only; nothing is delivered to Langfuse.+ None,Usage,Full,}+impl IngestionMode {+ pub fn as_str(self) -> &'static str {+ match self {+ Self::None => "none",+ Self::Usage => "usage",+ Self::Full => "full",+ }+ }+}+#[derive(Clone, Deserialize)]#[serde(untagged)]pub enum MetadataValue {diff --git a/ai-gateway/src/resolution/tests.rs b/ai-gateway/src/resolution/tests.rsindex 5e77af0eea..a19f831ea6 100644--- a/ai-gateway/src/resolution/tests.rs+++ b/ai-gateway/src/resolution/tests.rs@@ -731,7 +731,7 @@ async fn rejects_incompatible_or_incomplete_execution_contexts() {json!({"nested": {"key": "value"}}),),("/attribution/key_metadata", json!({"null": null})),- ("/ingestion_mode", json!("none")),+ ("/ingestion_mode", json!("off")),("/ingestion/access_token", json!("")),("/ingestion/token_type", json!("Basic")),("/ingestion/expires_at", json!(NOW)),diff --git a/packages/shared/prisma/migrations/20261019141500_add_gateway_ingestion_mode_none/migration.sql b/packages/shared/prisma/migrations/20261019141500_add_gateway_ingestion_mode_none/migration.sqlnew file mode 100644index 0000000000..1db6bf3220--- /dev/null+++ b/packages/shared/prisma/migrations/20261019141500_add_gateway_ingestion_mode_none/migration.sql@@ -0,0 +1 @@+ALTER TYPE "GatewayIngestionMode" ADD VALUE IF NOT EXISTS 'none' BEFORE 'usage';diff --git a/packages/shared/prisma/schema.prisma b/packages/shared/prisma/schema.prismaindex daa5af6e13..5e3a21a337 100644--- a/packages/shared/prisma/schema.prisma+++ b/packages/shared/prisma/schema.prisma@@ -320,6 +320,7 @@ model RoleAssignment {}enum GatewayIngestionMode {+ NONE @map("none")USAGE @map("usage")FULL @map("full")}diff --git a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts b/web/src/__tests__/server/ai-gateway-control-plane.servertest.tsindex 60a368a992..2d91bf2710 100644--- a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts+++ b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts@@ -832,6 +832,57 @@ describe("AI gateway control plane", () => {expect(result.ingestion?.expires_at).toBe(ingestionClaims.exp);});+ it("resolves NONE ingestion mode on self-hosted and rejects it on cloud", async () => {+ configureGatewayJwtSigning();+ const { caller, org, project } = await prepare();+ const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;+ try {+ Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined });+ await caller.aiGateway.updateConfig({+ orgId: org.id,+ defaultIngestionProjectId: project.id,+ ingestionMode: "NONE",+ });+ await caller.aiGateway.createConnection({+ orgId: org.id,+ name: "OpenAI",+ provider: "OPENAI",+ credential: "sk-test-openai",+ });+ const gatewayKey = await caller.aiGateway.createApiKey({+ orgId: org.id,+ metadata: {},+ });++ const result = await authenticateRequest({+ fastHashedSecretKey: createShaHash(gatewayKey.secretKey, env.SALT),+ apiFormat: "openai.responses",+ model: "gpt-4o",+ });+ expect(result.ingestion_mode).toBe("none");++ Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU" });+ await expect(+ caller.aiGateway.updateConfig({+ orgId: org.id,+ defaultIngestionProjectId: project.id,+ ingestionMode: "NONE",+ }),+ ).rejects.toThrow("only available on self-hosted deployments");+ await expect(+ caller.aiGateway.updateConfig({+ orgId: org.id,+ defaultIngestionProjectId: project.id,+ ingestionMode: "USAGE",+ }),+ ).resolves.toMatchObject({ ingestionMode: "USAGE" });+ } finally {+ Object.assign(env, {+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: originalCloudRegion,+ });+ }+ });+it("routes unprefixed models by provider and rejects unknown models", async () => {configureGatewayJwtSigning();const { caller, org, project } = await prepare();diff --git a/web/src/features/ai-gateway/components/GatewayConfigurationPage/GatewayConfigurationPage.tsx b/web/src/features/ai-gateway/components/GatewayConfigurationPage/GatewayConfigurationPage.tsxindex d47793e5ca..aeb244502e 100644--- a/web/src/features/ai-gateway/components/GatewayConfigurationPage/GatewayConfigurationPage.tsx+++ b/web/src/features/ai-gateway/components/GatewayConfigurationPage/GatewayConfigurationPage.tsx@@ -4,10 +4,17 @@ import Header from "@/src/components/layouts/header";import { Alert } from "@/src/components/design-system/Alert/Alert";import { Button } from "@/src/components/ui/button";import { Skeleton } from "@/src/components/ui/skeleton";-import { GatewayConfigurationView } from "@/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView";+import {+ GatewayConfigurationView,+ type IngestionMode,+} from "@/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView";import { showSuccessToast } from "@/src/features/notifications";+import { useLangfuseCloudRegion } from "@/src/features/organizations";import { api, reportNonTrpcError } from "@/src/utils/api";+const CLOUD_INGESTION_MODES = ["USAGE", "FULL"] as const;+const SELF_HOSTED_INGESTION_MODES = ["NONE", "USAGE", "FULL"] as const;+type Project = {id: string;name: string;@@ -22,6 +29,7 @@ export function GatewayConfigurationPage({projects: Project[];}) {const session = useSession();+ const { isLangfuseCloud } = useLangfuseCloudRegion();const configQuery = api.aiGateway.getConfig.useQuery({orgId: organizationId,});@@ -31,7 +39,7 @@ export function GatewayConfigurationPage({async function saveGatewayConfig(params: {projectId: string | null;- ingestionMode: "USAGE" | "FULL";+ ingestionMode: IngestionMode;}) {await updateConfig.mutateAsync({orgId: organizationId,@@ -79,6 +87,9 @@ export function GatewayConfigurationPage({gatewayBaseUrl={gatewayBaseUrl}initialProjectId={config?.defaultIngestionProjectId ?? null}initialIngestionMode={config?.ingestionMode ?? "USAGE"}+ availableIngestionModes={+ isLangfuseCloud ? CLOUD_INGESTION_MODES : SELF_HOSTED_INGESTION_MODES+ }isSaving={createProject.isPending || updateConfig.isPending}saveError={createProject.isError || updateConfig.isError}onSave={async ({ projectId, ingestionMode }) => {diff --git a/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.stories.tsx b/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.stories.tsxindex e3face3286..c9de3a77a6 100644--- a/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.stories.tsx+++ b/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.stories.tsx@@ -17,6 +17,21 @@ export const Enabled = meta.story({gatewayBaseUrl: "https://gateway.staging.langfuse.com/v1",initialProjectId: "project-production",initialIngestionMode: "USAGE",+ availableIngestionModes: ["USAGE", "FULL"],+ isSaving: false,+ saveError: false,+ onSave: fn(),+ onCreateProject: fn(),+ },+});++export const SelfHostedNoCapture = meta.story({+ args: {+ projects,+ gatewayBaseUrl: "https://gateway.example.com/v1",+ initialProjectId: "project-production",+ initialIngestionMode: "NONE",+ availableIngestionModes: ["NONE", "USAGE", "FULL"],isSaving: false,saveError: false,onSave: fn(),@@ -30,6 +45,7 @@ export const MissingProjectWarning = meta.story({gatewayBaseUrl: "https://gateway.staging.langfuse.com/v1",initialProjectId: "deleted-project",initialIngestionMode: "FULL",+ availableIngestionModes: ["USAGE", "FULL"],isSaving: false,saveError: false,onSave: fn(),diff --git a/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.tsx b/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.tsxindex c17f5f8a90..c7c3d528fe 100644--- a/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.tsx+++ b/web/src/features/ai-gateway/components/GatewayConfigurationPage/components/GatewayConfigurationView/GatewayConfigurationView.tsx@@ -2,6 +2,7 @@ import { useState } from "react";import {Check,Copy,+ EyeOff,Gauge,Plus,ScanText,@@ -32,7 +33,7 @@ import { DEFAULT_GATEWAY_INGESTION_PROJECT_NAME } from "@/src/features/ai-gatewaimport { useCopyToClipboard } from "@/src/hooks/useCopyToClipboard";import { cn } from "@/src/utils/tailwind";-type IngestionMode = "USAGE" | "FULL";+export type IngestionMode = "NONE" | "USAGE" | "FULL";type Project = {id: string;name: string;@@ -45,6 +46,12 @@ const ingestionModes: Array<{description: string;icon: IconComponent;}> = [+ {+ value: "NONE",+ title: "None",+ description: "Route requests without capturing anything in Langfuse.",+ icon: EyeOff,+ },{value: "USAGE",title: "Usage",@@ -64,6 +71,7 @@ export function GatewayConfigurationView({gatewayBaseUrl,initialProjectId,initialIngestionMode,+ availableIngestionModes,isSaving,saveError,onSave,@@ -73,6 +81,7 @@ export function GatewayConfigurationView({gatewayBaseUrl: string;initialProjectId: string | null;initialIngestionMode: IngestionMode;+ availableIngestionModes: readonly IngestionMode[];isSaving: boolean;saveError: boolean;onSave: (values: {@@ -96,6 +105,9 @@ export function GatewayConfigurationView({const [ingestionMode, setIngestionMode] =useState<IngestionMode>(initialIngestionMode);const projectId = projectSelection ?? null;+ const visibleIngestionModes = ingestionModes.filter((option) =>+ availableIngestionModes.includes(option.value),+ );const isDirty =projectId !== initialProjectId || ingestionMode !== initialIngestionMode;@@ -173,8 +185,15 @@ export function GatewayConfigurationView({<div><CardTitle className="mb-4 text-base">Ingestion mode</CardTitle>- <div className="grid gap-2 md:grid-cols-2">- {ingestionModes.map((option) => {+ <div+ className={cn(+ "grid gap-2",+ visibleIngestionModes.length > 2+ ? "md:grid-cols-3"+ : "md:grid-cols-2",+ )}+ >+ {visibleIngestionModes.map((option) => {const Icon = option.icon;const selected = ingestionMode === option.value;return (diff --git a/web/src/features/ai-gateway/server/provider/registry.ts b/web/src/features/ai-gateway/server/provider/registry.tsindex 8cebca0319..fa08daa629 100644--- a/web/src/features/ai-gateway/server/provider/registry.ts+++ b/web/src/features/ai-gateway/server/provider/registry.ts@@ -51,7 +51,7 @@ export const GatewayResolveResponseSchema = zprovider_connection_id: z.string(),}).strict(),- ingestion_mode: z.enum(["usage", "full"]),+ ingestion_mode: z.enum(["none", "usage", "full"]),ingestion: z.object({access_token: z.string(),diff --git a/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.ts b/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.tsindex fc97c44e90..6c81b3e4a7 100644--- a/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.ts+++ b/web/src/features/ai-gateway/server/resolve/gatewayResolveCache.ts@@ -28,7 +28,7 @@ export const CachedResolveContext = z.object({apiKeyId: z.string(),keyMetadata: GatewayMetadataSchema,ingestionProjectId: z.string().nullable(),- ingestionMode: z.enum(["USAGE", "FULL"]).nullable(),+ ingestionMode: z.enum(["NONE", "USAGE", "FULL"]).nullable(),connections: z.array(z.object({id: z.string(),diff --git a/web/src/features/ai-gateway/server/resolve/resolveService.ts b/web/src/features/ai-gateway/server/resolve/resolveService.tsindex b59f0b784f..642b476178 100644--- a/web/src/features/ai-gateway/server/resolve/resolveService.ts+++ b/web/src/features/ai-gateway/server/resolve/resolveService.ts@@ -117,7 +117,8 @@ export class GatewayResolveService {key_metadata: context.keyMetadata,provider_connection_id: context.connection.id,},- ingestion_mode: context.ingestionMode.toLowerCase() as "usage" | "full",+ ingestion_mode:+ context.ingestionMode.toLowerCase() as GatewayResolveResponse["ingestion_mode"],ingestion: this.createIngestionResponse({mode: context.ingestionMode,organizationId: context.organizationId,@@ -146,7 +147,8 @@ export class GatewayResolveService {version: 1,organization_id: params.organizationId,project_id: params.projectId,- ingestion_mode: params.mode.toLowerCase() as "usage" | "full",+ ingestion_mode:+ params.mode.toLowerCase() as GatewayResolveResponse["ingestion_mode"],scope: "gateway-ingest",},}),diff --git a/web/src/features/ai-gateway/server/router.ts b/web/src/features/ai-gateway/server/router.tsindex ef941974e4..8871935b8e 100644--- a/web/src/features/ai-gateway/server/router.ts+++ b/web/src/features/ai-gateway/server/router.ts@@ -108,7 +108,14 @@ export const aiGatewayRouter = createTRPCRouter({.input(organizationInput.extend({defaultIngestionProjectId: z.string().nullable(),- ingestionMode: z.enum(GatewayIngestionMode),+ ingestionMode: z+ .enum(GatewayIngestionMode)+ .refine(+ (mode) =>+ mode !== GatewayIngestionMode.NONE ||+ !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,+ "Ingestion mode NONE is only available on self-hosted deployments",+ ),}),).mutation(async ({ input, ctx }) => {