🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(ai-gateway): show the unpriced-usage banner above analytics and skip zero-token models (#19570)
Tobias Wochinger435ef1e9154a
diff --git a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsDashboard/GatewayAnalyticsDashboard.tsx b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsDashboard/GatewayAnalyticsDashboard.tsxindex f26ee7f63b..37bb981ab7 100644--- a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsDashboard/GatewayAnalyticsDashboard.tsx+++ b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsDashboard/GatewayAnalyticsDashboard.tsx@@ -81,10 +81,15 @@ function GatewayAnalyticsWidgets({timeRange,filters,});- const hasUnpricedUsage = (unpricedUsage?.requests ?? 0) > 0;return (<>+ {unpricedUsage && unpricedUsage.models.length > 0 ? (+ <GatewayUnpricedUsageAlert+ projectId={projectId}+ usage={unpricedUsage}+ />+ ) : null}<div className="grid grid-cols-1 gap-3 lg:grid-cols-6">{GATEWAY_ANALYTICS_WIDGETS.map((widget) => (<GatewayAnalyticsWidgetCard@@ -99,17 +104,9 @@ function GatewayAnalyticsWidgets({isWaitingForLabels={Boolean(widget.dimensionLabels) && labelsPending}- unpricedUsage={hasUnpricedUsage ? unpricedUsage : undefined}/>))}</div>- {/* Below the grid: it loads after the widgets and must not push them down. */}- {unpricedUsage && hasUnpricedUsage ? (- <GatewayUnpricedUsageAlert- projectId={projectId}- usage={unpricedUsage}- />- ) : null}</>);}diff --git a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsWidgetCard/GatewayAnalyticsWidgetCard.tsx b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsWidgetCard/GatewayAnalyticsWidgetCard.tsxindex b276ba58b1..452819126f 100644--- a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsWidgetCard/GatewayAnalyticsWidgetCard.tsx+++ b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsWidgetCard/GatewayAnalyticsWidgetCard.tsx@@ -1,10 +1,8 @@import { useMemo } from "react";import type { FilterState } from "@langfuse/shared";-import { GatewayUnpricedCostBadge } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayUnpricedCostBadge/GatewayUnpricedCostBadge";import { GATEWAY_ANALYTICS_WIDGET_SIZE_CLASSES } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/constants/gatewayAnalyticsWidgetSizeClasses";import { buildGatewayAnalyticsQuery } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/buildGatewayAnalyticsQuery";-import type { GatewayUnpricedUsage } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage";import type { GatewayAnalyticsWidget } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/types/gatewayAnalyticsWidget";import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";import { WidgetContent } from "@/src/features/widgets";@@ -29,7 +27,6 @@ export function GatewayAnalyticsWidgetCard({filters,dimensionLabelMap,isWaitingForLabels = false,- unpricedUsage,}: {projectId: string;widget: GatewayAnalyticsWidget;@@ -38,18 +35,11 @@ export function GatewayAnalyticsWidgetCard({dimensionLabelMap?: ReadonlyMap<string, string>;/** Covers the chart, which keeps loading underneath, so raw IDs never flash before their labels. */isWaitingForLabels?: boolean;- /** Flags cost widgets whose totals exclude usage of models without a price. */- unpricedUsage?: GatewayUnpricedUsage;}) {const query = useMemo(() => buildGatewayAnalyticsQuery(widget, timeRange, filters),[widget, timeRange, filters],);- const excludedUnpricedUsage = widget.metrics.some(- ({ measure }) => measure === "totalCost",- )- ? unpricedUsage- : undefined;return (<DashboardCard@@ -58,17 +48,7 @@ export function GatewayAnalyticsWidgetCard({"ph-no-capture min-w-0",GATEWAY_ANALYTICS_WIDGET_SIZE_CLASSES[widget.size],)}- title={- excludedUnpricedUsage ? (- // The title line is tall enough that the badge never reflows the card.- <span className="flex items-center gap-2">- {widget.name}- <GatewayUnpricedCostBadge usage={excludedUnpricedUsage} />- </span>- ) : (- widget.name- )- }+ title={widget.name}description={widget.description}// WidgetContent renders its own loading state.isLoading={false}diff --git a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayUnpricedCostBadge/GatewayUnpricedCostBadge.tsx b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayUnpricedCostBadge/GatewayUnpricedCostBadge.tsxdeleted file mode 100644index b887d0d5e2..0000000000--- a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayUnpricedCostBadge/GatewayUnpricedCostBadge.tsx+++ /dev/null@@ -1,36 +0,0 @@-import { AlertTriangle } from "@langfuse/icons";--import { Badge } from "@/src/components/design-system/Badge/Badge";-import { Tooltip } from "@/src/components/design-system/Tooltip/Tooltip";-import { describeUnpricedUsage } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/describeUnpricedUsage";-import type { GatewayUnpricedUsage } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage";--export type GatewayUnpricedCostBadgeProps = {- usage: GatewayUnpricedUsage;-};--export function GatewayUnpricedCostBadge({- usage,-}: GatewayUnpricedCostBadgeProps) {- return (- <Tooltip- label={`Excludes ${describeUnpricedUsage(usage)} to models without a price.`}- >- {({ getTriggerProps }) => (- <span- {...getTriggerProps()}- tabIndex={0}- className="focus-visible:ring-ring h-fit shrink-0 rounded-sm focus-visible:ring-2 focus-visible:outline-hidden"- >- <Badge- size="sm"- color="yellow"- leadingIcon={AlertTriangle}- interactive- text="Excludes unpriced"- />- </span>- )}- </Tooltip>- );-}diff --git a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.clienttest.ts b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.clienttest.tsindex 7421b6a248..428b20242d 100644--- a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.clienttest.ts+++ b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.clienttest.ts@@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest";import { summarizeUnpricedUsage } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage";describe("summarizeUnpricedUsage", () => {- it("sums requests and tokens of models without recorded cost", () => {+ it("sums requests and tokens of models that used tokens without recorded cost", () => {expect(summarizeUnpricedUsage([{@@ -24,6 +24,12 @@ describe("summarizeUnpricedUsage", () => {sum_totalTokens: 600,sum_totalCost: null,},+ {+ providedModelName: "embedding-without-usage",+ count_count: 5,+ sum_totalTokens: 0,+ sum_totalCost: 0,+ },]),).toEqual({requests: 15,diff --git a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.ts b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.tsindex f16d5b9dc4..4bb89631b4 100644--- a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.ts+++ b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.ts@@ -7,7 +7,8 @@ export type GatewayUnpricedUsage = {/*** Picks the models whose requests recorded no cost from a per-model breakdown.* Events store a missing price as a zero cost, so a model with requests and a- * zero total has no Langfuse price and its spend reads as $0.+ * zero total has no Langfuse price and its spend reads as $0. Models without+ * tokens are skipped: they would cost $0 even with a price.*/export function summarizeUnpricedUsage(rows: Record<string, unknown>[],@@ -15,10 +16,11 @@ export function summarizeUnpricedUsage(const summary: GatewayUnpricedUsage = { requests: 0, tokens: 0, models: [] };for (const row of rows) {const requests = Number(row.count_count ?? 0);+ const tokens = Number(row.sum_totalTokens ?? 0);const cost = Number(row.sum_totalCost ?? 0);- if (requests <= 0 || cost !== 0) continue;+ if (requests <= 0 || tokens <= 0 || cost !== 0) continue;summary.requests += requests;- summary.tokens += Number(row.sum_totalTokens ?? 0);+ summary.tokens += tokens;summary.models.push(String(row.providedModelName || "Unknown model"));}return summary;