Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(ai-gateway): show the unpriced-usage banner above analytics and skip zero-token models (#19570)

Tobias Wochinger435ef1e9154a
diff --git a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsDashboard/GatewayAnalyticsDashboard.tsx b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsDashboard/GatewayAnalyticsDashboard.tsx
index f26ee7f63b..37bb981ab7 100644
--- a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsDashboard/GatewayAnalyticsDashboard.tsx
+++ b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsDashboard/GatewayAnalyticsDashboard.tsx
@@ -81,10 +81,15 @@ function GatewayAnalyticsWidgets({
timeRange,
filters,
});
- const hasUnpricedUsage = (unpricedUsage?.requests ?? 0) > 0;
return (
<>
+ {unpricedUsage && unpricedUsage.models.length > 0 ? (
+ <GatewayUnpricedUsageAlert
+ projectId={projectId}
+ usage={unpricedUsage}
+ />
+ ) : null}
<div className="grid grid-cols-1 gap-3 lg:grid-cols-6">
{GATEWAY_ANALYTICS_WIDGETS.map((widget) => (
<GatewayAnalyticsWidgetCard
@@ -99,17 +104,9 @@ function GatewayAnalyticsWidgets({
isWaitingForLabels={
Boolean(widget.dimensionLabels) && labelsPending
}
- unpricedUsage={hasUnpricedUsage ? unpricedUsage : undefined}
/>
))}
</div>
- {/* Below the grid: it loads after the widgets and must not push them down. */}
- {unpricedUsage && hasUnpricedUsage ? (
- <GatewayUnpricedUsageAlert
- projectId={projectId}
- usage={unpricedUsage}
- />
- ) : null}
</>
);
}
diff --git a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsWidgetCard/GatewayAnalyticsWidgetCard.tsx b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsWidgetCard/GatewayAnalyticsWidgetCard.tsx
index b276ba58b1..452819126f 100644
--- a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsWidgetCard/GatewayAnalyticsWidgetCard.tsx
+++ b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayAnalyticsWidgetCard/GatewayAnalyticsWidgetCard.tsx
@@ -1,10 +1,8 @@
import { useMemo } from "react";
import type { FilterState } from "@langfuse/shared";
-import { GatewayUnpricedCostBadge } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayUnpricedCostBadge/GatewayUnpricedCostBadge";
import { GATEWAY_ANALYTICS_WIDGET_SIZE_CLASSES } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/constants/gatewayAnalyticsWidgetSizeClasses";
import { buildGatewayAnalyticsQuery } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/buildGatewayAnalyticsQuery";
-import type { GatewayUnpricedUsage } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage";
import type { GatewayAnalyticsWidget } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/types/gatewayAnalyticsWidget";
import { DashboardCard } from "@/src/features/dashboard/components/cards/DashboardCard";
import { WidgetContent } from "@/src/features/widgets";
@@ -29,7 +27,6 @@ export function GatewayAnalyticsWidgetCard({
filters,
dimensionLabelMap,
isWaitingForLabels = false,
- unpricedUsage,
}: {
projectId: string;
widget: GatewayAnalyticsWidget;
@@ -38,18 +35,11 @@ export function GatewayAnalyticsWidgetCard({
dimensionLabelMap?: ReadonlyMap<string, string>;
/** Covers the chart, which keeps loading underneath, so raw IDs never flash before their labels. */
isWaitingForLabels?: boolean;
- /** Flags cost widgets whose totals exclude usage of models without a price. */
- unpricedUsage?: GatewayUnpricedUsage;
}) {
const query = useMemo(
() => buildGatewayAnalyticsQuery(widget, timeRange, filters),
[widget, timeRange, filters],
);
- const excludedUnpricedUsage = widget.metrics.some(
- ({ measure }) => measure === "totalCost",
- )
- ? unpricedUsage
- : undefined;
return (
<DashboardCard
@@ -58,17 +48,7 @@ export function GatewayAnalyticsWidgetCard({
"ph-no-capture min-w-0",
GATEWAY_ANALYTICS_WIDGET_SIZE_CLASSES[widget.size],
)}
- title={
- excludedUnpricedUsage ? (
- // The title line is tall enough that the badge never reflows the card.
- <span className="flex items-center gap-2">
- {widget.name}
- <GatewayUnpricedCostBadge usage={excludedUnpricedUsage} />
- </span>
- ) : (
- widget.name
- )
- }
+ title={widget.name}
description={widget.description}
// WidgetContent renders its own loading state.
isLoading={false}
diff --git a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayUnpricedCostBadge/GatewayUnpricedCostBadge.tsx b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayUnpricedCostBadge/GatewayUnpricedCostBadge.tsx
deleted file mode 100644
index b887d0d5e2..0000000000
--- a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/components/GatewayUnpricedCostBadge/GatewayUnpricedCostBadge.tsx
+++ /dev/null
@@ -1,36 +0,0 @@
-import { AlertTriangle } from "@langfuse/icons";
-
-import { Badge } from "@/src/components/design-system/Badge/Badge";
-import { Tooltip } from "@/src/components/design-system/Tooltip/Tooltip";
-import { describeUnpricedUsage } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/describeUnpricedUsage";
-import type { GatewayUnpricedUsage } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage";
-
-export type GatewayUnpricedCostBadgeProps = {
- usage: GatewayUnpricedUsage;
-};
-
-export function GatewayUnpricedCostBadge({
- usage,
-}: GatewayUnpricedCostBadgeProps) {
- return (
- <Tooltip
- label={`Excludes ${describeUnpricedUsage(usage)} to models without a price.`}
- >
- {({ getTriggerProps }) => (
- <span
- {...getTriggerProps()}
- tabIndex={0}
- className="focus-visible:ring-ring h-fit shrink-0 rounded-sm focus-visible:ring-2 focus-visible:outline-hidden"
- >
- <Badge
- size="sm"
- color="yellow"
- leadingIcon={AlertTriangle}
- interactive
- text="Excludes unpriced"
- />
- </span>
- )}
- </Tooltip>
- );
-}
diff --git a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.clienttest.ts b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.clienttest.ts
index 7421b6a248..428b20242d 100644
--- a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.clienttest.ts
+++ b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.clienttest.ts
@@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest";
import { summarizeUnpricedUsage } from "@/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage";
describe("summarizeUnpricedUsage", () => {
- it("sums requests and tokens of models without recorded cost", () => {
+ it("sums requests and tokens of models that used tokens without recorded cost", () => {
expect(
summarizeUnpricedUsage([
{
@@ -24,6 +24,12 @@ describe("summarizeUnpricedUsage", () => {
sum_totalTokens: 600,
sum_totalCost: null,
},
+ {
+ providedModelName: "embedding-without-usage",
+ count_count: 5,
+ sum_totalTokens: 0,
+ sum_totalCost: 0,
+ },
]),
).toEqual({
requests: 15,
diff --git a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.ts b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.ts
index f16d5b9dc4..4bb89631b4 100644
--- a/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.ts
+++ b/web/src/features/ai-gateway/components/GatewayAnalyticsPage/fns/summarizeUnpricedUsage.ts
@@ -7,7 +7,8 @@ export type GatewayUnpricedUsage = {
/**
* Picks the models whose requests recorded no cost from a per-model breakdown.
* Events store a missing price as a zero cost, so a model with requests and a
- * zero total has no Langfuse price and its spend reads as $0.
+ * zero total has no Langfuse price and its spend reads as $0. Models without
+ * tokens are skipped: they would cost $0 even with a price.
*/
export function summarizeUnpricedUsage(
rows: Record<string, unknown>[],
@@ -15,10 +16,11 @@ export function summarizeUnpricedUsage(
const summary: GatewayUnpricedUsage = { requests: 0, tokens: 0, models: [] };
for (const row of rows) {
const requests = Number(row.count_count ?? 0);
+ const tokens = Number(row.sum_totalTokens ?? 0);
const cost = Number(row.sum_totalCost ?? 0);
- if (requests <= 0 || cost !== 0) continue;
+ if (requests <= 0 || tokens <= 0 || cost !== 0) continue;
summary.requests += requests;
- summary.tokens += Number(row.sum_totalTokens ?? 0);
+ summary.tokens += tokens;
summary.models.push(String(row.providedModelName || "Unknown model"));
}
return summary;