Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(trace-table): add cached input tokens and cost (#16767)

Nimare960a6cd1d2d
diff --git a/packages/shared/src/eventsTable.ts b/packages/shared/src/eventsTable.ts
index 3e53078414..bd3c2e936c 100644
--- a/packages/shared/src/eventsTable.ts
+++ b/packages/shared/src/eventsTable.ts
@@ -54,6 +54,47 @@ export const isRootObservation = ({
export const eventsTableHasInputSql = "e.input != ''";
export const eventsTableHasOutputSql = "e.output != ''";
+const isCachedInputMetric = (key: string): boolean => {
+ const normalizedKey = key.toLowerCase();
+ return (
+ normalizedKey.includes("cached") || normalizedKey.includes("cache_read")
+ );
+};
+
+const findCachedInputMetric = (
+ details?: Record<string, number> | null,
+): number | undefined => {
+ const values = Object.entries(details ?? {})
+ .filter(([key]) => isCachedInputMetric(key))
+ .map(([, value]) => Number(value));
+
+ return values.length > 0
+ ? values.reduce((sum, value) => sum + value, 0)
+ : undefined;
+};
+
+export const getCachedInputMetric = (
+ details?: Record<string, number> | null,
+): number | undefined => findCachedInputMetric(details);
+
+export const getCachedInputCost = (
+ details?: Record<string, number> | null,
+): number | undefined => findCachedInputMetric(details);
+
+const cachedInputMetricSql = (detailsColumn: string): string => {
+ const keyPredicate = (key: string) =>
+ `(positionCaseInsensitive(${key}, 'cached') > 0 OR positionCaseInsensitive(${key}, 'cache_read') > 0)`;
+ const filteredDetails = `mapFilter(x -> ${keyPredicate("x.1")}, ${detailsColumn})`;
+ const sum = `arraySum(mapValues(${filteredDetails}))`;
+
+ return `if(mapExists((k, v) -> ${keyPredicate("k")}, ${detailsColumn}), ${sum}, NULL)`;
+};
+
+export const eventsTableCachedInputTokensSql =
+ cachedInputMetricSql("e.usage_details");
+export const eventsTableCachedInputCostSql =
+ cachedInputMetricSql("e.cost_details");
+
type MutableDeep<T> = T extends readonly (infer U)[]
? MutableDeep<U>[]
: T extends object
@@ -238,6 +279,13 @@ const eventsTableColsDefinition = [
"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, usage_details)))",
nullable: true,
},
+ {
+ name: "Cached Input Tokens",
+ id: "cachedInputTokens",
+ type: "number",
+ internal: eventsTableCachedInputTokensSql,
+ nullable: true,
+ },
{
name: "Output Tokens",
id: "outputTokens",
@@ -262,6 +310,13 @@ const eventsTableColsDefinition = [
"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, cost_details)))",
nullable: true,
},
+ {
+ name: "Cached Input Cost ($)",
+ id: "cachedInputCost",
+ type: "number",
+ internal: eventsTableCachedInputCostSql,
+ nullable: true,
+ },
{
name: "Output Cost ($)",
id: "outputCost",
diff --git a/packages/shared/src/server/tableMappings/mapEventsTable.ts b/packages/shared/src/server/tableMappings/mapEventsTable.ts
index 68088ea4f5..cee429475b 100644
--- a/packages/shared/src/server/tableMappings/mapEventsTable.ts
+++ b/packages/shared/src/server/tableMappings/mapEventsTable.ts
@@ -8,6 +8,8 @@ import {
eventsTableIsRootObservationSql,
eventsTableHasInputSql,
eventsTableHasOutputSql,
+ eventsTableCachedInputCostSql,
+ eventsTableCachedInputTokensSql,
eventsTableTraceNameSql,
} from "../../eventsTable";
@@ -116,6 +118,12 @@ export const eventsTableNativeUiColumnDefinitions: UiColumnMappings = [
clickhouseSelect:
"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, cost_details)))",
},
+ {
+ uiTableName: "Cached Input Cost ($)",
+ uiTableId: "cachedInputCost",
+ clickhouseTableName: "events_proto",
+ clickhouseSelect: eventsTableCachedInputCostSql,
+ },
{
uiTableName: "Output Cost ($)",
uiTableId: "outputCost",
@@ -170,6 +178,13 @@ export const eventsTableNativeUiColumnDefinitions: UiColumnMappings = [
"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, usage_details)))",
clickhouseTypeOverwrite: "Decimal64(3)",
},
+ {
+ uiTableName: "Cached Input Tokens",
+ uiTableId: "cachedInputTokens",
+ clickhouseTableName: "events_proto",
+ clickhouseSelect: eventsTableCachedInputTokensSql,
+ clickhouseTypeOverwrite: "Decimal64(3)",
+ },
{
uiTableName: "Output Tokens",
uiTableId: "outputTokens",
diff --git a/web/src/__tests__/server/event-query-builder.servertest.ts b/web/src/__tests__/server/event-query-builder.servertest.ts
index 549bb5d735..adf6894f0d 100644
--- a/web/src/__tests__/server/event-query-builder.servertest.ts
+++ b/web/src/__tests__/server/event-query-builder.servertest.ts
@@ -9,9 +9,71 @@ import {
eventsTableUiColumnDefinitions,
ExperimentsAggregationQueryBuilder,
} from "@langfuse/shared/src/server";
-import { eventsTableCols } from "@langfuse/shared";
+import {
+ eventsTableCachedInputCostSql,
+ eventsTableCachedInputTokensSql,
+ eventsTableCols,
+} from "@langfuse/shared";
describe("buildEventsFilterOptionsForColumnsQuery", () => {
+ it.each([
+ ["cachedInputTokens", eventsTableCachedInputTokensSql, "Decimal64(3)"],
+ ["cachedInputCost", eventsTableCachedInputCostSql, "Decimal64(12)"],
+ ] as const)(
+ "maps %s filters to the cached-read metric expression",
+ (column, expression, clickhouseType) => {
+ const [filter] = createFilterFromFilterState(
+ [
+ {
+ column,
+ type: "number",
+ operator: "=",
+ value: 0,
+ },
+ ],
+ eventsTableUiColumnDefinitions,
+ eventsTableCols,
+ );
+
+ expect(filter).toBeDefined();
+ if (!filter) throw new Error("expected filter");
+ const applied = filter.apply();
+ expect(applied.query).toContain(expression);
+ expect(applied.query).toContain(clickhouseType);
+ expect(Object.values(applied.params)).toContain("0");
+ },
+ );
+
+ it.each([
+ ["cachedInputTokens", eventsTableCachedInputTokensSql],
+ ["cachedInputCost", eventsTableCachedInputCostSql],
+ ] as const)(
+ "keeps missing %s distinguishable from an explicit zero",
+ (column, expression) => {
+ expect(expression).toContain("mapExists");
+ expect(
+ eventsTableCols.find((definition) => definition.id === column),
+ ).toMatchObject({ nullable: true });
+
+ const [filter] = createFilterFromFilterState(
+ [
+ {
+ column,
+ type: "null",
+ operator: "is null",
+ value: "",
+ },
+ ],
+ eventsTableUiColumnDefinitions,
+ eventsTableCols,
+ );
+
+ expect(filter).toBeDefined();
+ if (!filter) throw new Error("expected filter");
+ expect(filter.apply().query).toContain(`${expression} is null`);
+ },
+ );
+
it("builds one events_core scan for multiple filter option columns", () => {
const built = buildEventsFilterOptionsForColumnsQuery({
projectId: "test-project",
diff --git a/web/src/features/chart-view/lib/chartFilterCompatibility.clienttest.ts b/web/src/features/chart-view/lib/chartFilterCompatibility.clienttest.ts
index 2bb4b44cc7..1369842431 100644
--- a/web/src/features/chart-view/lib/chartFilterCompatibility.clienttest.ts
+++ b/web/src/features/chart-view/lib/chartFilterCompatibility.clienttest.ts
@@ -32,6 +32,12 @@ describe("chartFilterExclusionReason", () => {
it("groups measures, scores, comments, and metadata by reason", () => {
expect(chartFilterExclusionReason("latency")).toMatch(/latency, cost/i);
+ expect(chartFilterExclusionReason("cachedInputTokens")).toMatch(
+ /latency, cost/i,
+ );
+ expect(chartFilterExclusionReason("cachedInputCost")).toMatch(
+ /latency, cost/i,
+ );
expect(chartFilterExclusionReason("totalCost")).toMatch(/latency, cost/i);
expect(chartFilterExclusionReason("scores_avg")).toMatch(/scores/i);
expect(chartFilterExclusionReason("trace_score_categories")).toMatch(
diff --git a/web/src/features/chart-view/lib/chartFilterCompatibility.ts b/web/src/features/chart-view/lib/chartFilterCompatibility.ts
index 4b47bd92b2..2f50ed0a54 100644
--- a/web/src/features/chart-view/lib/chartFilterCompatibility.ts
+++ b/web/src/features/chart-view/lib/chartFilterCompatibility.ts
@@ -68,9 +68,11 @@ const MEASURE_COLUMNS = new Set([
"latency",
"timeToFirstToken",
"inputTokens",
+ "cachedInputTokens",
"outputTokens",
"totalTokens",
"inputCost",
+ "cachedInputCost",
"outputCost",
"totalCost",
]);
diff --git a/web/src/features/events/components/EventsTable.tsx b/web/src/features/events/components/EventsTable.tsx
index 34f4626258..252d05bf0e 100644
--- a/web/src/features/events/components/EventsTable.tsx
+++ b/web/src/features/events/components/EventsTable.tsx
@@ -34,6 +34,8 @@ import {
type TracingSearchType,
type ScoreAggregate,
buildTracePath,
+ getCachedInputCost,
+ getCachedInputMetric,
} from "@langfuse/shared";
import { formatIntervalSeconds } from "@/src/utils/dates";
import { Skeleton } from "@/src/components/ui/skeleton";
@@ -1290,6 +1292,17 @@ export default function ObservationsEventsTable({
defaultHidden: true,
enableSorting,
}),
+ createNumberTableColumn<EventsTableRow>({
+ accessorFn: (row) => getCachedInputCost(row.costDetails),
+ id: "cachedInputCost",
+ header: getEventsColumnName("cachedInputCost"),
+ size: 140,
+ enableHiding: true,
+ defaultHidden: true,
+ enableSorting,
+ formatter: (value) => usdFormatter(value),
+ emptyValue: "-",
+ }),
createTextTableColumn<EventsTableRow>({
accessorFn: (row) =>
formatObservationCost(row.cost.outputCost, row.type),
@@ -1378,6 +1391,16 @@ export default function ObservationsEventsTable({
enableSorting,
formatter: (value) => numberFormatter(value, 0, 0),
}),
+ createNumberTableColumn<EventsTableRow>({
+ accessorFn: (row) => getCachedInputMetric(row.usageDetails),
+ id: "cachedInputTokens",
+ header: getEventsColumnName("cachedInputTokens"),
+ size: 140,
+ enableHiding: true,
+ defaultHidden: true,
+ enableSorting,
+ formatter: (value) => numberFormatter(value, 0, 0),
+ }),
createNumberTableColumn<EventsTableRow>({
id: "outputTokens",
accessorFn: (row) => row.usage.outputUsage,
diff --git a/web/src/features/events/config/filter-config.clienttest.ts b/web/src/features/events/config/filter-config.clienttest.ts
index 1c8e862862..facb752071 100644
--- a/web/src/features/events/config/filter-config.clienttest.ts
+++ b/web/src/features/events/config/filter-config.clienttest.ts
@@ -1,5 +1,7 @@
import {
eventsTableCols,
+ getCachedInputCost,
+ getCachedInputMetric,
observationsTableCols,
tracesTableCols,
} from "@langfuse/shared";
@@ -56,3 +58,45 @@ describe("observation status display name", () => {
}
});
});
+
+describe("cached input metrics", () => {
+ it("sums cache-read buckets and excludes cache creation", () => {
+ expect(
+ getCachedInputMetric({
+ input_cached_tokens: 20,
+ cache_read_input_tokens: 22,
+ input_cache_creation: 100,
+ }),
+ ).toBe(42);
+ });
+
+ it("distinguishes missing cached metrics from explicit zero values", () => {
+ expect(getCachedInputMetric()).toBeUndefined();
+ expect(getCachedInputCost()).toBeUndefined();
+ expect(getCachedInputCost({ input_cache_creation: 0.01 })).toBeUndefined();
+ expect(getCachedInputMetric({ input_cached_tokens: 0 })).toBe(0);
+ expect(getCachedInputCost({ input_cached_tokens: 0 })).toBe(0);
+ expect(getCachedInputCost({ input_cached_tokens: 0.004 })).toBe(0.004);
+ });
+
+ it("matches cache-read keys case-insensitively", () => {
+ expect(getCachedInputMetric({ INPUT_CACHED_TOKENS: 5 })).toBe(5);
+ });
+
+ it("exposes cached token and cost columns as numeric sidebar facets", () => {
+ expect(getEventsColumnName("cachedInputTokens")).toBe(
+ "Cached Input Tokens",
+ );
+ expect(getEventsColumnName("cachedInputCost")).toBe(
+ "Cached Input Cost ($)",
+ );
+
+ for (const column of ["cachedInputTokens", "cachedInputCost"]) {
+ expect(
+ observationEventsFilterConfig.facets.find(
+ (facet) => facet.column === column,
+ ),
+ ).toMatchObject({ type: "numeric" });
+ }
+ });
+});
diff --git a/web/src/features/events/config/filter-config.ts b/web/src/features/events/config/filter-config.ts
index af9e0af5c0..da39c8173c 100644
--- a/web/src/features/events/config/filter-config.ts
+++ b/web/src/features/events/config/filter-config.ts
@@ -245,6 +245,13 @@ export const observationEventsFilterConfig: FilterConfig = {
min: 0,
max: 1000000,
},
+ {
+ type: "numeric" as const,
+ column: "cachedInputTokens",
+ label: getEventsColumnName("cachedInputTokens"),
+ min: 0,
+ max: 1000000,
+ },
{
type: "numeric" as const,
column: "outputTokens",
@@ -267,6 +274,14 @@ export const observationEventsFilterConfig: FilterConfig = {
max: 100,
unit: "$",
},
+ {
+ type: "numeric" as const,
+ column: "cachedInputCost",
+ label: getEventsColumnName("cachedInputCost"),
+ min: 0,
+ max: 100,
+ unit: "$",
+ },
{
type: "numeric" as const,
column: "outputCost",
diff --git a/web/src/features/events/hooks/useEventsFilterOptions.ts b/web/src/features/events/hooks/useEventsFilterOptions.ts
index 370ae381d6..0a7feee729 100644
--- a/web/src/features/events/hooks/useEventsFilterOptions.ts
+++ b/web/src/features/events/hooks/useEventsFilterOptions.ts
@@ -350,9 +350,11 @@ export function useEventsFilterOptions({
timeToFirstToken: [],
tokensPerSecond: [],
inputTokens: [],
+ cachedInputTokens: [],
outputTokens: [],
totalTokens: [],
inputCost: [],
+ cachedInputCost: [],
outputCost: [],
totalCost: [],
score_categories: scoreCategories,
diff --git a/web/src/features/search-bar/lib/completions.ts b/web/src/features/search-bar/lib/completions.ts
index 779c5cb207..047eb9c68f 100644
--- a/web/src/features/search-bar/lib/completions.ts
+++ b/web/src/features/search-bar/lib/completions.ts
@@ -451,9 +451,11 @@ const NUMERIC_EXAMPLE: Record<string, string> = {
timeToFirstToken: "0.5",
tokensPerSecond: "50",
inputTokens: "1000",
+ cachedInputTokens: "500",
outputTokens: "500",
totalTokens: "1500",
inputCost: "0.001",
+ cachedInputCost: "0.0005",
outputCost: "0.001",
totalCost: "0.01",
promptVersion: "3",
diff --git a/web/src/features/search-bar/lib/fields.ts b/web/src/features/search-bar/lib/fields.ts
index 9aebb02719..538429272d 100644
--- a/web/src/features/search-bar/lib/fields.ts
+++ b/web/src/features/search-bar/lib/fields.ts
@@ -296,9 +296,11 @@ export const FIELDS: FieldDef[] = [
{ id: "timeToFirstToken", aliases: ["timetofirsttoken", "time_to_first_token", "ttft"], kind: "number", syncMode: "textSearch", label: "Time to first token", description: "Time to first token in seconds", unit: "s", nullable: true },
{ id: "tokensPerSecond", aliases: ["tokenspersecond", "tokens_per_second", "tps"], kind: "number", syncMode: "textSearch", label: "Tokens per second", description: "Output tokens per second", unit: "tok/s", nullable: true },
{ id: "inputTokens", aliases: ["inputtokens", "input_tokens"], kind: "number", syncMode: "textSearch", label: "Input token count", description: "Input token count", nullable: true },
+ { id: "cachedInputTokens", aliases: ["cachedinputtokens", "cached_input_tokens", "cachedtokens", "cached_tokens"], kind: "number", syncMode: "textSearch", label: "Cached input token count", description: "Cache-read input token count", nullable: true },
{ id: "outputTokens", aliases: ["outputtokens", "output_tokens"], kind: "number", syncMode: "textSearch", label: "Output token count", description: "Output token count", nullable: true },
{ id: "totalTokens", aliases: ["totaltokens", "total_tokens", "tokens"], kind: "number", syncMode: "textSearch", label: "Total token count", description: "Total token count", nullable: true },
{ id: "inputCost", aliases: ["inputcost", "input_cost"], kind: "number", syncMode: "textSearch", label: "Input cost", description: "Input cost in USD", unit: "$", nullable: true },
+ { id: "cachedInputCost", aliases: ["cachedinputcost", "cached_input_cost", "cachedcost", "cached_cost"], kind: "number", syncMode: "textSearch", label: "Cached input cost", description: "Cache-read input cost in USD", unit: "$", nullable: true },
{ id: "outputCost", aliases: ["outputcost", "output_cost"], kind: "number", syncMode: "textSearch", label: "Output cost", description: "Output cost in USD", unit: "$", nullable: true },
{ id: "totalCost", aliases: ["totalcost", "total_cost", "cost"], kind: "number", syncMode: "textSearch", label: "Total cost", description: "Total cost in USD", unit: "$", nullable: true },
{ id: "version", aliases: [], kind: "text", syncMode: "exactOption", label: "Version", description: "Version tag", nullable: true },
diff --git a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts
index be96432ba7..290987e1ae 100644
--- a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts
+++ b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts
@@ -218,6 +218,69 @@ describe("search bar invariants — events v4 registry", () => {
],
});
});
+
+ it("filters cached tokens and attributed cached cost", () => {
+ expect(
+ planCommit(
+ "cachedTokens:0 cachedCost:<=0",
+ undefined,
+ EVENTS_FIELD_REGISTRY,
+ ),
+ ).toMatchObject({
+ status: "committed",
+ filters: [
+ {
+ column: "cachedInputTokens",
+ type: "number",
+ operator: "=",
+ value: 0,
+ },
+ {
+ column: "cachedInputCost",
+ type: "number",
+ operator: "<=",
+ value: 0,
+ },
+ ],
+ });
+ });
+
+ it("supports presence filters for cached metrics", () => {
+ const query =
+ "has:cachedTokens -has:cachedTokens has:cachedCost -has:cachedCost";
+ expect(
+ validateQuery(query, undefined, EVENTS_FIELD_REGISTRY).diagnostics,
+ ).toEqual([]);
+ expect(planCommit(query, undefined, EVENTS_FIELD_REGISTRY)).toMatchObject({
+ status: "committed",
+ filters: [
+ {
+ column: "cachedInputTokens",
+ type: "null",
+ operator: "is not null",
+ value: "",
+ },
+ {
+ column: "cachedInputTokens",
+ type: "null",
+ operator: "is null",
+ value: "",
+ },
+ {
+ column: "cachedInputCost",
+ type: "null",
+ operator: "is not null",
+ value: "",
+ },
+ {
+ column: "cachedInputCost",
+ type: "null",
+ operator: "is null",
+ value: "",
+ },
+ ],
+ });
+ });
});
describe("search bar invariants — evaluation rules registry", () => {