🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(filters): compare token counts as Float64 (#18212)
Niklas Semmlera64a2f8b22e9
diff --git a/packages/shared/src/server/tableMappings/mapEventsTable.ts b/packages/shared/src/server/tableMappings/mapEventsTable.tsindex 96faf84172..039614390e 100644--- a/packages/shared/src/server/tableMappings/mapEventsTable.ts+++ b/packages/shared/src/server/tableMappings/mapEventsTable.ts@@ -11,6 +11,7 @@ import {eventsTableCachedInputTokensSql,eventsTableTraceNameSql,} from "../../eventsTable";+import { tokenCountFilterClickhouseType } from "./tokenCountFilterType";export const eventsTableNativeUiColumnDefinitions: UiColumnMappings = [{@@ -175,14 +176,14 @@ export const eventsTableNativeUiColumnDefinitions: UiColumnMappings = [clickhouseTableName: "events_proto",clickhouseSelect:"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, usage_details)))",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Cached Input Tokens",uiTableId: "cachedInputTokens",clickhouseTableName: "events_proto",clickhouseSelect: eventsTableCachedInputTokensSql,- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Output Tokens",@@ -190,7 +191,7 @@ export const eventsTableNativeUiColumnDefinitions: UiColumnMappings = [clickhouseTableName: "events_proto",clickhouseSelect:"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details)))",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Total Tokens",@@ -198,7 +199,7 @@ export const eventsTableNativeUiColumnDefinitions: UiColumnMappings = [clickhouseTableName: "events_proto",clickhouseSelect:"if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Tokens",@@ -206,7 +207,7 @@ export const eventsTableNativeUiColumnDefinitions: UiColumnMappings = [clickhouseTableName: "events_proto",clickhouseSelect:"if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Metadata",diff --git a/packages/shared/src/server/tableMappings/mapObservationsTable.ts b/packages/shared/src/server/tableMappings/mapObservationsTable.tsindex 5e02037b7a..412f5fe76e 100644--- a/packages/shared/src/server/tableMappings/mapObservationsTable.ts+++ b/packages/shared/src/server/tableMappings/mapObservationsTable.ts@@ -2,6 +2,7 @@// The frontend only sends the column names to the backend. This needs to be changed in the future to send column IDs.import { UiColumnMappings } from "../../tableDefinitions";+import { tokenCountFilterClickhouseType } from "./tokenCountFilterType";export const observationsTableTraceUiColumnDefinitions: UiColumnMappings = [{@@ -180,7 +181,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMappings = [clickhouseTableName: "observations",clickhouseSelect:"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, usage_details)))",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Output Tokens",@@ -188,7 +189,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMappings = [clickhouseTableName: "observations",clickhouseSelect:"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details)))",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Total Tokens",@@ -196,7 +197,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMappings = [clickhouseTableName: "observations",clickhouseSelect:"if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Tokens",@@ -204,7 +205,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMappings = [clickhouseTableName: "observations",clickhouseSelect:"if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Metadata",diff --git a/packages/shared/src/server/tableMappings/mapSessionTable.ts b/packages/shared/src/server/tableMappings/mapSessionTable.tsindex e194679a73..af943600a1 100644--- a/packages/shared/src/server/tableMappings/mapSessionTable.ts+++ b/packages/shared/src/server/tableMappings/mapSessionTable.ts@@ -1,4 +1,5 @@import { UiColumnMappings } from "../../tableDefinitions";+import { tokenCountFilterClickhouseType } from "./tokenCountFilterType";export const sessionCols: UiColumnMappings = [// we do not access the traces scores in ClickHouse. We default back to the trace timestamps.@@ -64,35 +65,35 @@ export const sessionCols: UiColumnMappings = [uiTableId: "inputTokens",clickhouseTableName: "traces",clickhouseSelect: "session_input_usage",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Output Tokens",uiTableId: "outputTokens",clickhouseTableName: "traces",clickhouseSelect: "session_output_usage",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Total Tokens",uiTableId: "totalTokens",clickhouseTableName: "traces",clickhouseSelect: "session_total_usage",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Usage",uiTableId: "totalTokens",clickhouseTableName: "traces",clickhouseSelect: "session_total_usage",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Session Total Usage",uiTableId: "usage",clickhouseTableName: "traces",clickhouseSelect: "session_total_usage",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Session Duration (s)",diff --git a/packages/shared/src/server/tableMappings/mapTracesTable.ts b/packages/shared/src/server/tableMappings/mapTracesTable.tsindex 6f24af1d5d..ca20fad782 100644--- a/packages/shared/src/server/tableMappings/mapTracesTable.ts+++ b/packages/shared/src/server/tableMappings/mapTracesTable.ts@@ -1,5 +1,6 @@import { UiColumnMappings } from "../../tableDefinitions";import { EvalExecutionMetadataKey } from "../../features/evals/evalExecutionMetadata";+import { tokenCountFilterClickhouseType } from "./tokenCountFilterType";export const tracesTableUiColumnDefinitions: UiColumnMappings = [{@@ -155,7 +156,7 @@ export const tracesTableUiColumnDefinitions: UiColumnMappings = [clickhouseTableName: "observations",clickhouseSelect:"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, o.usage_details)))",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Output Tokens",@@ -163,7 +164,7 @@ export const tracesTableUiColumnDefinitions: UiColumnMappings = [clickhouseTableName: "observations",clickhouseSelect:"arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, o.usage_details)))",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Total Tokens",@@ -171,7 +172,7 @@ export const tracesTableUiColumnDefinitions: UiColumnMappings = [clickhouseTableName: "observations",clickhouseSelect:"if(mapExists((k, v) -> (k = 'total'), o.usage_details), o.usage_details['total'], NULL)",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},{uiTableName: "Tokens",@@ -179,7 +180,7 @@ export const tracesTableUiColumnDefinitions: UiColumnMappings = [clickhouseTableName: "observations",clickhouseSelect:"if(mapExists((k, v) -> (k = 'total'), o.usage_details), o.usage_details['total'], NULL)",- clickhouseTypeOverwrite: "Decimal64(3)",+ clickhouseTypeOverwrite: tokenCountFilterClickhouseType,},// Scores column duplicated to allow renaming column name. Will be removed once session storage cache is outdated// Column names are cached in user sessions - changing them breaks existing filtersdiff --git a/packages/shared/src/server/tableMappings/tokenCountFilterType.ts b/packages/shared/src/server/tableMappings/tokenCountFilterType.tsnew file mode 100644index 0000000000..a4ca4cf729--- /dev/null+++ b/packages/shared/src/server/tableMappings/tokenCountFilterType.ts@@ -0,0 +1,5 @@+// usage_details is Map(String, UInt64). Comparing those counts with+// Decimal64(3) scales each stored value into Int64 and throws+// DECIMAL_OVERFLOW above ~9.2e15, which fails the whole query. Float64+// covers that range and still accepts fractional filter values.+export const tokenCountFilterClickhouseType = "Float64";diff --git a/packages/shared/src/server/tableMappings/tokenCountFilters.test.ts b/packages/shared/src/server/tableMappings/tokenCountFilters.test.tsnew file mode 100644index 0000000000..438c063632--- /dev/null+++ b/packages/shared/src/server/tableMappings/tokenCountFilters.test.ts@@ -0,0 +1,69 @@+import { describe, expect, it } from "vitest";++import { type UiColumnMappings } from "../../tableDefinitions";+import { createFilterFromFilterState } from "../queries/clickhouse-sql/factory";+import { eventsTableUiColumnDefinitions } from "./mapEventsTable";+import { observationsTableUiColumnDefinitions } from "./mapObservationsTable";+import { sessionCols } from "./mapSessionTable";+import { tracesTableUiColumnDefinitions } from "./mapTracesTable";++// usage_details is Map(String, UInt64). A Decimal64(3) comparison scales each+// stored count into Int64 and throws DECIMAL_OVERFLOW above ~9.2e15.+const isUsageCountSelect = (select: string) =>+ /usage_details|session_(input|output|total)_usage/.test(select) &&+ !select.includes("/");++const tables: { name: string; columns: UiColumnMappings }[] = [+ { name: "events", columns: eventsTableUiColumnDefinitions },+ { name: "observations", columns: observationsTableUiColumnDefinitions },+ { name: "traces", columns: tracesTableUiColumnDefinitions },+ { name: "sessions", columns: sessionCols },+];++describe("token count filters", () => {+ it.each(tables)(+ "compares $name usage counts as Float64 so huge UInt64 values do not overflow",+ ({ columns }) => {+ const tokenColumns = columns.filter((column) =>+ isUsageCountSelect(column.clickhouseSelect),+ );+ expect(tokenColumns.length).toBeGreaterThan(0);++ for (const column of tokenColumns) {+ const [filter] = createFilterFromFilterState(+ [+ {+ column: column.uiTableId,+ type: "number",+ operator: ">",+ value: 1.5,+ },+ ],+ columns,+ );+ const applied = filter.apply();++ expect(applied.query).toContain(column.clickhouseSelect);+ expect(applied.query).toContain(": Float64}");+ expect(applied.query).not.toContain("Decimal");+ expect(Object.values(applied.params)).toEqual(["1.5"]);+ }+ },+ );++ it.each(tables)(+ "keeps $name duration filters on Decimal64(3)",+ ({ columns }) => {+ const durationColumns = columns.filter((column) =>+ ["latency", "timeToFirstToken", "sessionDuration"].includes(+ column.uiTableId,+ ),+ );++ expect(durationColumns.length).toBeGreaterThan(0);+ for (const column of durationColumns) {+ expect(column.clickhouseTypeOverwrite).toBe("Decimal64(3)");+ }+ },+ );+});diff --git a/web/src/__tests__/server/event-query-builder.servertest.ts b/web/src/__tests__/server/event-query-builder.servertest.tsindex a5e55f2494..0b0998685e 100644--- a/web/src/__tests__/server/event-query-builder.servertest.ts+++ b/web/src/__tests__/server/event-query-builder.servertest.ts@@ -19,7 +19,7 @@ import {describe("buildEventsFilterOptionsForColumnsQuery", () => {it.each([- ["cachedInputTokens", eventsTableCachedInputTokensSql, "Decimal64(3)"],+ ["cachedInputTokens", eventsTableCachedInputTokensSql, "Float64"],["cachedInputCost", eventsTableCachedInputCostSql, "Decimal64(12)"],] as const)("maps %s filters to the cached-read metric expression",