πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(scores): drop ClickHouse FINAL from UI rows path (#17501)
Niklas Semmlerfd3630ef6580
diff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.tsindex f5cc7c4804..13d7861bda 100644--- a/packages/shared/src/server/repositories/scores.ts+++ b/packages/shared/src/server/repositories/scores.ts@@ -1459,7 +1459,8 @@ const SCORE_COUNT_DEDUP_COLUMNS = [] as const;/**- * Coarse toDate(timestamp) prune for the count subquery's pre-dedup scan.+ * Coarse toDate(timestamp) prune for a dedup subquery's pre-dedup scan (count+ * and rows paths share it).** The exact timestamp bound must NOT filter raw rows before dedup (a score's* timestamp is mutable across versions, so it could drop the latest one); it is@@ -1468,7 +1469,7 @@ const SCORE_COUNT_DEDUP_COLUMNS = [* survives to the dedup. Operators are relaxed (> / < to >= / <=) and both* sides wrapped in toDate() so boundary-day rows are never lost.*/-const buildScoresCountDatePrune = (+const buildScoresDatePrune = (filter: FilterState,): { query: string; params: Record<string, unknown> } => {const timestampColumn = scoresTableUiColumnDefinitionsFromEvents.find(@@ -1541,9 +1542,10 @@ const getScoresUiGenericFromEvents = async <T>(props: {);const scoreOnlyFilterRes = scoreOnlyFilters.apply();- // Coarse pre-dedup prune for the count subquery; see buildScoresCountDatePrune.+ // Coarse pre-dedup prune shared by the count and rows subqueries; see+ // buildScoresDatePrune.const { query: innerDatePruneQuery, params: innerDatePruneParams } =- buildScoresCountDatePrune(filter);+ buildScoresDatePrune(filter);// Trace-level filter entries from the frontend filter stateconst traceFilterState = filter.filter((filterEntry) =>@@ -1631,29 +1633,36 @@ const getScoresUiGenericFromEvents = async <T>(props: {${includeHasMetadataFlag ? ",length(mapKeys(s.metadata)) > 0 AS has_metadata" : ""}`;- const whereClause = `- WHERE s.project_id = {projectId: String}- AND s.data_type IN ({dataTypes: Array(String)})+ // Pre-dedup inner scan: project scope + coarse date prune only. Mutable-column+ // filters run post-dedup in the outer WHERE (see the dedup rule below).+ const innerScanWhere = `+ WHERE s.project_id = {projectId: String}+ ${innerDatePruneQuery ? `AND ${innerDatePruneQuery}` : ""}`;++ // Post-dedup outer filters, shared by the count and rows paths.+ const outerWhereClause = `+ WHERE s.data_type IN ({dataTypes: Array(String)})${scoreOnlyFilterRes?.query ? `AND ${scoreOnlyFilterRes.query}` : ""}`;- // ββ Count path: dedup without FINAL ββββββββββββββββββββββββββββββββββββββ- // Row reads dedup the ReplacingMergeTree with FINAL; for counts that is too- // slow. Instead reconstruct each score's latest version with one argMax pass:- // GROUP BY project_id, toDate(timestamp), name, id -- the sorting key- // FINAL collapses on- // argMax(col, event_ts) -- latest value per col+ // ββ Dedup without FINAL βββββββββββββββββββββββββββββββββββββββββββββββββββ+ // Reads dedup the ReplacingMergeTree by reconstructing each score's latest+ // version instead of FINAL: the count path via one argMax pass grouped on the+ // sorting key, the rows path via ORDER BY event_ts DESC + LIMIT 1 BY that same+ // key (keeps the whole latest row).//// Rule: filter AFTER dedup, never before. value / comment / timestamp /// trace_id are all mutable across a score's versions, so filtering raw rows- // can drop the true-latest version and let argMax rebuild a stale one β a- // count that disagrees with the row list. Example, filter `value > 0.5`:+ // can drop the true-latest version and surface a stale one β a result that+ // disagrees with FINAL. Example, filter `value > 0.5`:// v1 @10:02 value=0.9, v2 @10:07 value=0.1 (latest)- // filter-then-dedup -> v1 counted (WRONG)+ // filter-then-dedup -> v1 kept (WRONG)// dedup-then-filter -> 0.1 excluded (matches FINAL)//// So every filter and the trace join runs in the OUTER query. The inner scan// carries only a coarse toDate(timestamp) prune (see innerDatePruneQuery):// whole buckets are kept or dropped, so the latest is never lost pre-dedup.+ // Dedup granularity is the full sorting key, so different toDate(timestamp)+ // buckets of one id stay distinct β matching FINAL.const query =props.select === "count"? `@@ -1668,21 +1677,25 @@ const getScoresUiGenericFromEvents = async <T>(props: {(c) => `argMax(s.${c}, s.event_ts) AS ${c}`,).join(",\n ")}FROM scores s- WHERE s.project_id = {projectId: String}- ${innerDatePruneQuery ? `AND ${innerDatePruneQuery}` : ""}+ ${innerScanWhere}GROUP BY s.project_id, toDate(s.timestamp), s.name, s.id) s${eventsJoin}- WHERE s.data_type IN ({dataTypes: Array(String)})- ${scoreOnlyFilterRes?.query ? `AND ${scoreOnlyFilterRes.query}` : ""}+ ${outerWhereClause}`: `${tracesCTEClause}SELECT${rowSelect}- FROM scores s final+ FROM (+ SELECT *+ FROM scores s+ ${innerScanWhere}+ ORDER BY s.event_ts DESC+ LIMIT 1 BY s.project_id, toDate(s.timestamp), s.name, s.id+ ) s${eventsJoin}- ${whereClause}+ ${outerWhereClause}${orderByToClickhouseSql(orderBy ?? null, scoresTableUiColumnDefinitionsFromEvents)}${limit !== undefined && offset !== undefined ? `limit {limit: Int32} offset {offset: Int32}` : ""}`;diff --git a/web/src/__tests__/server/scores-trpc.servertest.ts b/web/src/__tests__/server/scores-trpc.servertest.tsindex 731de706fc..4ae3f366a9 100644--- a/web/src/__tests__/server/scores-trpc.servertest.ts+++ b/web/src/__tests__/server/scores-trpc.servertest.ts@@ -174,7 +174,7 @@ describe("scores trpc", () => {// toDate(timestamp) prune must drop other day-buckets pre-dedup, while the// exact timestamp bound (re-applied post-dedup) must still exclude an// in-bucket-but-out-of-window row. Regression guard for- // buildScoresCountDatePrune.+ // buildScoresDatePrune.const from = new Date("2024-06-15T10:00:00.000Z");const to = new Date("2024-06-15T14:00:00.000Z");