πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(scores): dedup UI count without ClickHouse FINAL (#17404)
Niklas Semmler2bbd69f683a6
diff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.tsindex 761e28954a..f5cc7c4804 100644--- a/packages/shared/src/server/repositories/scores.ts+++ b/packages/shared/src/server/repositories/scores.ts@@ -1438,6 +1438,62 @@ const scoresTraceFilterEventsMapping = [},];+// Score columns the count path reconstructs to their latest version via+// argMax(col, event_ts), replacing FINAL. id / project_id / name are the+// grouping keys and so are selected directly, not here.+const SCORE_COUNT_DEDUP_COLUMNS = [+ "timestamp",+ "environment",+ "trace_id",+ "observation_id",+ "session_id",+ "evaluator_id",+ "evaluation_rule_id",+ "value",+ "source",+ "comment",+ "author_user_id",+ "data_type",+ "string_value",+ "metadata",+] as const;++/**+ * Coarse toDate(timestamp) prune for the count subquery's pre-dedup scan.+ *+ * The exact timestamp bound must NOT filter raw rows before dedup (a score's+ * timestamp is mutable across versions, so it could drop the latest one); it is+ * re-applied post-dedup in the outer WHERE. This prune only skips partitions /+ * granules, keeping or dropping whole day-buckets, so the latest version always+ * survives to the dedup. Operators are relaxed (> / < to >= / <=) and both+ * sides wrapped in toDate() so boundary-day rows are never lost.+ */+const buildScoresCountDatePrune = (+ filter: FilterState,+): { query: string; params: Record<string, unknown> } => {+ const timestampColumn = scoresTableUiColumnDefinitionsFromEvents.find(+ (c) => c.uiTableId === "timestamp",+ );+ if (!timestampColumn) return { query: "", params: {} };++ const timestampFilters = createFilterFromFilterState(+ filter.filter((f) => matchesUiColumnMapping(timestampColumn, f.column)),+ scoresTableUiColumnDefinitionsFromEvents,+ scoresTableCols,+ ).filter((f): f is DateTimeFilter => f instanceof DateTimeFilter);++ const params: Record<string, unknown> = {};+ const clauses = timestampFilters.map((f) => {+ const dateOp = f.operator.startsWith(">") ? ">=" : "<=";+ const varName = `scoresInnerDatePrune${clickhouseCompliantRandomCharacters()}`;+ const column = `${f.tablePrefix ? `${f.tablePrefix}.` : ""}${f.field}`;+ params[varName] = convertDateToClickhouseDateTime(new Date(f.value));+ return `toDate(${column}) ${dateOp} toDate({${varName}: DateTime64(3, 'UTC')})`;+ });++ return { query: clauses.join(" AND "), params };+};+/*** v4 variant: scores query using a flat events CTE instead of the physical* traces table. Trace-level filters and sort use a "traces" CTE built by@@ -1485,6 +1541,10 @@ const getScoresUiGenericFromEvents = async <T>(props: {);const scoreOnlyFilterRes = scoreOnlyFilters.apply();+ // Coarse pre-dedup prune for the count subquery; see buildScoresCountDatePrune.+ const { query: innerDatePruneQuery, params: innerDatePruneParams } =+ buildScoresCountDatePrune(filter);+// Trace-level filter entries from the frontend filter stateconst traceFilterState = filter.filter((filterEntry) =>scoresTraceFilterEventsMapping.some((col) =>@@ -1540,10 +1600,7 @@ const getScoresUiGenericFromEvents = async <T>(props: {: `LEFT ANY JOIN traces e ON s.trace_id = e.id`: "";- const select =- props.select === "count"- ? "count(*) as count"- : `+ const rowSelect = `s.id,s.project_id,s.environment,@@ -1574,15 +1631,58 @@ const getScoresUiGenericFromEvents = async <T>(props: {${includeHasMetadataFlag ? ",length(mapKeys(s.metadata)) > 0 AS has_metadata" : ""}`;- const query = `+ const whereClause = `+ WHERE s.project_id = {projectId: String}+ AND s.data_type IN ({dataTypes: Array(String)})+ ${scoreOnlyFilterRes?.query ? `AND ${scoreOnlyFilterRes.query}` : ""}`;++ // ββ Count path: dedup without FINAL ββββββββββββββββββββββββββββββββββββββ+ // Row reads dedup the ReplacingMergeTree with FINAL; for counts that is too+ // slow. Instead reconstruct each score's latest version with one argMax pass:+ // GROUP BY project_id, toDate(timestamp), name, id -- the sorting key+ // FINAL collapses on+ // argMax(col, event_ts) -- latest value per col+ //+ // Rule: filter AFTER dedup, never before. value / comment / timestamp /+ // trace_id are all mutable across a score's versions, so filtering raw rows+ // can drop the true-latest version and let argMax rebuild a stale one β a+ // count that disagrees with the row list. Example, filter `value > 0.5`:+ // v1 @10:02 value=0.9, v2 @10:07 value=0.1 (latest)+ // filter-then-dedup -> v1 counted (WRONG)+ // dedup-then-filter -> 0.1 excluded (matches FINAL)+ //+ // So every filter and the trace join runs in the OUTER query. The inner scan+ // carries only a coarse toDate(timestamp) prune (see innerDatePruneQuery):+ // whole buckets are kept or dropped, so the latest is never lost pre-dedup.+ const query =+ props.select === "count"+ ? `+ ${tracesCTEClause}+ SELECT count(*) AS count+ FROM (+ SELECT+ s.id,+ s.project_id,+ s.name,+ ${SCORE_COUNT_DEDUP_COLUMNS.map(+ (c) => `argMax(s.${c}, s.event_ts) AS ${c}`,+ ).join(",\n ")}+ FROM scores s+ WHERE s.project_id = {projectId: String}+ ${innerDatePruneQuery ? `AND ${innerDatePruneQuery}` : ""}+ GROUP BY s.project_id, toDate(s.timestamp), s.name, s.id+ ) s+ ${eventsJoin}+ WHERE s.data_type IN ({dataTypes: Array(String)})+ ${scoreOnlyFilterRes?.query ? `AND ${scoreOnlyFilterRes.query}` : ""}+ `+ : `${tracesCTEClause}SELECT- ${select}+ ${rowSelect}FROM scores s final${eventsJoin}- WHERE s.project_id = {projectId: String}- AND s.data_type IN ({dataTypes: Array(String)})- ${scoreOnlyFilterRes?.query ? `AND ${scoreOnlyFilterRes.query}` : ""}+ ${whereClause}${orderByToClickhouseSql(orderBy ?? null, scoresTableUiColumnDefinitionsFromEvents)}${limit !== undefined && offset !== undefined ? `limit {limit: Int32} offset {offset: Int32}` : ""}`;@@ -1592,6 +1692,7 @@ const getScoresUiGenericFromEvents = async <T>(props: {projectId,dataTypes: LISTABLE_SCORE_TYPES,...(scoreOnlyFilterRes ? scoreOnlyFilterRes.params : {}),+ ...innerDatePruneParams,...tracesCTEParams,limit,offset,diff --git a/web/src/__tests__/server/scores-trpc.servertest.ts b/web/src/__tests__/server/scores-trpc.servertest.tsindex a4d06ba36d..731de706fc 100644--- a/web/src/__tests__/server/scores-trpc.servertest.ts+++ b/web/src/__tests__/server/scores-trpc.servertest.ts@@ -169,6 +169,67 @@ describe("scores trpc", () => {}});+ it("counts v4 scores within a timestamp window, pruning other day-buckets and honoring the exact bound", async () => {+ // Exercises the FINAL-free count path's date handling: the coarse+ // toDate(timestamp) prune must drop other day-buckets pre-dedup, while the+ // exact timestamp bound (re-applied post-dedup) must still exclude an+ // in-bucket-but-out-of-window row. Regression guard for+ // buildScoresCountDatePrune.+ const from = new Date("2024-06-15T10:00:00.000Z");+ const to = new Date("2024-06-15T14:00:00.000Z");++ await createScoresCh([+ createTraceScore({+ project_id: projectId,+ name: "in-window",+ timestamp: new Date("2024-06-15T12:00:00.000Z"),+ }),+ // Same day (survives the coarse toDate prune) but after `to` β only the+ // exact post-dedup bound excludes it.+ createTraceScore({+ project_id: projectId,+ name: "same-day-after-window",+ timestamp: new Date("2024-06-15T20:00:00.000Z"),+ }),+ createTraceScore({+ project_id: projectId,+ name: "earlier-day",+ timestamp: new Date("2024-06-10T12:00:00.000Z"),+ }),+ ]);++ const payload = {+ projectId,+ filter: [+ {+ column: "timestamp",+ type: "datetime" as const,+ operator: ">=" as const,+ value: from,+ },+ {+ column: "timestamp",+ type: "datetime" as const,+ operator: "<" as const,+ value: to,+ },+ ],+ };+ const [rows, count] = await Promise.all([+ caller.scores.allFromEvents({+ ...payload,+ orderBy: { column: "timestamp", order: "DESC" },+ page: 0,+ limit: 50,+ }),+ caller.scores.countAllFromEvents({ ...payload, orderBy: null }),+ ]);++ expect(rows.scores.map((score) => score.name)).toEqual(["in-window"]);+ expect(count.totalCount).toBe(rows.scores.length);+ expect(count.totalCount).toBe(1);+ });+it("returns the recorded evaluator and resolves legacy scores by rule assignment and score name", async () => {const [recordedEvaluator, matchingEvaluator, otherEvaluator] =await Promise.all(