Skip to content

πŸͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

perf(scores): dedup UI count without ClickHouse FINAL (#17404)

Niklas Semmler2bbd69f683a6
diff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.ts
index 761e28954a..f5cc7c4804 100644
--- a/packages/shared/src/server/repositories/scores.ts
+++ b/packages/shared/src/server/repositories/scores.ts
@@ -1438,6 +1438,62 @@ const scoresTraceFilterEventsMapping = [
},
];
+// Score columns the count path reconstructs to their latest version via
+// argMax(col, event_ts), replacing FINAL. id / project_id / name are the
+// grouping keys and so are selected directly, not here.
+const SCORE_COUNT_DEDUP_COLUMNS = [
+ "timestamp",
+ "environment",
+ "trace_id",
+ "observation_id",
+ "session_id",
+ "evaluator_id",
+ "evaluation_rule_id",
+ "value",
+ "source",
+ "comment",
+ "author_user_id",
+ "data_type",
+ "string_value",
+ "metadata",
+] as const;
+
+/**
+ * Coarse toDate(timestamp) prune for the count subquery's pre-dedup scan.
+ *
+ * The exact timestamp bound must NOT filter raw rows before dedup (a score's
+ * timestamp is mutable across versions, so it could drop the latest one); it is
+ * re-applied post-dedup in the outer WHERE. This prune only skips partitions /
+ * granules, keeping or dropping whole day-buckets, so the latest version always
+ * survives to the dedup. Operators are relaxed (> / < to >= / <=) and both
+ * sides wrapped in toDate() so boundary-day rows are never lost.
+ */
+const buildScoresCountDatePrune = (
+ filter: FilterState,
+): { query: string; params: Record<string, unknown> } => {
+ const timestampColumn = scoresTableUiColumnDefinitionsFromEvents.find(
+ (c) => c.uiTableId === "timestamp",
+ );
+ if (!timestampColumn) return { query: "", params: {} };
+
+ const timestampFilters = createFilterFromFilterState(
+ filter.filter((f) => matchesUiColumnMapping(timestampColumn, f.column)),
+ scoresTableUiColumnDefinitionsFromEvents,
+ scoresTableCols,
+ ).filter((f): f is DateTimeFilter => f instanceof DateTimeFilter);
+
+ const params: Record<string, unknown> = {};
+ const clauses = timestampFilters.map((f) => {
+ const dateOp = f.operator.startsWith(">") ? ">=" : "<=";
+ const varName = `scoresInnerDatePrune${clickhouseCompliantRandomCharacters()}`;
+ const column = `${f.tablePrefix ? `${f.tablePrefix}.` : ""}${f.field}`;
+ params[varName] = convertDateToClickhouseDateTime(new Date(f.value));
+ return `toDate(${column}) ${dateOp} toDate({${varName}: DateTime64(3, 'UTC')})`;
+ });
+
+ return { query: clauses.join(" AND "), params };
+};
+
/**
* v4 variant: scores query using a flat events CTE instead of the physical
* traces table. Trace-level filters and sort use a "traces" CTE built by
@@ -1485,6 +1541,10 @@ const getScoresUiGenericFromEvents = async <T>(props: {
);
const scoreOnlyFilterRes = scoreOnlyFilters.apply();
+ // Coarse pre-dedup prune for the count subquery; see buildScoresCountDatePrune.
+ const { query: innerDatePruneQuery, params: innerDatePruneParams } =
+ buildScoresCountDatePrune(filter);
+
// Trace-level filter entries from the frontend filter state
const traceFilterState = filter.filter((filterEntry) =>
scoresTraceFilterEventsMapping.some((col) =>
@@ -1540,10 +1600,7 @@ const getScoresUiGenericFromEvents = async <T>(props: {
: `LEFT ANY JOIN traces e ON s.trace_id = e.id`
: "";
- const select =
- props.select === "count"
- ? "count(*) as count"
- : `
+ const rowSelect = `
s.id,
s.project_id,
s.environment,
@@ -1574,15 +1631,58 @@ const getScoresUiGenericFromEvents = async <T>(props: {
${includeHasMetadataFlag ? ",length(mapKeys(s.metadata)) > 0 AS has_metadata" : ""}
`;
- const query = `
+ const whereClause = `
+ WHERE s.project_id = {projectId: String}
+ AND s.data_type IN ({dataTypes: Array(String)})
+ ${scoreOnlyFilterRes?.query ? `AND ${scoreOnlyFilterRes.query}` : ""}`;
+
+ // ── Count path: dedup without FINAL ──────────────────────────────────────
+ // Row reads dedup the ReplacingMergeTree with FINAL; for counts that is too
+ // slow. Instead reconstruct each score's latest version with one argMax pass:
+ // GROUP BY project_id, toDate(timestamp), name, id -- the sorting key
+ // FINAL collapses on
+ // argMax(col, event_ts) -- latest value per col
+ //
+ // Rule: filter AFTER dedup, never before. value / comment / timestamp /
+ // trace_id are all mutable across a score's versions, so filtering raw rows
+ // can drop the true-latest version and let argMax rebuild a stale one β€” a
+ // count that disagrees with the row list. Example, filter `value > 0.5`:
+ // v1 @10:02 value=0.9, v2 @10:07 value=0.1 (latest)
+ // filter-then-dedup -> v1 counted (WRONG)
+ // dedup-then-filter -> 0.1 excluded (matches FINAL)
+ //
+ // So every filter and the trace join runs in the OUTER query. The inner scan
+ // carries only a coarse toDate(timestamp) prune (see innerDatePruneQuery):
+ // whole buckets are kept or dropped, so the latest is never lost pre-dedup.
+ const query =
+ props.select === "count"
+ ? `
+ ${tracesCTEClause}
+ SELECT count(*) AS count
+ FROM (
+ SELECT
+ s.id,
+ s.project_id,
+ s.name,
+ ${SCORE_COUNT_DEDUP_COLUMNS.map(
+ (c) => `argMax(s.${c}, s.event_ts) AS ${c}`,
+ ).join(",\n ")}
+ FROM scores s
+ WHERE s.project_id = {projectId: String}
+ ${innerDatePruneQuery ? `AND ${innerDatePruneQuery}` : ""}
+ GROUP BY s.project_id, toDate(s.timestamp), s.name, s.id
+ ) s
+ ${eventsJoin}
+ WHERE s.data_type IN ({dataTypes: Array(String)})
+ ${scoreOnlyFilterRes?.query ? `AND ${scoreOnlyFilterRes.query}` : ""}
+ `
+ : `
${tracesCTEClause}
SELECT
- ${select}
+ ${rowSelect}
FROM scores s final
${eventsJoin}
- WHERE s.project_id = {projectId: String}
- AND s.data_type IN ({dataTypes: Array(String)})
- ${scoreOnlyFilterRes?.query ? `AND ${scoreOnlyFilterRes.query}` : ""}
+ ${whereClause}
${orderByToClickhouseSql(orderBy ?? null, scoresTableUiColumnDefinitionsFromEvents)}
${limit !== undefined && offset !== undefined ? `limit {limit: Int32} offset {offset: Int32}` : ""}
`;
@@ -1592,6 +1692,7 @@ const getScoresUiGenericFromEvents = async <T>(props: {
projectId,
dataTypes: LISTABLE_SCORE_TYPES,
...(scoreOnlyFilterRes ? scoreOnlyFilterRes.params : {}),
+ ...innerDatePruneParams,
...tracesCTEParams,
limit,
offset,
diff --git a/web/src/__tests__/server/scores-trpc.servertest.ts b/web/src/__tests__/server/scores-trpc.servertest.ts
index a4d06ba36d..731de706fc 100644
--- a/web/src/__tests__/server/scores-trpc.servertest.ts
+++ b/web/src/__tests__/server/scores-trpc.servertest.ts
@@ -169,6 +169,67 @@ describe("scores trpc", () => {
}
});
+ it("counts v4 scores within a timestamp window, pruning other day-buckets and honoring the exact bound", async () => {
+ // Exercises the FINAL-free count path's date handling: the coarse
+ // toDate(timestamp) prune must drop other day-buckets pre-dedup, while the
+ // exact timestamp bound (re-applied post-dedup) must still exclude an
+ // in-bucket-but-out-of-window row. Regression guard for
+ // buildScoresCountDatePrune.
+ const from = new Date("2024-06-15T10:00:00.000Z");
+ const to = new Date("2024-06-15T14:00:00.000Z");
+
+ await createScoresCh([
+ createTraceScore({
+ project_id: projectId,
+ name: "in-window",
+ timestamp: new Date("2024-06-15T12:00:00.000Z"),
+ }),
+ // Same day (survives the coarse toDate prune) but after `to` β€” only the
+ // exact post-dedup bound excludes it.
+ createTraceScore({
+ project_id: projectId,
+ name: "same-day-after-window",
+ timestamp: new Date("2024-06-15T20:00:00.000Z"),
+ }),
+ createTraceScore({
+ project_id: projectId,
+ name: "earlier-day",
+ timestamp: new Date("2024-06-10T12:00:00.000Z"),
+ }),
+ ]);
+
+ const payload = {
+ projectId,
+ filter: [
+ {
+ column: "timestamp",
+ type: "datetime" as const,
+ operator: ">=" as const,
+ value: from,
+ },
+ {
+ column: "timestamp",
+ type: "datetime" as const,
+ operator: "<" as const,
+ value: to,
+ },
+ ],
+ };
+ const [rows, count] = await Promise.all([
+ caller.scores.allFromEvents({
+ ...payload,
+ orderBy: { column: "timestamp", order: "DESC" },
+ page: 0,
+ limit: 50,
+ }),
+ caller.scores.countAllFromEvents({ ...payload, orderBy: null }),
+ ]);
+
+ expect(rows.scores.map((score) => score.name)).toEqual(["in-window"]);
+ expect(count.totalCount).toBe(rows.scores.length);
+ expect(count.totalCount).toBe(1);
+ });
+
it("returns the recorded evaluator and resolves legacy scores by rule assignment and score name", async () => {
const [recordedEvaluator, matchingEvaluator, otherEvaluator] =
await Promise.all(