Skip to content

πŸͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

perf(scores): add selective-seek path for scores dedup reads (#17842)

Niklas Semmler72294f582cb1
diff --git a/packages/shared/src/server/queries/clickhouse-sql/score-seek-eligibility.test.ts b/packages/shared/src/server/queries/clickhouse-sql/score-seek-eligibility.test.ts
new file mode 100644
index 0000000000..0cef3e201f
--- /dev/null
+++ b/packages/shared/src/server/queries/clickhouse-sql/score-seek-eligibility.test.ts
@@ -0,0 +1,232 @@
+import { describe, expect, it } from "vitest";
+
+import {
+ FilterList,
+ NullFilter,
+ NumberFilter,
+ StringFilter,
+ StringOptionsFilter,
+} from "./clickhouse-filter";
+import {
+ isSeekEligibleFilter,
+ scoreOnlyFiltersAreSeekEligible,
+} from "./score-seek-eligibility";
+
+const projectIdFilter = () =>
+ new StringFilter({
+ clickhouseTable: "scores",
+ field: "project_id",
+ operator: "=",
+ value: "p1",
+ });
+
+const traceIdEq = (value = "trace-1") =>
+ new StringFilter({
+ clickhouseTable: "scores",
+ field: "trace_id",
+ operator: "=",
+ value,
+ tablePrefix: "s",
+ });
+
+const observationIdEq = (value: string, emptyEqualsNull = true) =>
+ new StringFilter({
+ clickhouseTable: "scores",
+ field: "observation_id",
+ operator: "=",
+ value,
+ tablePrefix: "s",
+ emptyEqualsNull,
+ });
+
+const observationIdIn = (values: string[], emptyEqualsNull = true) =>
+ new StringOptionsFilter({
+ clickhouseTable: "scores",
+ field: "observation_id",
+ operator: "any of",
+ values,
+ tablePrefix: "s",
+ emptyEqualsNull,
+ });
+
+const nameIn = (values: string[]) =>
+ new StringOptionsFilter({
+ clickhouseTable: "scores",
+ field: "name",
+ operator: "any of",
+ values,
+ tablePrefix: "s",
+ });
+
+const idEq = (value = "id-1") =>
+ new StringFilter({
+ clickhouseTable: "scores",
+ field: "id",
+ operator: "=",
+ value,
+ tablePrefix: "s",
+ });
+
+const nameEq = (value = "accuracy") =>
+ new StringFilter({
+ clickhouseTable: "scores",
+ field: "name",
+ operator: "=",
+ value,
+ tablePrefix: "s",
+ });
+
+const sessionIdEq = (value = "sess-1") =>
+ new StringFilter({
+ clickhouseTable: "scores",
+ field: "session_id",
+ operator: "=",
+ value,
+ tablePrefix: "s",
+ emptyEqualsNull: true,
+ });
+
+const evaluatorIdEq = (value = "eval-1") =>
+ new StringFilter({
+ clickhouseTable: "scores",
+ field: "evaluator_id",
+ operator: "=",
+ value,
+ tablePrefix: "s",
+ emptyEqualsNull: true,
+ });
+
+const evaluationRuleIdIn = (values: string[]) =>
+ new StringOptionsFilter({
+ clickhouseTable: "scores",
+ field: "evaluation_rule_id",
+ operator: "any of",
+ values,
+ tablePrefix: "s",
+ emptyEqualsNull: true,
+ });
+
+const environmentNoneOf = () =>
+ new StringOptionsFilter({
+ clickhouseTable: "scores",
+ field: "environment",
+ operator: "none of",
+ values: ["default"],
+ tablePrefix: "s",
+ });
+
+const valueRange = () =>
+ new NumberFilter({
+ clickhouseTable: "scores",
+ field: "value",
+ operator: ">",
+ value: 0.5,
+ tablePrefix: "s",
+ });
+
+describe("isSeekEligibleFilter", () => {
+ it("accepts = on id / trace_id / name", () => {
+ expect(isSeekEligibleFilter(idEq())).toBe(true);
+ expect(isSeekEligibleFilter(traceIdEq())).toBe(true);
+ expect(isSeekEligibleFilter(nameEq())).toBe(true);
+ });
+
+ it("accepts IN on name / observation_id (no empty member)", () => {
+ expect(isSeekEligibleFilter(nameIn(["accuracy"]))).toBe(true);
+ expect(isSeekEligibleFilter(observationIdIn(["obs-1", "obs-2"]))).toBe(
+ true,
+ );
+ });
+
+ it("accepts the newly-indexed bloom columns (session / evaluator / rule)", () => {
+ expect(isSeekEligibleFilter(sessionIdEq())).toBe(true);
+ expect(isSeekEligibleFilter(evaluatorIdEq())).toBe(true);
+ expect(isSeekEligibleFilter(evaluationRuleIdIn(["r1"]))).toBe(true);
+ });
+
+ it("rejects the emptyEqualsNull degradation on a newly-indexed column", () => {
+ expect(isSeekEligibleFilter(sessionIdEq(""))).toBe(false);
+ expect(isSeekEligibleFilter(evaluationRuleIdIn(["", "r1"]))).toBe(false);
+ });
+
+ it("accepts a non-empty observation_id = despite emptyEqualsNull", () => {
+ // A non-empty value does not trigger the OR IS NULL branch.
+ expect(isSeekEligibleFilter(observationIdEq("obs-1"))).toBe(true);
+ });
+
+ it("rejects the emptyEqualsNull OR IS NULL degradations", () => {
+ // observation_id = '' compiles to (... OR observation_id IS NULL).
+ expect(isSeekEligibleFilter(observationIdEq(""))).toBe(false);
+ // observation_id IN ('', ...) adds an OR IS NULL branch.
+ expect(isSeekEligibleFilter(observationIdIn(["", "obs-1"]))).toBe(false);
+ });
+
+ it("rejects an empty IN set", () => {
+ expect(isSeekEligibleFilter(nameIn([]))).toBe(false);
+ });
+
+ it("rejects non-index columns and non-equality operators", () => {
+ expect(isSeekEligibleFilter(environmentNoneOf())).toBe(false);
+ expect(isSeekEligibleFilter(valueRange())).toBe(false);
+ // project_id equality exists on every query but is not a seek driver.
+ expect(isSeekEligibleFilter(projectIdFilter())).toBe(false);
+ });
+
+ it("rejects trace_id 'contains' (not an index-prunable operator)", () => {
+ expect(
+ isSeekEligibleFilter(
+ new StringFilter({
+ clickhouseTable: "scores",
+ field: "trace_id",
+ operator: "contains",
+ value: "trace",
+ tablePrefix: "s",
+ }),
+ ),
+ ).toBe(false);
+ });
+
+ it("rejects a null filter on an index column", () => {
+ expect(
+ isSeekEligibleFilter(
+ new NullFilter({
+ clickhouseTable: "scores",
+ field: "observation_id",
+ operator: "is null",
+ tablePrefix: "s",
+ emptyEqualsNull: true,
+ }),
+ ),
+ ).toBe(false);
+ });
+});
+
+describe("scoreOnlyFiltersAreSeekEligible", () => {
+ it("is eligible when trace_id = is ANDed with a non-prunable observation OR-null sibling", () => {
+ const filters = new FilterList([
+ projectIdFilter(),
+ traceIdEq(),
+ observationIdIn(["", "obs-1"]), // not prunable on its own
+ ]);
+ expect(scoreOnlyFiltersAreSeekEligible(filters)).toBe(true);
+ });
+
+ it("is ineligible for environment-only / value-range filters", () => {
+ expect(
+ scoreOnlyFiltersAreSeekEligible(
+ new FilterList([projectIdFilter(), environmentNoneOf()]),
+ ),
+ ).toBe(false);
+ expect(
+ scoreOnlyFiltersAreSeekEligible(
+ new FilterList([projectIdFilter(), valueRange()]),
+ ),
+ ).toBe(false);
+ });
+
+ it("is ineligible for the bare project filter (no user predicate)", () => {
+ expect(
+ scoreOnlyFiltersAreSeekEligible(new FilterList([projectIdFilter()])),
+ ).toBe(false);
+ });
+});
diff --git a/packages/shared/src/server/queries/clickhouse-sql/score-seek-eligibility.ts b/packages/shared/src/server/queries/clickhouse-sql/score-seek-eligibility.ts
new file mode 100644
index 0000000000..38717de27d
--- /dev/null
+++ b/packages/shared/src/server/queries/clickhouse-sql/score-seek-eligibility.ts
@@ -0,0 +1,68 @@
+import {
+ type Filter,
+ type FilterList,
+ StringFilter,
+ StringOptionsFilter,
+} from "./clickhouse-filter";
+
+// Score columns whose equality / IN predicate an index can prune, letting the
+// selective-seek path collect candidate dedup keys without a full-project scan:
+// - id β†’ idx_id (bloom_filter)
+// - trace_id / observation_id β†’ idx_project_trace_observation (bloom_filter)
+// - session_id β†’ idx_project_session (bloom_filter)
+// - dataset_run_id β†’ idx_project_dataset_run (bloom_filter)
+// - evaluator_id β†’ idx_evaluator_id (bloom_filter)
+// - evaluation_rule_id β†’ idx_evaluation_rule_id (bloom_filter)
+// - name β†’ PRIMARY KEY / ORDER BY (project_id, toDate(timestamp), name)
+//
+// created_at carries a minmax index (idx_created_at), which prunes ranges rather
+// than equality/IN, so it is intentionally excluded here β€” the seek predicate
+// below only handles `=` / `IN`.
+const SEEK_ELIGIBLE_FIELDS = new Set([
+ "id",
+ "trace_id",
+ "observation_id",
+ "session_id",
+ "dataset_run_id",
+ "evaluator_id",
+ "evaluation_rule_id",
+ "name",
+]);
+
+/**
+ * Is this single top-level AND-conjunct index-prunable for the seek phase?
+ *
+ * Eligible only for a clean `=` (StringFilter) or `IN` (StringOptionsFilter
+ * "any of") on an index-backed column. The `emptyEqualsNull` degradation is
+ * rejected: `observation_id = ''` / `IN ('', …)` compiles to
+ * `(… OR observation_id IS NULL)` (see StringFilter / StringOptionsFilter
+ * apply()), and a bloom filter cannot skip granules across an `OR … IS NULL`.
+ *
+ * Conjuncts are judged individually, so a non-prunable sibling (e.g.
+ * `observation_id IN (…) OR observation_id IS NULL`) does not disqualify a query
+ * that also carries a prunable `trace_id = …`.
+ */
+export const isSeekEligibleFilter = (filter: Filter): boolean => {
+ if (filter.clickhouseTable !== "scores") return false;
+ if (!SEEK_ELIGIBLE_FIELDS.has(filter.field)) return false;
+
+ if (filter instanceof StringFilter && filter.operator === "=") {
+ return !(filter.emptyEqualsNull && filter.value === "");
+ }
+
+ if (filter instanceof StringOptionsFilter && filter.operator === "any of") {
+ if (filter.values.length === 0) return false;
+ return !(filter.emptyEqualsNull && filter.values.includes(""));
+ }
+
+ return false;
+};
+
+/**
+ * The seek path is worthwhile when at least one score-only conjunct is
+ * index-prunable. The seek still carries the *entire* score-only predicate
+ * (a superset collection); this only decides whether to take the seek path at
+ * all versus the single-pass fallback.
+ */
+export const scoreOnlyFiltersAreSeekEligible = (filters: FilterList): boolean =>
+ filters.some(isSeekEligibleFilter);
diff --git a/packages/shared/src/server/queries/index.ts b/packages/shared/src/server/queries/index.ts
index fbcbe58e88..cd8ee8c16f 100644
--- a/packages/shared/src/server/queries/index.ts
+++ b/packages/shared/src/server/queries/index.ts
@@ -109,3 +109,7 @@ export {
type EventsObservationRowSelectionInput,
} from "./clickhouse-sql/events-observation-row-selection";
export { extractTimeFilter } from "./clickhouse-sql/filter-utils";
+export {
+ isSeekEligibleFilter,
+ scoreOnlyFiltersAreSeekEligible,
+} from "./clickhouse-sql/score-seek-eligibility";
diff --git a/packages/shared/src/server/repositories/__snapshots__/scores-selective-seek.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/scores-selective-seek.golden.test.ts.snap
new file mode 100644
index 0000000000..796677a4d3
--- /dev/null
+++ b/packages/shared/src/server/repositories/__snapshots__/scores-selective-seek.golden.test.ts.snap
@@ -0,0 +1,241 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`scores selective-seek: emitted SQL > full SQL text (one representative per branch) > eligible (trace_id =): count 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "seek-project",
+ "param2": "seek-project",
+ "param3": "T2",
+ "param4": [
+ "NUMERIC",
+ "BOOLEAN",
+ "CATEGORICAL",
+ "TEXT",
+ ],
+ },
+ "route": "unknown",
+ "sql": "SELECT count(*) AS count
+FROM
+(
+ SELECT
+ s.id,
+ s.project_id,
+ s.name,
+ argMax(s.timestamp, s.event_ts) AS timestamp,
+ argMax(s.environment, s.event_ts) AS environment,
+ argMax(s.trace_id, s.event_ts) AS trace_id,
+ argMax(s.observation_id, s.event_ts) AS observation_id,
+ argMax(s.session_id, s.event_ts) AS session_id,
+ argMax(s.evaluator_id, s.event_ts) AS evaluator_id,
+ argMax(s.evaluation_rule_id, s.event_ts) AS evaluation_rule_id,
+ argMax(s.value, s.event_ts) AS value,
+ argMax(s.source, s.event_ts) AS source,
+ argMax(s.comment, s.event_ts) AS comment,
+ argMax(s.author_user_id, s.event_ts) AS author_user_id,
+ argMax(s.data_type, s.event_ts) AS data_type,
+ argMax(s.string_value, s.event_ts) AS string_value,
+ argMax(s.metadata, s.event_ts) AS metadata
+ FROM scores AS s
+ WHERE (s.project_id = {param1:String}) AND ((s.project_id, toDate(s.timestamp), s.name, s.id) IN (
+ SELECT DISTINCT
+ s.project_id,
+ toDate(s.timestamp),
+ s.name,
+ s.id
+ FROM scores AS s
+ WHERE (s.project_id = {param1:String}) AND (project_id = {param2:String}) AND (s.trace_id = {param3:String})
+ ))
+ GROUP BY
+ s.project_id,
+ toDate(s.timestamp),
+ s.name,
+ s.id
+) AS s
+WHERE (s.data_type IN ({param4:Array(String)})) AND (project_id = {param2:String}) AND (s.trace_id = {param3:String})",
+ },
+]
+`;
+
+exports[`scores selective-seek: emitted SQL > full SQL text (one representative per branch) > eligible (trace_id =): rows 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "seek-project",
+ "param2": "seek-project",
+ "param3": "T2",
+ "param4": [
+ "NUMERIC",
+ "BOOLEAN",
+ "CATEGORICAL",
+ "TEXT",
+ ],
+ },
+ "route": "unknown",
+ "sql": "SELECT
+ s.id,
+ s.project_id,
+ s.environment,
+ s.name,
+ s.value,
+ s.string_value,
+ s.timestamp,
+ s.source,
+ s.data_type,
+ s.comment,
+ s.evaluator_id,
+ s.trace_id,
+ s.session_id,
+ s.dataset_run_id,
+ s.observation_id,
+ s.author_user_id,
+ s.created_at,
+ s.updated_at,
+ s.config_id,
+ s.queue_id,
+ s.execution_trace_id,
+ s.ingestion_api_key,
+ s.ingestion_sdk_name,
+ s.ingestion_sdk_version,
+ s.is_deleted,
+ s.event_ts,
+ length(mapKeys(s.metadata)) > 0 AS has_metadata
+FROM
+(
+ SELECT *
+ FROM scores AS s
+ WHERE (s.project_id = {param1:String}) AND ((s.project_id, toDate(s.timestamp), s.name, s.id) IN (
+ SELECT DISTINCT
+ s.project_id,
+ toDate(s.timestamp),
+ s.name,
+ s.id
+ FROM scores AS s
+ WHERE (s.project_id = {param1:String}) AND (project_id = {param2:String}) AND (s.trace_id = {param3:String})
+ ))
+ ORDER BY s.event_ts DESC
+ LIMIT 1 BY
+ s.project_id,
+ toDate(s.timestamp),
+ s.name,
+ s.id
+) AS s
+WHERE (s.data_type IN ({param4:Array(String)})) AND (project_id = {param2:String}) AND (s.trace_id = {param3:String})
+ORDER BY s.timestamp DESC",
+ },
+]
+`;
+
+exports[`scores selective-seek: emitted SQL > full SQL text (one representative per branch) > ineligible (value range): count 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "seek-project",
+ "param2": [
+ "NUMERIC",
+ "BOOLEAN",
+ "CATEGORICAL",
+ "TEXT",
+ ],
+ "param3": "seek-project",
+ "param4": "0.5",
+ },
+ "route": "unknown",
+ "sql": "SELECT count(*) AS count
+FROM
+(
+ SELECT
+ s.id,
+ s.project_id,
+ s.name,
+ argMax(s.timestamp, s.event_ts) AS timestamp,
+ argMax(s.environment, s.event_ts) AS environment,
+ argMax(s.trace_id, s.event_ts) AS trace_id,
+ argMax(s.observation_id, s.event_ts) AS observation_id,
+ argMax(s.session_id, s.event_ts) AS session_id,
+ argMax(s.evaluator_id, s.event_ts) AS evaluator_id,
+ argMax(s.evaluation_rule_id, s.event_ts) AS evaluation_rule_id,
+ argMax(s.value, s.event_ts) AS value,
+ argMax(s.source, s.event_ts) AS source,
+ argMax(s.comment, s.event_ts) AS comment,
+ argMax(s.author_user_id, s.event_ts) AS author_user_id,
+ argMax(s.data_type, s.event_ts) AS data_type,
+ argMax(s.string_value, s.event_ts) AS string_value,
+ argMax(s.metadata, s.event_ts) AS metadata
+ FROM scores AS s
+ WHERE s.project_id = {param1:String}
+ GROUP BY
+ s.project_id,
+ toDate(s.timestamp),
+ s.name,
+ s.id
+) AS s
+WHERE (s.data_type IN ({param2:Array(String)})) AND (project_id = {param3:String}) AND (s.value > {param4:Decimal64(12)})",
+ },
+]
+`;
+
+exports[`scores selective-seek: emitted SQL > full SQL text (one representative per branch) > ineligible (value range): rows 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "seek-project",
+ "param2": [
+ "NUMERIC",
+ "BOOLEAN",
+ "CATEGORICAL",
+ "TEXT",
+ ],
+ "param3": "seek-project",
+ "param4": "0.5",
+ },
+ "route": "unknown",
+ "sql": "SELECT
+ s.id,
+ s.project_id,
+ s.environment,
+ s.name,
+ s.value,
+ s.string_value,
+ s.timestamp,
+ s.source,
+ s.data_type,
+ s.comment,
+ s.evaluator_id,
+ s.trace_id,
+ s.session_id,
+ s.dataset_run_id,
+ s.observation_id,
+ s.author_user_id,
+ s.created_at,
+ s.updated_at,
+ s.config_id,
+ s.queue_id,
+ s.execution_trace_id,
+ s.ingestion_api_key,
+ s.ingestion_sdk_name,
+ s.ingestion_sdk_version,
+ s.is_deleted,
+ s.event_ts,
+ length(mapKeys(s.metadata)) > 0 AS has_metadata
+FROM
+(
+ SELECT *
+ FROM scores AS s
+ WHERE s.project_id = {param1:String}
+ ORDER BY s.event_ts DESC
+ LIMIT 1 BY
+ s.project_id,
+ toDate(s.timestamp),
+ s.name,
+ s.id
+) AS s
+WHERE (s.data_type IN ({param2:Array(String)})) AND (project_id = {param3:String}) AND (s.value > {param4:Decimal64(12)})
+ORDER BY s.timestamp DESC",
+ },
+]
+`;
diff --git a/packages/shared/src/server/repositories/scores-selective-seek.golden.test.ts b/packages/shared/src/server/repositories/scores-selective-seek.golden.test.ts
new file mode 100644
index 0000000000..b87e6ccd8d
--- /dev/null
+++ b/packages/shared/src/server/repositories/scores-selective-seek.golden.test.ts
@@ -0,0 +1,346 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+// A hand-written exec-seam mock (no importOriginal) records each query and keeps
+// the real repositories/queries barrels out of the mock resolution order, which
+// otherwise reorders module init into the barrel cycle.
+const recorder = vi.hoisted(() => {
+ const captured: Array<{
+ fn: string;
+ query: string;
+ params?: Record<string, unknown>;
+ tags?: unknown;
+ }> = [];
+ // Unique per call β€” a constant would collide the param names of two filters of
+ // the same class (e.g. the default project_id and a trace_id equality), and the
+ // later bind would clobber the earlier one.
+ let n = 0;
+ const uid = () => `v${n++}`;
+ return { captured, uid };
+});
+
+vi.mock("./clickhouse", () => ({
+ queryClickhouse: vi.fn(async (opts: any) => {
+ recorder.captured.push({ fn: "queryClickhouse", ...opts });
+ // The count path reads rows[0].count; give it a benign shape. The rows path
+ // maps over the result, so an empty array is correct there.
+ return /count\(\*\) AS count/.test(opts.query) ? [{ count: "0" }] : [];
+ }),
+ queryClickhouseStream: vi.fn(),
+ queryClickhouseExecRaw: vi.fn(),
+ commandClickhouse: vi.fn(),
+ upsertClickhouse: vi.fn(),
+ parseClickhouseUTCDateTimeFormat: vi.fn(),
+ clickhouseCompliantRandomCharacters: vi.fn(() => recorder.uid()),
+ BLOB_EXPORT_PARQUET_CLICKHOUSE_SETTINGS: {},
+}));
+
+// query-options imports the server barrel, which cannot finish initializing when
+// the graph is entered from this repository file; the events scores path never
+// calls it.
+vi.mock("../queries/clickhouse-sql/query-options", () => ({
+ shouldSkipObservationsFinal: vi.fn().mockResolvedValue(false),
+}));
+
+// Import via the server barrel so it initializes in the correct order; entering
+// the graph directly through ./scores trips a barrel init cycle.
+import {
+ getScoresUiCountFromEvents,
+ getScoresUiTableFromEvents,
+} from "../index";
+
+import type { FilterState } from "../../types";
+import {
+ clickhouseFormatAvailable,
+ clickhouseLocalAvailable,
+ executeClickhouseLocal,
+ normalizeCapturedQueries,
+ substituteNamedParams,
+} from "./goldenHarness";
+
+const PROJECT_ID = "seek-project";
+const ORDER_BY = { column: "timestamp", order: "DESC" as const };
+
+// ── Filter fixtures ─────────────────────────────────────────────────────────
+const traceIdEq = (value: string): FilterState => [
+ { column: "traceId", type: "string", operator: "=", value },
+];
+const observationIdEq = (value: string): FilterState => [
+ { column: "observationId", type: "string", operator: "=", value },
+];
+const nameIn = (values: string[]): FilterState => [
+ { column: "name", type: "stringOptions", operator: "any of", value: values },
+];
+const valueGt = (value: number): FilterState => [
+ { column: "value", type: "number", operator: ">", value },
+];
+const environmentNoneOf = (values: string[]): FilterState => [
+ {
+ column: "environment",
+ type: "stringOptions",
+ operator: "none of",
+ value: values,
+ },
+];
+
+const resetCaptures = () => {
+ recorder.captured.length = 0;
+};
+
+async function captureCountSql(filter: FilterState) {
+ resetCaptures();
+ await getScoresUiCountFromEvents({
+ projectId: PROJECT_ID,
+ filter,
+ orderBy: null,
+ });
+ expect(recorder.captured).toHaveLength(1);
+ return recorder.captured[0];
+}
+
+async function captureRowsSql(filter: FilterState) {
+ resetCaptures();
+ await getScoresUiTableFromEvents({
+ projectId: PROJECT_ID,
+ filter,
+ orderBy: ORDER_BY,
+ });
+ expect(recorder.captured).toHaveLength(1);
+ return recorder.captured[0];
+}
+
+const SEEK_TUPLE_IN =
+ /\(s\.project_id, toDate\(s\.timestamp\), s\.name, s\.id\) IN \(/;
+
+const describeWithFormat = clickhouseFormatAvailable()
+ ? describe
+ : describe.skip;
+const describeWithLocal = clickhouseLocalAvailable() ? describe : describe.skip;
+
+if (!clickhouseFormatAvailable()) {
+ console.warn(
+ "[scores-seek] `clickhouse format` unavailable β€” skipping golden SQL tests.",
+ );
+}
+if (!clickhouseLocalAvailable()) {
+ console.warn(
+ "[scores-seek] `clickhouse local` unavailable β€” skipping parity tests.",
+ );
+}
+
+// ── Emitted SQL shape ───────────────────────────────────────────────────────
+// The full matrix asserts *routing* only (does the seek phase fire for this
+// filter?); whether a filter is eligible is unit-tested in
+// score-seek-eligibility.test.ts. The full SQL text is locked by a single
+// eligible + single ineligible snapshot below β€” snapshotting every shape just
+// duplicates the same query skeleton.
+describeWithFormat("scores selective-seek: emitted SQL", () => {
+ beforeEach(() => resetCaptures());
+
+ const eligible = [
+ { name: "trace_id =", filter: traceIdEq("T2") },
+ { name: "observation_id =", filter: observationIdEq("O1") },
+ { name: "name IN", filter: nameIn(["n1"]) },
+ ];
+ const ineligible = [
+ { name: "value range", filter: valueGt(0.5) },
+ { name: "environment none of", filter: environmentNoneOf(["dev"]) },
+ { name: "no filter", filter: [] as FilterState },
+ ];
+
+ describe("eligible β†’ seek phase present", () => {
+ for (const { name, filter } of eligible) {
+ it(`count: ${name}`, async () => {
+ const q = await captureCountSql(filter);
+ expect(q.query).toContain("SELECT DISTINCT");
+ expect(q.query).toMatch(SEEK_TUPLE_IN);
+ });
+ it(`rows: ${name}`, async () => {
+ const q = await captureRowsSql(filter);
+ expect(q.query).toContain("SELECT DISTINCT");
+ expect(q.query).toMatch(SEEK_TUPLE_IN);
+ });
+ }
+ });
+
+ describe("ineligible β†’ fallback unchanged, no seek", () => {
+ for (const { name, filter } of ineligible) {
+ it(`count: ${name}`, async () => {
+ const q = await captureCountSql(filter);
+ expect(q.query).not.toContain("SELECT DISTINCT");
+ expect(q.query).not.toMatch(SEEK_TUPLE_IN);
+ });
+ it(`rows: ${name}`, async () => {
+ const q = await captureRowsSql(filter);
+ expect(q.query).not.toContain("SELECT DISTINCT");
+ expect(q.query).not.toMatch(SEEK_TUPLE_IN);
+ });
+ }
+ });
+
+ // One representative shape per branch locks the full SQL text; the routing
+ // matrix above covers the remaining shapes structurally.
+ describe("full SQL text (one representative per branch)", () => {
+ it("eligible (trace_id =): count", async () => {
+ const q = await captureCountSql(traceIdEq("T2"));
+ expect(normalizeCapturedQueries([q])).toMatchSnapshot();
+ });
+ it("eligible (trace_id =): rows", async () => {
+ const q = await captureRowsSql(traceIdEq("T2"));
+ expect(normalizeCapturedQueries([q])).toMatchSnapshot();
+ });
+ it("ineligible (value range): count", async () => {
+ const q = await captureCountSql(valueGt(0.5));
+ expect(normalizeCapturedQueries([q])).toMatchSnapshot();
+ });
+ it("ineligible (value range): rows", async () => {
+ const q = await captureRowsSql(valueGt(0.5));
+ expect(normalizeCapturedQueries([q])).toMatchSnapshot();
+ });
+ });
+});
+
+// ── Correctness parity vs FINAL over a multi-version fixture ─────────────────
+
+// scores columns referenced by the count (argMax) and rows (SELECT *) paths.
+const SCORES_DDL = `
+ CREATE TABLE scores (
+ id String,
+ timestamp DateTime64(3),
+ project_id String,
+ trace_id String,
+ observation_id Nullable(String),
+ name String,
+ value Float64,
+ source String,
+ comment Nullable(String),
+ author_user_id Nullable(String),
+ config_id Nullable(String),
+ data_type String,
+ string_value Nullable(String),
+ queue_id Nullable(String),
+ environment String,
+ session_id Nullable(String),
+ dataset_run_id Nullable(String),
+ evaluator_id Nullable(String),
+ evaluation_rule_id Nullable(String),
+ execution_trace_id Nullable(String),
+ metadata Map(String, String),
+ ingestion_api_key String,
+ ingestion_sdk_name String,
+ ingestion_sdk_version String,
+ created_at DateTime64(3),
+ updated_at DateTime64(3),
+ event_ts DateTime64(3),
+ is_deleted UInt8,
+ INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+ INDEX idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1,
+ INDEX idx_project_session (project_id, session_id) TYPE bloom_filter(0.001) GRANULARITY 1,
+ INDEX idx_project_dataset_run (project_id, dataset_run_id) TYPE bloom_filter(0.001) GRANULARITY 1,
+ INDEX idx_created_at created_at TYPE minmax GRANULARITY 1,
+ INDEX idx_evaluator_id evaluator_id TYPE bloom_filter(0.001) GRANULARITY 1,
+ INDEX idx_evaluation_rule_id evaluation_rule_id TYPE bloom_filter(0.001) GRANULARITY 1
+ ) ENGINE = ReplacingMergeTree(event_ts, is_deleted)
+ PARTITION BY toYYYYMM(timestamp)
+ PRIMARY KEY (project_id, toDate(timestamp), name)
+ ORDER BY (project_id, toDate(timestamp), name, id);
+`;
+
+// (id, name, ts, trace_id, observation_id, value, event_ts). Every row:
+// project_id = PROJECT_ID, data_type = NUMERIC (listable), is_deleted = 0.
+// a: trace_id mutates T1 -> T2 (latest), value 0.9 -> 0.1
+// b: single version, trace_id T1
+// c: two day-buckets (distinct dedup groups), trace_id T3
+// d: has observation_id O1
+// e: value mutates 0.9 -> 0.1 (latest) β€” exercises dedup-then-filter
+const ROWS: Array<
+ [string, string, string, string, string | null, number, number]
+> = [
+ ["a", "n1", "2026-01-10 10:00:00.000", "T1", null, 0.9, 1],
+ ["a", "n1", "2026-01-10 10:05:00.000", "T2", null, 0.1, 2],
+ ["b", "n2", "2026-01-10 09:00:00.000", "T1", null, 0.2, 1],
+ ["c", "n1", "2026-01-10 08:00:00.000", "T3", null, 0.5, 1],
+ ["c", "n1", "2026-01-11 08:00:00.000", "T3", null, 0.6, 1],
+ ["d", "n3", "2026-01-10 07:00:00.000", "T4", "O1", 0.4, 1],
+ ["e", "n4", "2026-01-10 06:00:00.000", "T5", null, 0.9, 1],
+ ["e", "n4", "2026-01-10 06:05:00.000", "T5", null, 0.1, 2],
+];
+
+const insertRows = () => {
+ const values = ROWS.map(
+ ([id, name, ts, traceId, obsId, value, eventTs]) =>
+ `('${id}', '${ts}', '${PROJECT_ID}', '${traceId}', ${
+ obsId === null ? "NULL" : `'${obsId}'`
+ }, '${name}', ${value}, 'API', NULL, NULL, NULL, 'NUMERIC', NULL, NULL, 'default', NULL, NULL, NULL, NULL, NULL, map(), 'pk', 'sdk', '1', '${ts}', '${ts}', toDateTime64(${eventTs}, 3), 0)`,
+ ).join(",\n");
+ return `INSERT INTO scores VALUES ${values};`;
+};
+
+// FINAL ground truth: dedup on the ORDER BY key keeping max event_ts, then apply
+// the same post-dedup predicate the outer WHERE applies.
+const finalCount = (predicate: string) =>
+ `SELECT count(*) FROM scores FINAL WHERE project_id = '${PROJECT_ID}' AND data_type = 'NUMERIC' AND (${predicate});`;
+const finalIds = (predicate: string) =>
+ `SELECT id FROM scores FINAL WHERE project_id = '${PROJECT_ID}' AND data_type = 'NUMERIC' AND (${predicate}) ORDER BY id;`;
+
+const runLocal = (sql: string) =>
+ executeClickhouseLocal(`${SCORES_DDL}\n${insertRows()}\n${sql}`);
+
+const sortedLines = (out: string) =>
+ out
+ .split("\n")
+ .map((l) => l.trim())
+ .filter(Boolean)
+ .sort();
+
+describeWithLocal("scores selective-seek: parity vs FINAL", () => {
+ // predicate is the equivalent hand-written SQL for the app filter, applied to
+ // the deduped (FINAL) rows.
+ const scenarios = [
+ {
+ name: "trace_id = T2 (latest matches)",
+ filter: traceIdEq("T2"),
+ predicate: "trace_id = 'T2'",
+ },
+ {
+ name: "trace_id = T1 (old matched, latest does not)",
+ filter: traceIdEq("T1"),
+ predicate: "trace_id = 'T1'",
+ },
+ {
+ name: "name IN [n1] (spans two day-buckets)",
+ filter: nameIn(["n1"]),
+ predicate: "name = 'n1'",
+ },
+ {
+ name: "observation_id = O1",
+ filter: observationIdEq("O1"),
+ predicate: "observation_id = 'O1'",
+ },
+ {
+ name: "value > 0.5 (ineligible fallback)",
+ filter: valueGt(0.5),
+ predicate: "value > 0.5",
+ },
+ ];
+
+ for (const { name, filter, predicate } of scenarios) {
+ it(`count parity: ${name}`, async () => {
+ const q = await captureCountSql(filter);
+ const appCount = runLocal(substituteNamedParams(q.query, q.params ?? {}));
+ const refCount = runLocal(finalCount(predicate));
+ expect(appCount.trim()).toBe(refCount.trim());
+ });
+
+ it(`rows parity: ${name}`, async () => {
+ const q = await captureRowsSql(filter);
+ const appOut = runLocal(substituteNamedParams(q.query, q.params ?? {}));
+ const refOut = runLocal(finalIds(predicate));
+ // The app rows query returns the full projection; its first column is id.
+ const appIds = sortedLines(appOut).map((l) => l.split("\t")[0]);
+ const refIds = sortedLines(refOut);
+ expect(appIds).toEqual(refIds);
+ });
+ }
+});
+
+afterEach(() => resetCaptures());
diff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.ts
index c2fe2e4d50..5bcaaf0283 100644
--- a/packages/shared/src/server/repositories/scores.ts
+++ b/packages/shared/src/server/repositories/scores.ts
@@ -37,6 +37,7 @@ import {
DateTimeFilter,
CTEQueryBuilder,
NumberFilter,
+ scoreOnlyFiltersAreSeekEligible,
} from "../queries";
import { FilterCondition, FilterState, TimeFilter } from "../../types";
import {
@@ -1634,11 +1635,45 @@ const getScoresUiGenericFromEvents = async <T>(props: {
${includeHasMetadataFlag ? ",length(mapKeys(s.metadata)) > 0 AS has_metadata" : ""}
`;
+ // ── Selective seek ────────────────────────────────────────────────────────
+ // A highly selective score-only filter (single trace/observation/id, or a
+ // name) otherwise reconstructs the whole project before the outer WHERE
+ // discards almost everything. When at least one score-only conjunct is
+ // index-prunable (see scoreOnlyFiltersAreSeekEligible), first collect the
+ // matching dedup keys via the bloom / primary indexes, then reconstruct only
+ // those keys.
+ //
+ // The seek carries the *entire* score-only predicate, not just the prunable
+ // conjuncts: it stays a superset collection (proof below), the index prunes
+ // granules via the eligible conjuncts, and the remaining conjuncts shrink the
+ // key set within surviving granules.
+ //
+ // Correctness: if a key's deduped-latest satisfies predicate P, its latest raw
+ // row satisfies P (the latest *is* a raw row), so `SELECT DISTINCT … WHERE P`
+ // over raw rows collects it β€” `{latest matches P} βŠ† {some raw row matches P}`
+ // for any P (negation, null, range included). False positives (a key seeked
+ // via an older matching version whose latest no longer matches) are removed by
+ // re-applying P in the outer WHERE post-dedup. The seek's projection is the
+ // dedup key only, all immutable within a group, so the resulting tuple IN
+ // touches only dedup-key columns and is safe pre-dedup.
+ const seekEligible = scoreOnlyFiltersAreSeekEligible(scoreOnlyFilters);
+ const seekSubquery = seekEligible
+ ? `
+ SELECT DISTINCT s.project_id, toDate(s.timestamp), s.name, s.id
+ FROM scores s
+ WHERE s.project_id = {projectId: String}
+ ${innerDatePruneQuery ? `AND ${innerDatePruneQuery}` : ""}
+ ${scoreOnlyFilterRes?.query ? `AND ${scoreOnlyFilterRes.query}` : ""}`
+ : "";
+
// Pre-dedup inner scan: project scope + coarse date prune only. Mutable-column
- // filters run post-dedup in the outer WHERE (see the dedup rule below).
+ // filters run post-dedup in the outer WHERE (see the dedup rule below). When
+ // the seek fired, the tuple IN restricts the reconstruct scan to seeked keys;
+ // it references only dedup-key columns, so it is safe pre-dedup.
const innerScanWhere = `
WHERE s.project_id = {projectId: String}
- ${innerDatePruneQuery ? `AND ${innerDatePruneQuery}` : ""}`;
+ ${innerDatePruneQuery ? `AND ${innerDatePruneQuery}` : ""}
+ ${seekSubquery ? `AND (s.project_id, toDate(s.timestamp), s.name, s.id) IN (${seekSubquery}\n )` : ""}`;
// Post-dedup outer filters, shared by the count and rows paths.
const outerWhereClause = `