πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(scores): add selective-seek path for scores dedup reads (#17842)
Niklas Semmler72294f582cb1
diff --git a/packages/shared/src/server/queries/clickhouse-sql/score-seek-eligibility.test.ts b/packages/shared/src/server/queries/clickhouse-sql/score-seek-eligibility.test.tsnew file mode 100644index 0000000000..0cef3e201f--- /dev/null+++ b/packages/shared/src/server/queries/clickhouse-sql/score-seek-eligibility.test.ts@@ -0,0 +1,232 @@+import { describe, expect, it } from "vitest";++import {+ FilterList,+ NullFilter,+ NumberFilter,+ StringFilter,+ StringOptionsFilter,+} from "./clickhouse-filter";+import {+ isSeekEligibleFilter,+ scoreOnlyFiltersAreSeekEligible,+} from "./score-seek-eligibility";++const projectIdFilter = () =>+ new StringFilter({+ clickhouseTable: "scores",+ field: "project_id",+ operator: "=",+ value: "p1",+ });++const traceIdEq = (value = "trace-1") =>+ new StringFilter({+ clickhouseTable: "scores",+ field: "trace_id",+ operator: "=",+ value,+ tablePrefix: "s",+ });++const observationIdEq = (value: string, emptyEqualsNull = true) =>+ new StringFilter({+ clickhouseTable: "scores",+ field: "observation_id",+ operator: "=",+ value,+ tablePrefix: "s",+ emptyEqualsNull,+ });++const observationIdIn = (values: string[], emptyEqualsNull = true) =>+ new StringOptionsFilter({+ clickhouseTable: "scores",+ field: "observation_id",+ operator: "any of",+ values,+ tablePrefix: "s",+ emptyEqualsNull,+ });++const nameIn = (values: string[]) =>+ new StringOptionsFilter({+ clickhouseTable: "scores",+ field: "name",+ operator: "any of",+ values,+ tablePrefix: "s",+ });++const idEq = (value = "id-1") =>+ new StringFilter({+ clickhouseTable: "scores",+ field: "id",+ operator: "=",+ value,+ tablePrefix: "s",+ });++const nameEq = (value = "accuracy") =>+ new StringFilter({+ clickhouseTable: "scores",+ field: "name",+ operator: "=",+ value,+ tablePrefix: "s",+ });++const sessionIdEq = (value = "sess-1") =>+ new StringFilter({+ clickhouseTable: "scores",+ field: "session_id",+ operator: "=",+ value,+ tablePrefix: "s",+ emptyEqualsNull: true,+ });++const evaluatorIdEq = (value = "eval-1") =>+ new StringFilter({+ clickhouseTable: "scores",+ field: "evaluator_id",+ operator: "=",+ value,+ tablePrefix: "s",+ emptyEqualsNull: true,+ });++const evaluationRuleIdIn = (values: string[]) =>+ new StringOptionsFilter({+ clickhouseTable: "scores",+ field: "evaluation_rule_id",+ operator: "any of",+ values,+ tablePrefix: "s",+ emptyEqualsNull: true,+ });++const environmentNoneOf = () =>+ new StringOptionsFilter({+ clickhouseTable: "scores",+ field: "environment",+ operator: "none of",+ values: ["default"],+ tablePrefix: "s",+ });++const valueRange = () =>+ new NumberFilter({+ clickhouseTable: "scores",+ field: "value",+ operator: ">",+ value: 0.5,+ tablePrefix: "s",+ });++describe("isSeekEligibleFilter", () => {+ it("accepts = on id / trace_id / name", () => {+ expect(isSeekEligibleFilter(idEq())).toBe(true);+ expect(isSeekEligibleFilter(traceIdEq())).toBe(true);+ expect(isSeekEligibleFilter(nameEq())).toBe(true);+ });++ it("accepts IN on name / observation_id (no empty member)", () => {+ expect(isSeekEligibleFilter(nameIn(["accuracy"]))).toBe(true);+ expect(isSeekEligibleFilter(observationIdIn(["obs-1", "obs-2"]))).toBe(+ true,+ );+ });++ it("accepts the newly-indexed bloom columns (session / evaluator / rule)", () => {+ expect(isSeekEligibleFilter(sessionIdEq())).toBe(true);+ expect(isSeekEligibleFilter(evaluatorIdEq())).toBe(true);+ expect(isSeekEligibleFilter(evaluationRuleIdIn(["r1"]))).toBe(true);+ });++ it("rejects the emptyEqualsNull degradation on a newly-indexed column", () => {+ expect(isSeekEligibleFilter(sessionIdEq(""))).toBe(false);+ expect(isSeekEligibleFilter(evaluationRuleIdIn(["", "r1"]))).toBe(false);+ });++ it("accepts a non-empty observation_id = despite emptyEqualsNull", () => {+ // A non-empty value does not trigger the OR IS NULL branch.+ expect(isSeekEligibleFilter(observationIdEq("obs-1"))).toBe(true);+ });++ it("rejects the emptyEqualsNull OR IS NULL degradations", () => {+ // observation_id = '' compiles to (... OR observation_id IS NULL).+ expect(isSeekEligibleFilter(observationIdEq(""))).toBe(false);+ // observation_id IN ('', ...) adds an OR IS NULL branch.+ expect(isSeekEligibleFilter(observationIdIn(["", "obs-1"]))).toBe(false);+ });++ it("rejects an empty IN set", () => {+ expect(isSeekEligibleFilter(nameIn([]))).toBe(false);+ });++ it("rejects non-index columns and non-equality operators", () => {+ expect(isSeekEligibleFilter(environmentNoneOf())).toBe(false);+ expect(isSeekEligibleFilter(valueRange())).toBe(false);+ // project_id equality exists on every query but is not a seek driver.+ expect(isSeekEligibleFilter(projectIdFilter())).toBe(false);+ });++ it("rejects trace_id 'contains' (not an index-prunable operator)", () => {+ expect(+ isSeekEligibleFilter(+ new StringFilter({+ clickhouseTable: "scores",+ field: "trace_id",+ operator: "contains",+ value: "trace",+ tablePrefix: "s",+ }),+ ),+ ).toBe(false);+ });++ it("rejects a null filter on an index column", () => {+ expect(+ isSeekEligibleFilter(+ new NullFilter({+ clickhouseTable: "scores",+ field: "observation_id",+ operator: "is null",+ tablePrefix: "s",+ emptyEqualsNull: true,+ }),+ ),+ ).toBe(false);+ });+});++describe("scoreOnlyFiltersAreSeekEligible", () => {+ it("is eligible when trace_id = is ANDed with a non-prunable observation OR-null sibling", () => {+ const filters = new FilterList([+ projectIdFilter(),+ traceIdEq(),+ observationIdIn(["", "obs-1"]), // not prunable on its own+ ]);+ expect(scoreOnlyFiltersAreSeekEligible(filters)).toBe(true);+ });++ it("is ineligible for environment-only / value-range filters", () => {+ expect(+ scoreOnlyFiltersAreSeekEligible(+ new FilterList([projectIdFilter(), environmentNoneOf()]),+ ),+ ).toBe(false);+ expect(+ scoreOnlyFiltersAreSeekEligible(+ new FilterList([projectIdFilter(), valueRange()]),+ ),+ ).toBe(false);+ });++ it("is ineligible for the bare project filter (no user predicate)", () => {+ expect(+ scoreOnlyFiltersAreSeekEligible(new FilterList([projectIdFilter()])),+ ).toBe(false);+ });+});diff --git a/packages/shared/src/server/queries/clickhouse-sql/score-seek-eligibility.ts b/packages/shared/src/server/queries/clickhouse-sql/score-seek-eligibility.tsnew file mode 100644index 0000000000..38717de27d--- /dev/null+++ b/packages/shared/src/server/queries/clickhouse-sql/score-seek-eligibility.ts@@ -0,0 +1,68 @@+import {+ type Filter,+ type FilterList,+ StringFilter,+ StringOptionsFilter,+} from "./clickhouse-filter";++// Score columns whose equality / IN predicate an index can prune, letting the+// selective-seek path collect candidate dedup keys without a full-project scan:+// - id β idx_id (bloom_filter)+// - trace_id / observation_id β idx_project_trace_observation (bloom_filter)+// - session_id β idx_project_session (bloom_filter)+// - dataset_run_id β idx_project_dataset_run (bloom_filter)+// - evaluator_id β idx_evaluator_id (bloom_filter)+// - evaluation_rule_id β idx_evaluation_rule_id (bloom_filter)+// - name β PRIMARY KEY / ORDER BY (project_id, toDate(timestamp), name)+//+// created_at carries a minmax index (idx_created_at), which prunes ranges rather+// than equality/IN, so it is intentionally excluded here β the seek predicate+// below only handles `=` / `IN`.+const SEEK_ELIGIBLE_FIELDS = new Set([+ "id",+ "trace_id",+ "observation_id",+ "session_id",+ "dataset_run_id",+ "evaluator_id",+ "evaluation_rule_id",+ "name",+]);++/**+ * Is this single top-level AND-conjunct index-prunable for the seek phase?+ *+ * Eligible only for a clean `=` (StringFilter) or `IN` (StringOptionsFilter+ * "any of") on an index-backed column. The `emptyEqualsNull` degradation is+ * rejected: `observation_id = ''` / `IN ('', β¦)` compiles to+ * `(β¦ OR observation_id IS NULL)` (see StringFilter / StringOptionsFilter+ * apply()), and a bloom filter cannot skip granules across an `OR β¦ IS NULL`.+ *+ * Conjuncts are judged individually, so a non-prunable sibling (e.g.+ * `observation_id IN (β¦) OR observation_id IS NULL`) does not disqualify a query+ * that also carries a prunable `trace_id = β¦`.+ */+export const isSeekEligibleFilter = (filter: Filter): boolean => {+ if (filter.clickhouseTable !== "scores") return false;+ if (!SEEK_ELIGIBLE_FIELDS.has(filter.field)) return false;++ if (filter instanceof StringFilter && filter.operator === "=") {+ return !(filter.emptyEqualsNull && filter.value === "");+ }++ if (filter instanceof StringOptionsFilter && filter.operator === "any of") {+ if (filter.values.length === 0) return false;+ return !(filter.emptyEqualsNull && filter.values.includes(""));+ }++ return false;+};++/**+ * The seek path is worthwhile when at least one score-only conjunct is+ * index-prunable. The seek still carries the *entire* score-only predicate+ * (a superset collection); this only decides whether to take the seek path at+ * all versus the single-pass fallback.+ */+export const scoreOnlyFiltersAreSeekEligible = (filters: FilterList): boolean =>+ filters.some(isSeekEligibleFilter);diff --git a/packages/shared/src/server/queries/index.ts b/packages/shared/src/server/queries/index.tsindex fbcbe58e88..cd8ee8c16f 100644--- a/packages/shared/src/server/queries/index.ts+++ b/packages/shared/src/server/queries/index.ts@@ -109,3 +109,7 @@ export {type EventsObservationRowSelectionInput,} from "./clickhouse-sql/events-observation-row-selection";export { extractTimeFilter } from "./clickhouse-sql/filter-utils";+export {+ isSeekEligibleFilter,+ scoreOnlyFiltersAreSeekEligible,+} from "./clickhouse-sql/score-seek-eligibility";diff --git a/packages/shared/src/server/repositories/__snapshots__/scores-selective-seek.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/scores-selective-seek.golden.test.ts.snapnew file mode 100644index 0000000000..796677a4d3--- /dev/null+++ b/packages/shared/src/server/repositories/__snapshots__/scores-selective-seek.golden.test.ts.snap@@ -0,0 +1,241 @@+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html++exports[`scores selective-seek: emitted SQL > full SQL text (one representative per branch) > eligible (trace_id =): count 1`] = `+[+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": "seek-project",+ "param2": "seek-project",+ "param3": "T2",+ "param4": [+ "NUMERIC",+ "BOOLEAN",+ "CATEGORICAL",+ "TEXT",+ ],+ },+ "route": "unknown",+ "sql": "SELECT count(*) AS count+FROM+(+ SELECT+ s.id,+ s.project_id,+ s.name,+ argMax(s.timestamp, s.event_ts) AS timestamp,+ argMax(s.environment, s.event_ts) AS environment,+ argMax(s.trace_id, s.event_ts) AS trace_id,+ argMax(s.observation_id, s.event_ts) AS observation_id,+ argMax(s.session_id, s.event_ts) AS session_id,+ argMax(s.evaluator_id, s.event_ts) AS evaluator_id,+ argMax(s.evaluation_rule_id, s.event_ts) AS evaluation_rule_id,+ argMax(s.value, s.event_ts) AS value,+ argMax(s.source, s.event_ts) AS source,+ argMax(s.comment, s.event_ts) AS comment,+ argMax(s.author_user_id, s.event_ts) AS author_user_id,+ argMax(s.data_type, s.event_ts) AS data_type,+ argMax(s.string_value, s.event_ts) AS string_value,+ argMax(s.metadata, s.event_ts) AS metadata+ FROM scores AS s+ WHERE (s.project_id = {param1:String}) AND ((s.project_id, toDate(s.timestamp), s.name, s.id) IN (+ SELECT DISTINCT+ s.project_id,+ toDate(s.timestamp),+ s.name,+ s.id+ FROM scores AS s+ WHERE (s.project_id = {param1:String}) AND (project_id = {param2:String}) AND (s.trace_id = {param3:String})+ ))+ GROUP BY+ s.project_id,+ toDate(s.timestamp),+ s.name,+ s.id+) AS s+WHERE (s.data_type IN ({param4:Array(String)})) AND (project_id = {param2:String}) AND (s.trace_id = {param3:String})",+ },+]+`;++exports[`scores selective-seek: emitted SQL > full SQL text (one representative per branch) > eligible (trace_id =): rows 1`] = `+[+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": "seek-project",+ "param2": "seek-project",+ "param3": "T2",+ "param4": [+ "NUMERIC",+ "BOOLEAN",+ "CATEGORICAL",+ "TEXT",+ ],+ },+ "route": "unknown",+ "sql": "SELECT+ s.id,+ s.project_id,+ s.environment,+ s.name,+ s.value,+ s.string_value,+ s.timestamp,+ s.source,+ s.data_type,+ s.comment,+ s.evaluator_id,+ s.trace_id,+ s.session_id,+ s.dataset_run_id,+ s.observation_id,+ s.author_user_id,+ s.created_at,+ s.updated_at,+ s.config_id,+ s.queue_id,+ s.execution_trace_id,+ s.ingestion_api_key,+ s.ingestion_sdk_name,+ s.ingestion_sdk_version,+ s.is_deleted,+ s.event_ts,+ length(mapKeys(s.metadata)) > 0 AS has_metadata+FROM+(+ SELECT *+ FROM scores AS s+ WHERE (s.project_id = {param1:String}) AND ((s.project_id, toDate(s.timestamp), s.name, s.id) IN (+ SELECT DISTINCT+ s.project_id,+ toDate(s.timestamp),+ s.name,+ s.id+ FROM scores AS s+ WHERE (s.project_id = {param1:String}) AND (project_id = {param2:String}) AND (s.trace_id = {param3:String})+ ))+ ORDER BY s.event_ts DESC+ LIMIT 1 BY+ s.project_id,+ toDate(s.timestamp),+ s.name,+ s.id+) AS s+WHERE (s.data_type IN ({param4:Array(String)})) AND (project_id = {param2:String}) AND (s.trace_id = {param3:String})+ORDER BY s.timestamp DESC",+ },+]+`;++exports[`scores selective-seek: emitted SQL > full SQL text (one representative per branch) > ineligible (value range): count 1`] = `+[+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": "seek-project",+ "param2": [+ "NUMERIC",+ "BOOLEAN",+ "CATEGORICAL",+ "TEXT",+ ],+ "param3": "seek-project",+ "param4": "0.5",+ },+ "route": "unknown",+ "sql": "SELECT count(*) AS count+FROM+(+ SELECT+ s.id,+ s.project_id,+ s.name,+ argMax(s.timestamp, s.event_ts) AS timestamp,+ argMax(s.environment, s.event_ts) AS environment,+ argMax(s.trace_id, s.event_ts) AS trace_id,+ argMax(s.observation_id, s.event_ts) AS observation_id,+ argMax(s.session_id, s.event_ts) AS session_id,+ argMax(s.evaluator_id, s.event_ts) AS evaluator_id,+ argMax(s.evaluation_rule_id, s.event_ts) AS evaluation_rule_id,+ argMax(s.value, s.event_ts) AS value,+ argMax(s.source, s.event_ts) AS source,+ argMax(s.comment, s.event_ts) AS comment,+ argMax(s.author_user_id, s.event_ts) AS author_user_id,+ argMax(s.data_type, s.event_ts) AS data_type,+ argMax(s.string_value, s.event_ts) AS string_value,+ argMax(s.metadata, s.event_ts) AS metadata+ FROM scores AS s+ WHERE s.project_id = {param1:String}+ GROUP BY+ s.project_id,+ toDate(s.timestamp),+ s.name,+ s.id+) AS s+WHERE (s.data_type IN ({param2:Array(String)})) AND (project_id = {param3:String}) AND (s.value > {param4:Decimal64(12)})",+ },+]+`;++exports[`scores selective-seek: emitted SQL > full SQL text (one representative per branch) > ineligible (value range): rows 1`] = `+[+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": "seek-project",+ "param2": [+ "NUMERIC",+ "BOOLEAN",+ "CATEGORICAL",+ "TEXT",+ ],+ "param3": "seek-project",+ "param4": "0.5",+ },+ "route": "unknown",+ "sql": "SELECT+ s.id,+ s.project_id,+ s.environment,+ s.name,+ s.value,+ s.string_value,+ s.timestamp,+ s.source,+ s.data_type,+ s.comment,+ s.evaluator_id,+ s.trace_id,+ s.session_id,+ s.dataset_run_id,+ s.observation_id,+ s.author_user_id,+ s.created_at,+ s.updated_at,+ s.config_id,+ s.queue_id,+ s.execution_trace_id,+ s.ingestion_api_key,+ s.ingestion_sdk_name,+ s.ingestion_sdk_version,+ s.is_deleted,+ s.event_ts,+ length(mapKeys(s.metadata)) > 0 AS has_metadata+FROM+(+ SELECT *+ FROM scores AS s+ WHERE s.project_id = {param1:String}+ ORDER BY s.event_ts DESC+ LIMIT 1 BY+ s.project_id,+ toDate(s.timestamp),+ s.name,+ s.id+) AS s+WHERE (s.data_type IN ({param2:Array(String)})) AND (project_id = {param3:String}) AND (s.value > {param4:Decimal64(12)})+ORDER BY s.timestamp DESC",+ },+]+`;diff --git a/packages/shared/src/server/repositories/scores-selective-seek.golden.test.ts b/packages/shared/src/server/repositories/scores-selective-seek.golden.test.tsnew file mode 100644index 0000000000..b87e6ccd8d--- /dev/null+++ b/packages/shared/src/server/repositories/scores-selective-seek.golden.test.ts@@ -0,0 +1,346 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++// A hand-written exec-seam mock (no importOriginal) records each query and keeps+// the real repositories/queries barrels out of the mock resolution order, which+// otherwise reorders module init into the barrel cycle.+const recorder = vi.hoisted(() => {+ const captured: Array<{+ fn: string;+ query: string;+ params?: Record<string, unknown>;+ tags?: unknown;+ }> = [];+ // Unique per call β a constant would collide the param names of two filters of+ // the same class (e.g. the default project_id and a trace_id equality), and the+ // later bind would clobber the earlier one.+ let n = 0;+ const uid = () => `v${n++}`;+ return { captured, uid };+});++vi.mock("./clickhouse", () => ({+ queryClickhouse: vi.fn(async (opts: any) => {+ recorder.captured.push({ fn: "queryClickhouse", ...opts });+ // The count path reads rows[0].count; give it a benign shape. The rows path+ // maps over the result, so an empty array is correct there.+ return /count\(\*\) AS count/.test(opts.query) ? [{ count: "0" }] : [];+ }),+ queryClickhouseStream: vi.fn(),+ queryClickhouseExecRaw: vi.fn(),+ commandClickhouse: vi.fn(),+ upsertClickhouse: vi.fn(),+ parseClickhouseUTCDateTimeFormat: vi.fn(),+ clickhouseCompliantRandomCharacters: vi.fn(() => recorder.uid()),+ BLOB_EXPORT_PARQUET_CLICKHOUSE_SETTINGS: {},+}));++// query-options imports the server barrel, which cannot finish initializing when+// the graph is entered from this repository file; the events scores path never+// calls it.+vi.mock("../queries/clickhouse-sql/query-options", () => ({+ shouldSkipObservationsFinal: vi.fn().mockResolvedValue(false),+}));++// Import via the server barrel so it initializes in the correct order; entering+// the graph directly through ./scores trips a barrel init cycle.+import {+ getScoresUiCountFromEvents,+ getScoresUiTableFromEvents,+} from "../index";++import type { FilterState } from "../../types";+import {+ clickhouseFormatAvailable,+ clickhouseLocalAvailable,+ executeClickhouseLocal,+ normalizeCapturedQueries,+ substituteNamedParams,+} from "./goldenHarness";++const PROJECT_ID = "seek-project";+const ORDER_BY = { column: "timestamp", order: "DESC" as const };++// ββ Filter fixtures βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+const traceIdEq = (value: string): FilterState => [+ { column: "traceId", type: "string", operator: "=", value },+];+const observationIdEq = (value: string): FilterState => [+ { column: "observationId", type: "string", operator: "=", value },+];+const nameIn = (values: string[]): FilterState => [+ { column: "name", type: "stringOptions", operator: "any of", value: values },+];+const valueGt = (value: number): FilterState => [+ { column: "value", type: "number", operator: ">", value },+];+const environmentNoneOf = (values: string[]): FilterState => [+ {+ column: "environment",+ type: "stringOptions",+ operator: "none of",+ value: values,+ },+];++const resetCaptures = () => {+ recorder.captured.length = 0;+};++async function captureCountSql(filter: FilterState) {+ resetCaptures();+ await getScoresUiCountFromEvents({+ projectId: PROJECT_ID,+ filter,+ orderBy: null,+ });+ expect(recorder.captured).toHaveLength(1);+ return recorder.captured[0];+}++async function captureRowsSql(filter: FilterState) {+ resetCaptures();+ await getScoresUiTableFromEvents({+ projectId: PROJECT_ID,+ filter,+ orderBy: ORDER_BY,+ });+ expect(recorder.captured).toHaveLength(1);+ return recorder.captured[0];+}++const SEEK_TUPLE_IN =+ /\(s\.project_id, toDate\(s\.timestamp\), s\.name, s\.id\) IN \(/;++const describeWithFormat = clickhouseFormatAvailable()+ ? describe+ : describe.skip;+const describeWithLocal = clickhouseLocalAvailable() ? describe : describe.skip;++if (!clickhouseFormatAvailable()) {+ console.warn(+ "[scores-seek] `clickhouse format` unavailable β skipping golden SQL tests.",+ );+}+if (!clickhouseLocalAvailable()) {+ console.warn(+ "[scores-seek] `clickhouse local` unavailable β skipping parity tests.",+ );+}++// ββ Emitted SQL shape βββββββββββββββββββββββββββββββββββββββββββββββββββββββ+// The full matrix asserts *routing* only (does the seek phase fire for this+// filter?); whether a filter is eligible is unit-tested in+// score-seek-eligibility.test.ts. The full SQL text is locked by a single+// eligible + single ineligible snapshot below β snapshotting every shape just+// duplicates the same query skeleton.+describeWithFormat("scores selective-seek: emitted SQL", () => {+ beforeEach(() => resetCaptures());++ const eligible = [+ { name: "trace_id =", filter: traceIdEq("T2") },+ { name: "observation_id =", filter: observationIdEq("O1") },+ { name: "name IN", filter: nameIn(["n1"]) },+ ];+ const ineligible = [+ { name: "value range", filter: valueGt(0.5) },+ { name: "environment none of", filter: environmentNoneOf(["dev"]) },+ { name: "no filter", filter: [] as FilterState },+ ];++ describe("eligible β seek phase present", () => {+ for (const { name, filter } of eligible) {+ it(`count: ${name}`, async () => {+ const q = await captureCountSql(filter);+ expect(q.query).toContain("SELECT DISTINCT");+ expect(q.query).toMatch(SEEK_TUPLE_IN);+ });+ it(`rows: ${name}`, async () => {+ const q = await captureRowsSql(filter);+ expect(q.query).toContain("SELECT DISTINCT");+ expect(q.query).toMatch(SEEK_TUPLE_IN);+ });+ }+ });++ describe("ineligible β fallback unchanged, no seek", () => {+ for (const { name, filter } of ineligible) {+ it(`count: ${name}`, async () => {+ const q = await captureCountSql(filter);+ expect(q.query).not.toContain("SELECT DISTINCT");+ expect(q.query).not.toMatch(SEEK_TUPLE_IN);+ });+ it(`rows: ${name}`, async () => {+ const q = await captureRowsSql(filter);+ expect(q.query).not.toContain("SELECT DISTINCT");+ expect(q.query).not.toMatch(SEEK_TUPLE_IN);+ });+ }+ });++ // One representative shape per branch locks the full SQL text; the routing+ // matrix above covers the remaining shapes structurally.+ describe("full SQL text (one representative per branch)", () => {+ it("eligible (trace_id =): count", async () => {+ const q = await captureCountSql(traceIdEq("T2"));+ expect(normalizeCapturedQueries([q])).toMatchSnapshot();+ });+ it("eligible (trace_id =): rows", async () => {+ const q = await captureRowsSql(traceIdEq("T2"));+ expect(normalizeCapturedQueries([q])).toMatchSnapshot();+ });+ it("ineligible (value range): count", async () => {+ const q = await captureCountSql(valueGt(0.5));+ expect(normalizeCapturedQueries([q])).toMatchSnapshot();+ });+ it("ineligible (value range): rows", async () => {+ const q = await captureRowsSql(valueGt(0.5));+ expect(normalizeCapturedQueries([q])).toMatchSnapshot();+ });+ });+});++// ββ Correctness parity vs FINAL over a multi-version fixture βββββββββββββββββ++// scores columns referenced by the count (argMax) and rows (SELECT *) paths.+const SCORES_DDL = `+ CREATE TABLE scores (+ id String,+ timestamp DateTime64(3),+ project_id String,+ trace_id String,+ observation_id Nullable(String),+ name String,+ value Float64,+ source String,+ comment Nullable(String),+ author_user_id Nullable(String),+ config_id Nullable(String),+ data_type String,+ string_value Nullable(String),+ queue_id Nullable(String),+ environment String,+ session_id Nullable(String),+ dataset_run_id Nullable(String),+ evaluator_id Nullable(String),+ evaluation_rule_id Nullable(String),+ execution_trace_id Nullable(String),+ metadata Map(String, String),+ ingestion_api_key String,+ ingestion_sdk_name String,+ ingestion_sdk_version String,+ created_at DateTime64(3),+ updated_at DateTime64(3),+ event_ts DateTime64(3),+ is_deleted UInt8,+ INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,+ INDEX idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1,+ INDEX idx_project_session (project_id, session_id) TYPE bloom_filter(0.001) GRANULARITY 1,+ INDEX idx_project_dataset_run (project_id, dataset_run_id) TYPE bloom_filter(0.001) GRANULARITY 1,+ INDEX idx_created_at created_at TYPE minmax GRANULARITY 1,+ INDEX idx_evaluator_id evaluator_id TYPE bloom_filter(0.001) GRANULARITY 1,+ INDEX idx_evaluation_rule_id evaluation_rule_id TYPE bloom_filter(0.001) GRANULARITY 1+ ) ENGINE = ReplacingMergeTree(event_ts, is_deleted)+ PARTITION BY toYYYYMM(timestamp)+ PRIMARY KEY (project_id, toDate(timestamp), name)+ ORDER BY (project_id, toDate(timestamp), name, id);+`;++// (id, name, ts, trace_id, observation_id, value, event_ts). Every row:+// project_id = PROJECT_ID, data_type = NUMERIC (listable), is_deleted = 0.+// a: trace_id mutates T1 -> T2 (latest), value 0.9 -> 0.1+// b: single version, trace_id T1+// c: two day-buckets (distinct dedup groups), trace_id T3+// d: has observation_id O1+// e: value mutates 0.9 -> 0.1 (latest) β exercises dedup-then-filter+const ROWS: Array<+ [string, string, string, string, string | null, number, number]+> = [+ ["a", "n1", "2026-01-10 10:00:00.000", "T1", null, 0.9, 1],+ ["a", "n1", "2026-01-10 10:05:00.000", "T2", null, 0.1, 2],+ ["b", "n2", "2026-01-10 09:00:00.000", "T1", null, 0.2, 1],+ ["c", "n1", "2026-01-10 08:00:00.000", "T3", null, 0.5, 1],+ ["c", "n1", "2026-01-11 08:00:00.000", "T3", null, 0.6, 1],+ ["d", "n3", "2026-01-10 07:00:00.000", "T4", "O1", 0.4, 1],+ ["e", "n4", "2026-01-10 06:00:00.000", "T5", null, 0.9, 1],+ ["e", "n4", "2026-01-10 06:05:00.000", "T5", null, 0.1, 2],+];++const insertRows = () => {+ const values = ROWS.map(+ ([id, name, ts, traceId, obsId, value, eventTs]) =>+ `('${id}', '${ts}', '${PROJECT_ID}', '${traceId}', ${+ obsId === null ? "NULL" : `'${obsId}'`+ }, '${name}', ${value}, 'API', NULL, NULL, NULL, 'NUMERIC', NULL, NULL, 'default', NULL, NULL, NULL, NULL, NULL, map(), 'pk', 'sdk', '1', '${ts}', '${ts}', toDateTime64(${eventTs}, 3), 0)`,+ ).join(",\n");+ return `INSERT INTO scores VALUES ${values};`;+};++// FINAL ground truth: dedup on the ORDER BY key keeping max event_ts, then apply+// the same post-dedup predicate the outer WHERE applies.+const finalCount = (predicate: string) =>+ `SELECT count(*) FROM scores FINAL WHERE project_id = '${PROJECT_ID}' AND data_type = 'NUMERIC' AND (${predicate});`;+const finalIds = (predicate: string) =>+ `SELECT id FROM scores FINAL WHERE project_id = '${PROJECT_ID}' AND data_type = 'NUMERIC' AND (${predicate}) ORDER BY id;`;++const runLocal = (sql: string) =>+ executeClickhouseLocal(`${SCORES_DDL}\n${insertRows()}\n${sql}`);++const sortedLines = (out: string) =>+ out+ .split("\n")+ .map((l) => l.trim())+ .filter(Boolean)+ .sort();++describeWithLocal("scores selective-seek: parity vs FINAL", () => {+ // predicate is the equivalent hand-written SQL for the app filter, applied to+ // the deduped (FINAL) rows.+ const scenarios = [+ {+ name: "trace_id = T2 (latest matches)",+ filter: traceIdEq("T2"),+ predicate: "trace_id = 'T2'",+ },+ {+ name: "trace_id = T1 (old matched, latest does not)",+ filter: traceIdEq("T1"),+ predicate: "trace_id = 'T1'",+ },+ {+ name: "name IN [n1] (spans two day-buckets)",+ filter: nameIn(["n1"]),+ predicate: "name = 'n1'",+ },+ {+ name: "observation_id = O1",+ filter: observationIdEq("O1"),+ predicate: "observation_id = 'O1'",+ },+ {+ name: "value > 0.5 (ineligible fallback)",+ filter: valueGt(0.5),+ predicate: "value > 0.5",+ },+ ];++ for (const { name, filter, predicate } of scenarios) {+ it(`count parity: ${name}`, async () => {+ const q = await captureCountSql(filter);+ const appCount = runLocal(substituteNamedParams(q.query, q.params ?? {}));+ const refCount = runLocal(finalCount(predicate));+ expect(appCount.trim()).toBe(refCount.trim());+ });++ it(`rows parity: ${name}`, async () => {+ const q = await captureRowsSql(filter);+ const appOut = runLocal(substituteNamedParams(q.query, q.params ?? {}));+ const refOut = runLocal(finalIds(predicate));+ // The app rows query returns the full projection; its first column is id.+ const appIds = sortedLines(appOut).map((l) => l.split("\t")[0]);+ const refIds = sortedLines(refOut);+ expect(appIds).toEqual(refIds);+ });+ }+});++afterEach(() => resetCaptures());diff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.tsindex c2fe2e4d50..5bcaaf0283 100644--- a/packages/shared/src/server/repositories/scores.ts+++ b/packages/shared/src/server/repositories/scores.ts@@ -37,6 +37,7 @@ import {DateTimeFilter,CTEQueryBuilder,NumberFilter,+ scoreOnlyFiltersAreSeekEligible,} from "../queries";import { FilterCondition, FilterState, TimeFilter } from "../../types";import {@@ -1634,11 +1635,45 @@ const getScoresUiGenericFromEvents = async <T>(props: {${includeHasMetadataFlag ? ",length(mapKeys(s.metadata)) > 0 AS has_metadata" : ""}`;+ // ββ Selective seek ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+ // A highly selective score-only filter (single trace/observation/id, or a+ // name) otherwise reconstructs the whole project before the outer WHERE+ // discards almost everything. When at least one score-only conjunct is+ // index-prunable (see scoreOnlyFiltersAreSeekEligible), first collect the+ // matching dedup keys via the bloom / primary indexes, then reconstruct only+ // those keys.+ //+ // The seek carries the *entire* score-only predicate, not just the prunable+ // conjuncts: it stays a superset collection (proof below), the index prunes+ // granules via the eligible conjuncts, and the remaining conjuncts shrink the+ // key set within surviving granules.+ //+ // Correctness: if a key's deduped-latest satisfies predicate P, its latest raw+ // row satisfies P (the latest *is* a raw row), so `SELECT DISTINCT β¦ WHERE P`+ // over raw rows collects it β `{latest matches P} β {some raw row matches P}`+ // for any P (negation, null, range included). False positives (a key seeked+ // via an older matching version whose latest no longer matches) are removed by+ // re-applying P in the outer WHERE post-dedup. The seek's projection is the+ // dedup key only, all immutable within a group, so the resulting tuple IN+ // touches only dedup-key columns and is safe pre-dedup.+ const seekEligible = scoreOnlyFiltersAreSeekEligible(scoreOnlyFilters);+ const seekSubquery = seekEligible+ ? `+ SELECT DISTINCT s.project_id, toDate(s.timestamp), s.name, s.id+ FROM scores s+ WHERE s.project_id = {projectId: String}+ ${innerDatePruneQuery ? `AND ${innerDatePruneQuery}` : ""}+ ${scoreOnlyFilterRes?.query ? `AND ${scoreOnlyFilterRes.query}` : ""}`+ : "";+// Pre-dedup inner scan: project scope + coarse date prune only. Mutable-column- // filters run post-dedup in the outer WHERE (see the dedup rule below).+ // filters run post-dedup in the outer WHERE (see the dedup rule below). When+ // the seek fired, the tuple IN restricts the reconstruct scan to seeked keys;+ // it references only dedup-key columns, so it is safe pre-dedup.const innerScanWhere = `WHERE s.project_id = {projectId: String}- ${innerDatePruneQuery ? `AND ${innerDatePruneQuery}` : ""}`;+ ${innerDatePruneQuery ? `AND ${innerDatePruneQuery}` : ""}+ ${seekSubquery ? `AND (s.project_id, toDate(s.timestamp), s.name, s.id) IN (${seekSubquery}\n )` : ""}`;// Post-dedup outer filters, shared by the count and rows paths.const outerWhereClause = `