🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(scores): sort skip-dedup score lists by day first (#19311)
Niklas Semmler606072890440
diff --git a/packages/shared/src/server/repositories/scores-selective-seek.golden.test.ts b/packages/shared/src/server/repositories/scores-selective-seek.golden.test.tsindex e8a9959bac..1de3f2a14c 100644--- a/packages/shared/src/server/repositories/scores-selective-seek.golden.test.ts+++ b/packages/shared/src/server/repositories/scores-selective-seek.golden.test.ts@@ -393,6 +393,18 @@ describe("scores skip-dedup: plain scan", () => {}});+ it("orders a timestamp sort by the sort-key day first", async () => {+ const rows = await captureRowsSql([]);+ expect(rows.query).toContain(+ "ORDER BY toDate(s.timestamp) DESC, s.timestamp DESC, s.id DESC",+ );++ mockShouldSkipScoresDedup.mockReturnValue(false);+ const deduped = await captureRowsSql([]);+ expect(deduped.query).toContain("ORDER BY s.timestamp DESC, s.id DESC");+ expect(deduped.query).not.toContain("toDate(s.timestamp) DESC");+ });+const scenarios = [{ name: "no filter", filter: [] as FilterState, predicate: "1" },{diff --git a/packages/shared/src/server/repositories/scores.test.ts b/packages/shared/src/server/repositories/scores.test.tsindex 88df04900f..8b001e1a38 100644--- a/packages/shared/src/server/repositories/scores.test.ts+++ b/packages/shared/src/server/repositories/scores.test.ts@@ -1,6 +1,9 @@import { describe, it, expect, beforeEach, vi } from "vitest";const mockQueryClickhouse = vi.hoisted(() => vi.fn());+const mockShouldSkipScoresDedup = vi.hoisted(() =>+ vi.fn<(projectId: string) => boolean>(() => false),+);vi.mock("./clickhouse", () => ({queryClickhouse: mockQueryClickhouse,@@ -17,13 +20,16 @@ vi.mock("./clickhouse", () => ({// module initialization when the graph is entered from this repository file.vi.mock("../queries/clickhouse-sql/query-options", () => ({shouldSkipObservationsFinal: vi.fn().mockResolvedValue(false),- shouldSkipScoresDedup: vi.fn().mockReturnValue(false),+ shouldSkipScoresDedup: mockShouldSkipScoresDedup,}));import {getScoresForTraces,getScoresForSessions,getScoresForExperiments,+ listScoresV3ForPublicApi,+ _handleGenerateScoresForPublicApi,+ FilterList,} from "../index";const PAGINATION_CLAUSE = "limit {limit: Int32} offset {offset: Int32}";@@ -84,3 +90,55 @@ describe("score repository pagination", () => {expect(query).not.toContain(PAGINATION_CLAUSE);});});++describe("public score lists without dedup", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ mockQueryClickhouse.mockResolvedValue([]);+ });++ const listV3 = async (skipDedup: boolean) => {+ mockQueryClickhouse.mockClear();+ mockShouldSkipScoresDedup.mockReturnValue(skipDedup);+ await listScoresV3ForPublicApi({+ projectId: "proj-1",+ limit: 50,+ fields: ["core"],+ });+ return mockQueryClickhouse.mock.calls[0][0].query as string;+ };++ const listV1V2 = async (skipDedup: boolean, needsTraceJoin: boolean) => {+ mockQueryClickhouse.mockClear();+ mockShouldSkipScoresDedup.mockReturnValue(skipDedup);+ await _handleGenerateScoresForPublicApi({+ projectId: "proj-1",+ scoresFilter: new FilterList([]),+ tracesFilter: new FilterList([]),+ scoreScope: "all",+ includeTrace: false,+ needsTraceJoin,+ pagination: { limit: 50, page: 1 },+ });+ return mockQueryClickhouse.mock.calls[0][0].query as string;+ };++ it("v3 list orders by the sort-key day first", async () => {+ expect(await listV3(true)).toContain(+ "ORDER BY toDate(s.timestamp) DESC, s.timestamp DESC, s.id DESC",+ );+ expect(await listV3(false)).not.toContain("toDate(s.timestamp) DESC");+ });++ it("v1/v2 list orders by the sort-key day first", async () => {+ expect(await listV1V2(true, false)).toMatch(+ /ORDER BY\s+toDate\(s\.timestamp\) desc, s\.timestamp desc/,+ );+ expect(await listV1V2(true, true)).not.toContain(+ "toDate(s.timestamp) desc",+ );+ expect(await listV1V2(false, false)).not.toContain(+ "toDate(s.timestamp) desc",+ );+ });+});diff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.tsindex 4e9c129557..d1912af10b 100644--- a/packages/shared/src/server/repositories/scores.ts+++ b/packages/shared/src/server/repositories/scores.ts@@ -1779,8 +1779,28 @@ const getScoresUiGenericFromEvents = async <T>(props: {// The trace join, ORDER BY, and LIMIT/OFFSET run OUTSIDE the subquery, at one// level, so pagination follows the requested sort (and a traces-column sort can// reference the joined `e`).- const orderAndPage = `- ${withIdTiebreaker(orderByToClickhouseSql(orderBy ?? null, scoresTableUiColumnDefinitionsFromEvents), orderBy)}+ const orderByClause = withIdTiebreaker(+ orderByToClickhouseSql(+ orderBy ?? null,+ scoresTableUiColumnDefinitionsFromEvents,+ ),+ orderBy,+ );+ // The table sorts by day, not by timestamp. Putting the day first keeps the+ // order but lets ClickHouse read the newest days first and stop after the+ // page. Only the plain scan reads the table directly, so only it benefits.+ const sortsByScoreTimestamp =+ matchedOrderByColumn?.clickhouseTableName === "scores" &&+ matchedOrderByColumn.clickhouseSelect === "timestamp";+ const sortKeyOrderByClause =+ sortsByScoreTimestamp && orderBy+ ? orderByClause.replace(+ /^ORDER BY /,+ `ORDER BY toDate(s.timestamp) ${orderBy.order}, `,+ )+ : orderByClause;+ const orderAndPage = (clause: string) => `+ ${clause}${limit !== undefined && offset !== undefined ? `limit {limit: Int32} offset {offset: Int32}` : ""}`;// Projects whose scores are never updated have one row per key, so they@@ -1796,7 +1816,7 @@ const getScoresUiGenericFromEvents = async <T>(props: {) s${eventsJoin}${outerWhereClause}- ${props.select === "count" ? "" : orderAndPage}+ ${props.select === "count" ? "" : orderAndPage(sortKeyOrderByClause)}`;const dedupCountQuery = () => `@@ -1845,7 +1865,7 @@ const getScoresUiGenericFromEvents = async <T>(props: {LIMIT 1 BY s.project_id, toDate(s.timestamp), s.name, s.id) s${eventsJoin}- ${orderAndPage}+ ${orderAndPage(orderByClause)}`;const query = (() => {@@ -3114,14 +3134,14 @@ export const _handleGenerateScoresForPublicApi = async ({${scoreScope === "traces_only" ? "AND s.session_id IS NULL AND s.dataset_run_id IS NULL" : ""}${appliedScoresFilter.query ? `AND ${appliedScoresFilter.query}` : ""}${tracesFilter.length() > 0 ? `AND ${appliedTracesFilter.query}` : ""}- ORDER BY- s.timestamp desc, s.event_ts desc${// The traces join is not deduplicated, so LIMIT 1 BY also collapses- // one score row per unmerged trace version.+ // one score row per unmerged trace version. Without it the scan reads+ // the table directly; the day first keeps the order but lets ClickHouse+ // read the newest days first and stop after the page.shouldSkipScoresDedup(projectId) && !needsTraceJoin- ? ""- : "LIMIT 1 BY s.id, s.project_id"+ ? "ORDER BY toDate(s.timestamp) desc, s.timestamp desc, s.event_ts desc"+ : "ORDER BY s.timestamp desc, s.event_ts desc\n LIMIT 1 BY s.id, s.project_id"}${pagination !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}`;@@ -3478,8 +3498,13 @@ const buildV3ListQuery = (: ""}${filterClause ? `AND ${filterClause}` : ""}- ORDER BY s.timestamp DESC, s.id DESC, s.event_ts DESC- ${skipDedup ? "" : "LIMIT 1 BY s.id, s.project_id"}+ ${+ // Without dedup the scan reads the table directly; the day first keeps the+ // order but lets ClickHouse read the newest days first and stop after the page.+ skipDedup+ ? "ORDER BY toDate(s.timestamp) DESC, s.timestamp DESC, s.id DESC, s.event_ts DESC"+ : "ORDER BY s.timestamp DESC, s.id DESC, s.event_ts DESC\n LIMIT 1 BY s.id, s.project_id"+ }LIMIT {limit: Int32}`;