🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(query-ast): compile session trace identifiers through the AST (#19085)
Niklas Semmlerd43abd9ea7aa
diff --git a/packages/shared/src/server/query-ast/README.md b/packages/shared/src/server/query-ast/README.mdindex a3f7a7a114..709710ba0b 100644--- a/packages/shared/src/server/query-ast/README.md+++ b/packages/shared/src/server/query-ast/README.md@@ -166,8 +166,12 @@ suffix to run there. Promote it to a required check once it has proven stable.- **Build shared expressions from the calling query's expression builder.**A reusable events expression or select list is wrapped in`scopedToEvents((eb) => …)` from `../repositories/events-selects.ts`, then- passed as `.select(traceAggregationSelects)` or called inside- `.where((eb) => …)` with the query's own `eb`. That only type-checks in a+ passed as `.select(traceAggregationSelects(TRACE_AGGREGATION_FIELDS))` or+ called inside `.where((eb) => …)` with the query's own `eb`. Keyed select+ lists (`traceAggregationSelects`, `sessionAggregationSelects`,+ `eventsRowFieldSelects`) take the fields to select: pick only what the query+ reads, because one selected `input`, `output` or metadata value routes a+ logical `events` relation, CTE bodies included, to `events_full`. That only type-checks in aquery that reads a full events row as `e` and nothing else: no join, nonarrower `e`. Columns inside such a helper are written `e.…`, so an aliasof the same name in the calling query can't shadow them. A helper typeddiff --git a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snapindex 02d1daae3b..73472621e8 100644--- a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap+++ b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap@@ -3781,7 +3781,7 @@ exports[`golden: events trace aggregation family > getTracesIdentifierForSession"param2": "golden-session",},"route": "unknown",- "sql": "WITH traces AS+ "sql": "WITH event_traces AS(SELECTe.trace_id AS id,@@ -3795,57 +3795,16 @@ exports[`golden: events trace aggregation family > getTracesIdentifierForSession) AS name,min(e.start_time) AS timestamp,argMaxIf(e.environment, e.event_ts, e.environment != '') AS environment,- argMaxIf(e.version, e.event_ts, e.version != '') AS version,argMaxIf(e.session_id, e.event_ts, e.session_id != '') AS session_id,- argMaxIf(e.user_id, e.event_ts, e.user_id != '') AS user_id,- argMaxIf(e.input, e.event_ts, e.parent_span_id = '') AS input,- argMaxIf(e.output, e.event_ts, e.parent_span_id = '') AS output,- argMaxIf(- mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)),- e.event_ts,- e.parent_span_id = ''- ) AS metadata,- min(e.created_at) AS created_at,- max(e.updated_at) AS updated_at,- sum(e.total_cost) AS total_cost,- if(- max(e.end_time) IS NULL,- NULL,- dateDiff('millisecond', min(e.start_time), greatest(max(e.start_time), max(e.end_time)))- ) AS latency_milliseconds,- groupUniqArrayIf(e.span_id, e.span_id != '') AS observation_ids,- length(- groupUniqArrayIf(e.span_id, (e.span_id != '') AND (e.span_id != concat('t-', e.trace_id)))- ) AS observation_count,- argMaxIf(e.bookmarked, e.event_ts, e.parent_span_id = '') AS bookmarked,- max(e.public) AS public,- argMaxIf(e.experiment_item_id, e.event_ts, e.experiment_item_id != '') AS experiment_item_id,- sumMap(e.usage_details) AS usage_details,- sumMap(e.cost_details) AS cost_details,- multiIf(- arrayExists(x -> (x = 'ERROR'), groupArray(e.level)),- 'ERROR',- arrayExists(x -> (x = 'WARNING'), groupArray(e.level)),- 'WARNING',- arrayExists(x -> (x = 'DEFAULT'), groupArray(e.level)),- 'DEFAULT',- 'DEBUG'- ) AS aggregated_level,- countIf(e.level = 'WARNING') AS warning_count,- countIf(e.level = 'ERROR') AS error_count,- countIf(e.level = 'DEFAULT') AS default_count,- countIf(e.level = 'DEBUG') AS debug_count,- argMaxIf(e.tags, e.event_ts, notEmpty(e.tags)) AS tags,- argMaxIf(e.release, e.event_ts, e.release != '') AS release,- argMaxIf(e.evaluator_id, e.event_ts, e.evaluator_id != '') AS evaluator_id,- argMaxIf(e.evaluation_rule_id, e.event_ts, e.evaluation_rule_id != '') AS evaluation_rule_id,- any(e.experiment_id) AS experiment_id+ argMaxIf(e.user_id, e.event_ts, e.user_id != '') AS user_idFROM events_core AS eWHERE (e.project_id = {param1:String})+ AND (e.is_deleted = 0)AND (e.trace_id IN (SELECT DISTINCT trace_id- FROM events_core- WHERE (project_id = {param1:String})+ FROM events_core AS events+ WHERE (events.project_id = {param1:String})+ AND (events.is_deleted = 0)AND (session_id = {param2:String})))GROUP BY@@ -3860,7 +3819,7 @@ SELECTt.timestamp,t.project_id,t.environment-FROM traces AS t+FROM event_traces AS tORDER BY t.timestamp ASCLIMIT 1 BYt.id,diff --git a/packages/shared/src/server/repositories/events-selects.ts b/packages/shared/src/server/repositories/events-selects.tsindex a4667f4920..809596db29 100644--- a/packages/shared/src/server/repositories/events-selects.ts+++ b/packages/shared/src/server/repositories/events-selects.ts@@ -139,6 +139,42 @@ const OBSERVATION_BY_ID_FIELDS = orderedFieldKeys(["release",] as const satisfies readonly EventsFieldKey[]);+/** Every trace aggregate, in the column order full trace reads select them. */+export const TRACE_AGGREGATION_FIELDS = [+ "id",+ "project_id",+ "name",+ "timestamp",+ "environment",+ "version",+ "session_id",+ "user_id",+ "input",+ "output",+ "metadata",+ "created_at",+ "updated_at",+ "total_cost",+ "latency_milliseconds",+ "observation_ids",+ "observation_count",+ "bookmarked",+ "public",+ "experiment_item_id",+ "usage_details",+ "cost_details",+ "aggregated_level",+ "warning_count",+ "error_count",+ "default_count",+ "debug_count",+ "tags",+ "release",+ "evaluator_id",+ "evaluation_rule_id",+ "experiment_id",+] as const;+const eventsMetadataMap = (eb: EventsExpressionBuilder) =>metadataMap(eb.ref("e.metadata_names"), eb.ref("e.metadata_values"));@@ -204,109 +240,117 @@ const ofRootSpan = (eb: EventsExpressionBuilder, column: `e.${EventsColumn}`) =>const countLevel = (eb: EventsExpressionBuilder, level: string) =>countIf(eb("e.level", "=", literal(level)));+type TraceAggregationField = (typeof TRACE_AGGREGATION_FIELDS)[number];+/**- * One row per trace, aggregated from its events. Grouped by- * `trace_id, project_id`.+ * Trace-level aggregates over events for `fields`, in that order: one row per+ * trace. Grouped by `trace_id, project_id`.*/-export const traceAggregationSelects = scopedToEvents(- (eb) =>- [- eb.ref("e.trace_id").as("id"),- "e.project_id as project_id",- eb.fn- .coalesce(- nullIf(- argMaxIf(- eb.ref("e.trace_name"),- eb.ref("e.event_ts"),- eb("e.trace_name", "<>", literal("")),- ),- literal(""),+export const traceAggregationSelects = pickSelects(traceAggregationSelectMap);++/** Trace-level aggregates, keyed by field. */+function traceAggregationSelectMap(eb: EventsExpressionBuilder) {+ return {+ id: eb.ref("e.trace_id").as("id"),+ project_id: "e.project_id as project_id",+ name: eb.fn+ .coalesce(+ nullIf(+ argMaxIf(+ eb.ref("e.trace_name"),+ eb.ref("e.event_ts"),+ eb("e.trace_name", "<>", literal("")),),- nullIf(- argMaxIf(- eb.ref("e.name"),- eb.ref("e.event_ts"),- eb.and([- eventsIsRootObservation(eb),- eb("e.name", "<>", literal("")),- ]),- ),- literal(""),+ literal(""),+ ),+ nullIf(+ argMaxIf(+ eb.ref("e.name"),+ eb.ref("e.event_ts"),+ eb.and([+ eventsIsRootObservation(eb),+ eb("e.name", "<>", literal("")),+ ]),),- )- .as("name"),- eb.fn.min("e.start_time").as("timestamp"),- latestNonEmptyOf(eb, "e.environment").as("environment"),- latestNonEmptyOf(eb, "e.version").as("version"),- latestNonEmptyOf(eb, "e.session_id").as("session_id"),- latestNonEmptyOf(eb, "e.user_id").as("user_id"),- ofRootSpan(eb, "e.input").as("input"),- ofRootSpan(eb, "e.output").as("output"),- argMaxIf(- eventsMetadataMap(eb),- eb.ref("e.event_ts"),- eb("e.parent_span_id", "=", literal("")),- ).as("metadata"),- eb.fn.min("e.created_at").as("created_at"),- eb.fn.max("e.updated_at").as("updated_at"),- eb.fn.sum("e.total_cost").as("total_cost"),- ifElse(- eb(eb.fn.max("e.end_time"), "is", null),- literal(null),- dateDiff(- "millisecond",- eb.fn.min("e.start_time"),- greatest(eb.fn.max("e.start_time"), eb.fn.max("e.end_time")),+ literal(""),),- ).as("latency_milliseconds"),+ )+ .as("name"),+ timestamp: eb.fn.min("e.start_time").as("timestamp"),+ environment: latestNonEmptyOf(eb, "e.environment").as("environment"),+ version: latestNonEmptyOf(eb, "e.version").as("version"),+ session_id: latestNonEmptyOf(eb, "e.session_id").as("session_id"),+ user_id: latestNonEmptyOf(eb, "e.user_id").as("user_id"),+ input: ofRootSpan(eb, "e.input").as("input"),+ output: ofRootSpan(eb, "e.output").as("output"),+ metadata: argMaxIf(+ eventsMetadataMap(eb),+ eb.ref("e.event_ts"),+ eb("e.parent_span_id", "=", literal("")),+ ).as("metadata"),+ created_at: eb.fn.min("e.created_at").as("created_at"),+ updated_at: eb.fn.max("e.updated_at").as("updated_at"),+ total_cost: eb.fn.sum("e.total_cost").as("total_cost"),+ latency_milliseconds: ifElse(+ eb(eb.fn.max("e.end_time"), "is", null),+ literal(null),+ dateDiff(+ "millisecond",+ eb.fn.min("e.start_time"),+ greatest(eb.fn.max("e.start_time"), eb.fn.max("e.end_time")),+ ),+ ).as("latency_milliseconds"),+ observation_ids: groupUniqArrayIf(+ eb.ref("e.span_id"),+ eb("e.span_id", "<>", literal("")),+ ).as("observation_ids"),+ observation_count: arrayLength(groupUniqArrayIf(eb.ref("e.span_id"),- eb("e.span_id", "<>", literal("")),- ).as("observation_ids"),- arrayLength(- groupUniqArrayIf(- eb.ref("e.span_id"),- eb.and([- eb("e.span_id", "<>", literal("")),- eb("e.span_id", "<>", concat(literal("t-"), eb.ref("e.trace_id"))),- ]),- ),- ).as("observation_count"),- ofRootSpan(eb, "e.bookmarked").as("bookmarked"),- eb.fn.max("e.public").as("public"),- latestNonEmptyOf(eb, "e.experiment_item_id").as("experiment_item_id"),- sumMap(eb.ref("e.usage_details")).as("usage_details"),- sumMap(eb.ref("e.cost_details")).as("cost_details"),- multiIf(- ["ERROR", "WARNING", "DEFAULT"].map(- (level) =>- [- arrayExists(- (x) => eb(x, "=", literal(level)),- groupArray(eb.ref("e.level")),- ),- literal(level),- ] as const,- ),- literal("DEBUG"),- ).as("aggregated_level"),- countLevel(eb, "WARNING").as("warning_count"),- countLevel(eb, "ERROR").as("error_count"),- countLevel(eb, "DEFAULT").as("default_count"),- countLevel(eb, "DEBUG").as("debug_count"),- argMaxIf(- eb.ref("e.tags"),- eb.ref("e.event_ts"),- notEmpty(eb.ref("e.tags")),- ).as("tags"),- latestNonEmptyOf(eb, "e.release").as("release"),- // Evaluator execution fields are stamped on every internal evaluation span.- latestNonEmptyOf(eb, "e.evaluator_id").as("evaluator_id"),- latestNonEmptyOf(eb, "e.evaluation_rule_id").as("evaluation_rule_id"),- anyValue(eb.ref("e.experiment_id")).as("experiment_id"),- ] satisfies EventsSelection[],-);+ eb.and([+ eb("e.span_id", "<>", literal("")),+ eb("e.span_id", "<>", concat(literal("t-"), eb.ref("e.trace_id"))),+ ]),+ ),+ ).as("observation_count"),+ bookmarked: ofRootSpan(eb, "e.bookmarked").as("bookmarked"),+ public: eb.fn.max("e.public").as("public"),+ experiment_item_id: latestNonEmptyOf(eb, "e.experiment_item_id").as(+ "experiment_item_id",+ ),+ usage_details: sumMap(eb.ref("e.usage_details")).as("usage_details"),+ cost_details: sumMap(eb.ref("e.cost_details")).as("cost_details"),+ aggregated_level: multiIf(+ ["ERROR", "WARNING", "DEFAULT"].map(+ (level) =>+ [+ arrayExists(+ (x) => eb(x, "=", literal(level)),+ groupArray(eb.ref("e.level")),+ ),+ literal(level),+ ] as const,+ ),+ literal("DEBUG"),+ ).as("aggregated_level"),+ warning_count: countLevel(eb, "WARNING").as("warning_count"),+ error_count: countLevel(eb, "ERROR").as("error_count"),+ default_count: countLevel(eb, "DEFAULT").as("default_count"),+ debug_count: countLevel(eb, "DEBUG").as("debug_count"),+ tags: argMaxIf(+ eb.ref("e.tags"),+ eb.ref("e.event_ts"),+ notEmpty(eb.ref("e.tags")),+ ).as("tags"),+ release: latestNonEmptyOf(eb, "e.release").as("release"),+ // Evaluator execution fields are stamped on every internal evaluation span.+ evaluator_id: latestNonEmptyOf(eb, "e.evaluator_id").as("evaluator_id"),+ evaluation_rule_id: latestNonEmptyOf(eb, "e.evaluation_rule_id").as(+ "evaluation_rule_id",+ ),+ experiment_id: anyValue(eb.ref("e.experiment_id")).as("experiment_id"),+ } satisfies Record<TraceAggregationField, EventsSelection>;+}/*** Session-level aggregates over events for `fields`, in that order. Grouped by@@ -572,7 +616,7 @@ export const eventsMetricsSecondsSelects = scopedToEvents(/** {@link traceAggregationSelects} as SQL text, keyed by output name. */export const TRACE_AGGREGATION_SQL = renderEventsSelects((eb) =>- traceAggregationSelects(eb),+ traceAggregationSelects(TRACE_AGGREGATION_FIELDS)(eb),);/** Every session aggregate as SQL text, keyed by field. */diff --git a/packages/shared/src/server/repositories/events-selects.types.assert.ts b/packages/shared/src/server/repositories/events-selects.types.assert.tsindex 0cbc0902fc..7450ca9413 100644--- a/packages/shared/src/server/repositories/events-selects.types.assert.ts+++ b/packages/shared/src/server/repositories/events-selects.types.assert.ts@@ -7,6 +7,7 @@ import {eventsMetricsSecondsSelects,eventsTraceName,scopedToEvents,+ TRACE_AGGREGATION_FIELDS,traceAggregationSelects,} from "./events-selects";@@ -19,27 +20,28 @@ import {*/export function eventsSelectsScopeTypeAssertions(): void {const db = getClickhouseKysely();+ const allTraceAggregates = traceAggregationSelects(TRACE_AGGREGATION_FIELDS);- db.selectFrom("events_core as e").select(traceAggregationSelects);- db.selectFrom("events_full as e").select(traceAggregationSelects);- db.selectFrom("events as e").select(traceAggregationSelects);+ db.selectFrom("events_core as e").select(allTraceAggregates);+ db.selectFrom("events_full as e").select(allTraceAggregates);+ db.selectFrom("events as e").select(allTraceAggregates);db.selectFrom("events as e").innerJoin("traces as t", "t.id", "e.trace_id")// @ts-expect-error a join- .select(traceAggregationSelects);+ .select(allTraceAggregates);db.with("e", (qb) => qb.selectFrom("events_core").selectAll()).selectFrom("traces as t")// @ts-expect-error a CTE named `e` is not read in FROM- .select(traceAggregationSelects);+ .select(allTraceAggregates);db.selectFrom("traces as t")// @ts-expect-error no `e` in scope- .select(traceAggregationSelects);+ .select(allTraceAggregates);db.selectFrom("events_core")// @ts-expect-error the events table is not aliased `e`- .select(traceAggregationSelects);+ .select(allTraceAggregates);db.selectFrom("traces as t").selectAll()diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex 5b9ec644b3..b3dd485687 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -144,6 +144,7 @@ import {eventsTraceName,observationByIdSelects,sessionAggregationSelects,+ TRACE_AGGREGATION_FIELDS,traceAggregationSelects,type SessionEventsField,type SessionEventsMetricsRow,@@ -190,6 +191,17 @@ import type {SkillsResourceLoaded,} from "../../features/skills/trace";+/** Trace aggregates a session's trace identifiers read: no input/output, so `events` routes to `events_core`. */+const SESSION_TRACE_FIELDS = [+ "id",+ "project_id",+ "name",+ "timestamp",+ "environment",+ "session_id",+ "user_id",+] as const;+/** The `input`/`output` pair an I/O select reads from one relation. */type IoColumns = { input: string; output: string };@@ -1205,7 +1217,7 @@ export const getTraceByIdFromEventsTable = async ({const tracesCte = (qb: QueryCreator<ClickHouseDatabase>) =>qb.selectFrom(truncated ? "events_core as e" : "events_full as e")- .select((eb) => traceAggregationSelects(eb))+ .select((eb) => traceAggregationSelects(TRACE_AGGREGATION_FIELDS)(eb)).where("trace_id", "in", [traceId]).$if(fromTimestamp != null, (inner) =>inner.where(@@ -4066,46 +4078,42 @@ export const getTracesIdentifierForSessionFromEvents = async (projectId: string,sessionId: string,) => {- // Build traces CTE using eventsTracesAggregation- const tracesBuilder = eventsTracesAggregation({- projectId,- truncated: true,- orderByTimestamp: false,- });-- tracesBuilder.whereRaw(- `e.trace_id IN (- SELECT DISTINCT trace_id- FROM events_core- WHERE project_id = {projectId: String}- AND session_id = {sessionId: String}- )`,- {- projectId: projectId,- sessionId: sessionId,- },- );-- tracesBuilder.havingRaw("session_id = {sessionId: String}", {- sessionId: sessionId,- });+ /** tracesCte aggregates traces with a span in the session, keeping those whose aggregated session_id equals sessionId */+ const tracesCte = (qb: QueryCreator<ClickHouseDatabase>) =>+ qb+ .selectFrom("events as e")+ .select((eb) => traceAggregationSelects(SESSION_TRACE_FIELDS)(eb))+ .where("e.trace_id", "in", (eb) =>+ eb+ .selectFrom("events")+ .select("trace_id")+ .distinct()+ .where("session_id", "=", sessionId),+ )+ .groupBy(["trace_id", "project_id"])+ .having("session_id", "=", sessionId);- // Build the final query- const queryBuilder = new CTEQueryBuilder()- .withCTEFromBuilder("traces", tracesBuilder)- .from("traces", "t")- .selectColumns(+ /** builder emits `select t.id, t.user_id, … from event_traces t order by t.timestamp limit 1 by t.id, t.project_id` */+ const builder = getClickhouseKysely()+ .with("event_traces", tracesCte)+ .selectFrom("event_traces as t")+ .select(["t.id","t.user_id","t.name","t.timestamp","t.project_id","t.environment",- )- .orderBy("ORDER BY t.timestamp ASC")- .limitBy("t.id", "t.project_id");+ ])+ .orderBy("t.timestamp", "asc")+ .$call(+ limitBy({+ count: 1,+ columns: (eb) => [eb.ref("t.id"), eb.ref("t.project_id")],+ }),+ );- const { query, params } = queryBuilder.buildWithParams();+ const { sql: query, params } = compileClickhouseQuery(builder, projectId);const rows = await queryClickhouse<{id: string;diff --git a/packages/shared/src/server/services/sessions-ui-table-events-service.ts b/packages/shared/src/server/services/sessions-ui-table-events-service.tsindex 4e11f0ffca..7d29809bf8 100644--- a/packages/shared/src/server/services/sessions-ui-table-events-service.ts+++ b/packages/shared/src/server/services/sessions-ui-table-events-service.ts@@ -25,7 +25,10 @@ import {eventsSessionScoresAggregation,} from "../queries/clickhouse-sql/query-fragments";import { queryClickhouse } from "../repositories";-import { traceAggregationSelects } from "../repositories/events-selects";+import {+ TRACE_AGGREGATION_FIELDS,+ traceAggregationSelects,+} from "../repositories/events-selects";import {sessionEventsCols,sessionEventsOrderByCols,@@ -51,7 +54,7 @@ export const getSessionTracesFromEvents = async (props: {const { projectId, sessionId } = props;const builder = getClickhouseKysely().selectFrom("events_full as e")- .select(traceAggregationSelects)+ .select(traceAggregationSelects(TRACE_AGGREGATION_FIELDS)).where("e.session_id", "=", sessionId).groupBy(["trace_id", "project_id"]).orderBy("timestamp", "asc");diff --git a/web/src/__tests__/server/repositories/event-repository.servertest.ts b/web/src/__tests__/server/repositories/event-repository.servertest.tsindex 7f870316f6..d6a304cdfc 100644--- a/web/src/__tests__/server/repositories/event-repository.servertest.ts+++ b/web/src/__tests__/server/repositories/event-repository.servertest.ts@@ -4701,6 +4701,44 @@ describe("Clickhouse Events Repository Test", () => {]);});});++ it("should skip traces whose session spans are all soft-deleted", async () => {+ const uniqueProjectId = randomUUID();+ const sessionId = randomUUID();+ const deletedTraceId = randomUUID();+ const liveTraceId = randomUUID();+ const nowMicro = Date.now() * 1000;++ await createEventsCh([+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: uniqueProjectId,+ trace_id: deletedTraceId,+ session_id: sessionId,+ start_time: nowMicro,+ event_ts: nowMicro,+ is_deleted: 1,+ }),+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: uniqueProjectId,+ trace_id: liveTraceId,+ session_id: sessionId,+ start_time: nowMicro + 1_000,+ event_ts: nowMicro + 1_000,+ }),+ ]);++ await waitForExpect(async () => {+ const traces = await getTracesIdentifierForSessionFromEvents(+ uniqueProjectId,+ sessionId,+ );+ expect(traces.map((trace) => trace.id)).toEqual([liveTraceId]);+ });+ });});maybe("Update methods", () => {