🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(query-ast): build shared events selects from the calling query's builder (#19038)
Niklas Semmler4da3cea2e54f
diff --git a/knip.jsonc b/knip.jsoncindex 941ecbd65c..76ff4e64ac 100644--- a/knip.jsonc+++ b/knip.jsonc@@ -20,6 +20,10 @@"ignoreBinaries": ["change"],},"packages/shared": {+ "entry": [+ // Compile-time type assertions: `tsc` checks their `@ts-expect-error` cases, nothing imports them.+ "src/**/*types.assert.ts",+ ],"ignoreFiles": [// Standalone helper invoked manually as documented in the adjacent framework-traces README."scripts/seeder/utils/framework-traces/merge-observations.ts",diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.tsindex b4aecdbac8..25e6baed7c 100644--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts@@ -5,7 +5,7 @@ import { renderClickhouseExpression } from "../../query-ast/compile";import {metadataMap,toBool,- traceSessionId,+ latestNonEmpty,} from "../../query-ast/extensions";import type { ClickHouseDatabase } from "../../query-ast/schema";import { OBSERVATIONS_TO_TRACE_INTERVAL } from "../../repositories/constants";@@ -40,7 +40,9 @@ const eventsMetadataMap = metadataMap(const EVENTS_METADATA_MAP_SQL = renderClickhouseExpression(eventsMetadataMap);-const TRACE_SESSION_ID_SQL = renderClickhouseExpression(traceSessionId());+const TRACE_SESSION_ID_SQL = renderClickhouseExpression(+ latestNonEmpty(eventsRef.ref("session_id"), eventsRef.ref("event_ts")),+);/*** Extract the output column alias from a field expression (unquoted).@@ -154,7 +156,7 @@ const EVENTS_FIELDS = {environment: "e.environment as environment",type: "e.type as type",parentObservationId: "e.parent_span_id as parent_observation_id",- isRootObservation: `${renderClickhouseExpression(toBool(eventsIsRootObservation))} as "is_root_observation"`,+ isRootObservation: `${renderClickhouseExpression(toBool(eventsIsRootObservation(eventsRef)))} as "is_root_observation"`,isAppRoot: "e.is_app_root as is_app_root",name: "e.name as name",level: "e.level as level",@@ -164,7 +166,7 @@ const EVENTS_FIELDS = {public: "e.public as public",userId: "e.user_id as user_id",sessionId: "e.session_id as session_id",- traceName: `${renderClickhouseExpression(eventsTraceNameOrEmpty)} as "trace_name"`,+ traceName: `${renderClickhouseExpression(eventsTraceNameOrEmpty(eventsRef))} as "trace_name"`,// Time fieldsstartTime: "e.start_time as start_time",@@ -234,8 +236,8 @@ const EVENTS_FIELDS = {experimentItemVersion: "e.experiment_item_version as experiment_item_version",// Calculated fields- latency: `${renderClickhouseExpression(millisecondsSinceStart("e.end_time"))} as latency`,- timeToFirstToken: `${renderClickhouseExpression(millisecondsSinceStart("e.completion_start_time"))} as "time_to_first_token"`,+ latency: `${renderClickhouseExpression(millisecondsSinceStart(eventsRef, "e.end_time"))} as latency`,+ timeToFirstToken: `${renderClickhouseExpression(millisecondsSinceStart(eventsRef, "e.completion_start_time"))} as "time_to_first_token"`,} as const satisfies Record<EventsFieldKey, string>;function eventsFieldSelectExpressions(fieldKeys: readonly string[]): string[] {diff --git a/packages/shared/src/server/query-ast/README.md b/packages/shared/src/server/query-ast/README.mdindex 312b980d42..d238f6af13 100644--- a/packages/shared/src/server/query-ast/README.md+++ b/packages/shared/src/server/query-ast/README.md@@ -135,6 +135,21 @@ suffix to run there. Promote it to a required check once it has proven stable.`UInt64` is `number` so binds type-check, but query results arrive as rawJSON (dates as strings). Type rows with a read schema and convert them; donot type results with the builder's `InferResult`.+- **Build shared expressions from the calling query's expression builder.**+ A reusable events expression or select list is wrapped in+ `scopedToEvents((eb) => …)` from `../repositories/events-selects.ts`, then+ passed as `.select(traceAggregationSelects)` or called inside+ `.where((eb) => …)` with the query's own `eb`. That only type-checks in a+ query that reads a full events row as `e` and nothing else: no join, no+ narrower `e`. Columns inside such a helper are written `e.…`, so an alias+ of the same name in the calling query can't shadow them. A helper typed+ `ExpressionBuilder<{ e: … }, "e">` checks the alias and the columns its type+ names, but not joins or the rest of the row: fine for a fixed small column+ set (`observationIoSelects` over `{ e: IoColumns }`), not a substitute for+ `scopedToEvents`. A module-level constant built from a detached+ `expressionBuilder()` loses the check entirely: helper results don't carry+ their table scope in their type, and Kysely's `select()` / `where()` accept+ any expression.- **ClickHouse-only clauses use `$call(helper())`** — not fluent buildermethods, so they compose inside CTEs, subqueries, and views. See the recipesbelow.@@ -181,13 +196,13 @@ db.selectFrom("scores as s")// … from scores as s final```-**Filter on a metadata value** (`metadataValue` — lowers `metadata[key]` to a subscript of `metadataMap`, so the last duplicate key wins; like `mapValue`, a string key is bound and a `literal(...)` key is inlined):+**Filter on a metadata value** (`metadataValue` — lowers `metadata[key]` to a subscript of `metadataMap`, so the last duplicate key wins; like `mapValue`, a string key is bound and a `literal(...)` key is inlined; `names` / `values` must be one relation's `metadata_names` / `metadata_values` refs, anything else throws. Events reads use `eventsMetadataValue(eb, key)` from `../repositories/events-selects.ts`):```tsconst filterKey: string = input.metadataKey; // caller-supplied, so bounddb.selectFrom("events as e").select("e.span_id")- .where((eb) => eb(metadataValue("e", filterKey), "=", "x"));+ .where((eb) => eb(eventsMetadataValue(eb, filterKey), "=", "x"));// select e.span_id from events_core as e where … and mapfromarrays(arrayreverse(e.metadata_names), arrayreverse(e.metadata_values))[{p:String}] = {p:String}```@@ -330,9 +345,9 @@ documented-protected API.### Types are asserted at compile time`types.assert.ts` holds `tsc`-only assertions (schema typing, view opacity,-arrayJoin widening, limitBy / useFinal preservation). It is never run; `schema.test.ts`-anchors it so it stays in the build graph. `@ts-expect-error` lines there must-stay live.+arrayJoin widening, limitBy / useFinal preservation). It is never run: `tsc`+checks it through the package's `include`, and knip treats `*types.assert.ts`+as an entry. `@ts-expect-error` lines there must stay live.### Verifying changesdiff --git a/packages/shared/src/server/query-ast/eventsRouting.test.ts b/packages/shared/src/server/query-ast/eventsRouting.test.tsindex 3f13462461..c083088a76 100644--- a/packages/shared/src/server/query-ast/eventsRouting.test.ts+++ b/packages/shared/src/server/query-ast/eventsRouting.test.ts@@ -52,6 +52,9 @@ type EventsEb = ExpressionBuilder<"e">;+const eventsMetadata = (eb: EventsEb, key: string) =>+ metadataValue(eb.ref("e.metadata_names"), eb.ref("e.metadata_values"), key);+describe("logical events routing", () => {it("routes a read of untruncated columns to events_core", () => {expect(@@ -84,7 +87,7 @@ describe("logical events routing", () => {compile(db.selectFrom("events as e")- .select(() => [metadataValue("e", "k").as("v")]),+ .select((eb) => [eventsMetadata(eb, "k").as("v")]),),),).toEqual(["events_full"]);@@ -532,10 +535,10 @@ describe("logical events routing of metadata predicates", () => {);it("keeps a short `=` and `startsWith` on events_core", () => {- expect(route((eb) => eb(metadataValue("e", "k"), "=", "x"))).toEqual([+ expect(route((eb) => eb(eventsMetadata(eb, "k"), "=", "x"))).toEqual(["events_core",]);- expect(route(() => startsWith(metadataValue("e", "k"), "x"))).toEqual([+ expect(route((eb) => startsWith(eventsMetadata(eb, "k"), "x"))).toEqual(["events_core",]);});@@ -547,7 +550,7 @@ describe("logical events routing of metadata predicates", () => {db.selectFrom("events as e").select("e.span_id")- .where((eb) => eb(metadataValue("e", "k"), "=", "x"))+ .where((eb) => eb(eventsMetadata(eb, "k"), "=", "x")).$call(limitBy({ count: 1, columns: (eb) => [eb.ref("e.span_id")] }),),@@ -558,22 +561,22 @@ describe("logical events routing of metadata predicates", () => {it("keeps a negated short `=` on events_core", () => {expect(- route((eb) => eb.not(eb(metadataValue("e", "k"), "=", "x"))),+ route((eb) => eb.not(eb(eventsMetadata(eb, "k"), "=", "x"))),).toEqual(["events_core"]);});it("routes other comparison shapes on a metadata value to events_full", () => {// Operator outside the allow-list.- expect(route((eb) => eb(metadataValue("e", "k"), "!=", "x"))).toEqual([+ expect(route((eb) => eb(eventsMetadata(eb, "k"), "!=", "x"))).toEqual(["events_full",]);// Value on the left.expect(- route((eb) => eb(eb.val("x"), "=", metadataValue("e", "k"))),+ route((eb) => eb(eb.val("x"), "=", eventsMetadata(eb, "k"))),).toEqual(["events_full"]);// Compared to a column, not a bound value.expect(- route((eb) => eb(metadataValue("e", "k"), "=", eb.ref("e.name"))),+ route((eb) => eb(eventsMetadata(eb, "k"), "=", eb.ref("e.name"))),).toEqual(["events_full"]);});@@ -614,8 +617,8 @@ describe("logical events routing agrees with filtersRequireEventsFull", () => {| "does not contain";const astPredicate = (operator: MetadataOperator, value: string) => {- const metadata = metadataValue("e", "k");return (eb: EventsEb): Expression<SqlBool> => {+ const metadata = eventsMetadata(eb, "k");switch (operator) {case "=":return eb(metadata, "=", value);diff --git a/packages/shared/src/server/query-ast/eventsRouting.ts b/packages/shared/src/server/query-ast/eventsRouting.tsindex 21fcd76dbb..2365deffc0 100644--- a/packages/shared/src/server/query-ast/eventsRouting.ts+++ b/packages/shared/src/server/query-ast/eventsRouting.ts@@ -7,9 +7,9 @@* resolves to it and names a column in {@link EVENTS_CORE_TRUNCATED_COLUMNS},* outside the metadata predicates `metadataFilterIsEventsCoreSafe` accepts:*- * metadataValue(e, k) = 'short' → events_core (< 200 code points)- * startsWith(metadataValue(e, k), 'p') → events_core (≤ 200 code points)- * e.input = 'x', select e.input → events_full+ * e.metadata[k] = 'short' → events_core (< 200 code points)+ * startsWith(e.metadata[k], 'p') → events_core (≤ 200 code points)+ * e.input = 'x', select e.input → events_full** A qualifier resolves to the innermost FROM/JOIN alias (or unaliased table* name) of that name. Shapes that cannot be resolved to one relation are rejected@@ -367,8 +367,8 @@ function isCountStar(node: AggregateFunctionNode): boolean {}/**- * The metadata lookup of `metadataValue(e, k) = v` or- * `startsWith(metadataValue(e, k), v)`, for a bound `v` that+ * The metadata lookup of `metadataValue(…, k) = v` or+ * `startsWith(metadataValue(…, k), v)`, for a bound `v` that* `metadataFilterIsEventsCoreSafe` accepts.*/function coreSafeMetadataLookup(diff --git a/packages/shared/src/server/query-ast/extensions.test.ts b/packages/shared/src/server/query-ast/extensions.test.tsindex 4d0505e268..805accf6fd 100644--- a/packages/shared/src/server/query-ast/extensions.test.ts+++ b/packages/shared/src/server/query-ast/extensions.test.ts@@ -1,4 +1,4 @@-import { expressionBuilder, sql } from "kysely";+import { expressionBuilder, sql, type ExpressionBuilder } from "kysely";import { describe, expect, expectTypeOf, it } from "vitest";import { compileClickhouseQuery, renderClickhouseExpression } from "./compile";@@ -14,13 +14,13 @@ import {arrayMap,count,countIf,+ latestNonEmpty,leftUTF8,literal,mapValue,medianExact,metadataMap,metadataValue,- traceSessionId,uniq,uniqIf,} from "./extensions";@@ -31,6 +31,12 @@ import { defineView, fromView } from "./views";const projectId = "proj-1";+const eventsMetadata = (+ eb: ExpressionBuilder<{ e: ClickHouseDatabase["events_core"] }, "e">,+ key: string,+) =>+ metadataValue(eb.ref("e.metadata_names"), eb.ref("e.metadata_values"), key);+describe("schema-typed selection (condition 7)", () => {it("accepts sum() over a numeric column and emits SQL", () => {const qb = getClickhouseKysely()@@ -65,8 +71,8 @@ describe("metadata subscript access (condition 7b)", () => {it("lowers metadata[key] to ArrayIndexNode + mapFromArrays FunctionNode, not RawNode", () => {const qb = getClickhouseKysely().selectFrom("events_core as e")- .select((_eb) => [metadataValue("e", "a").as("a")])- .where((eb) => eb(metadataValue("e", "a"), "=", "x"));+ .select((eb) => [eventsMetadata(eb, "a").as("a")])+ .where((eb) => eb(eventsMetadata(eb, "a"), "=", "x"));const node = qb.withPlugin(new TenancyInjectionPlugin(projectId))@@ -87,8 +93,8 @@ describe("metadata subscript access (condition 7b)", () => {it("emits subscript + bound key param in SELECT and WHERE", () => {const qb = getClickhouseKysely().selectFrom("events_core as e")- .select((_eb) => [metadataValue("e", "a").as("a")])- .where((eb) => eb(metadataValue("e", "a"), "=", "x"));+ .select((eb) => [eventsMetadata(eb, "a").as("a")])+ .where((eb) => eb(eventsMetadata(eb, "a"), "=", "x"));const { sql: compiled, params } = compileClickhouseQuery(qb, projectId);const lower = compiled.toLowerCase();@@ -106,7 +112,7 @@ describe("metadata subscript access (condition 7b)", () => {.selectFrom("events_core as e").select("environment").groupBy("environment")- .having((eb) => eb(metadataValue("e", "a"), "=", "x"));+ .having((eb) => eb(eventsMetadata(eb, "a"), "=", "x"));const { sql: compiled, params } = compileClickhouseQuery(qb, projectId);expect(compiled.toLowerCase()).toContain("having");@@ -115,6 +121,71 @@ describe("metadata subscript access (condition 7b)", () => {);expect(Object.values(params)).toContain("a");});++ // The routing pass keeps a short metadata `=` / `startsWith` on events_core+ // by trusting the lookup's marker, so the lookup may only read the metadata+ // columns of one relation.+ describe("rejects a lookup over anything but one relation's metadata columns", () => {+ const eb = expressionBuilder<+ { e: ClickHouseDatabase["events_core"] },+ "e"+ >();+ it.each([+ // Any expression, not only `input`: the pass doesn't walk the+ // arguments, so a column inside `values` would escape routing.+ [+ "an expression as values",+ () =>+ metadataValue(+ eb.ref("e.metadata_names"),+ arrayMap<string, string>(+ () => eb.ref("e.input"),+ eb.ref("e.metadata_names"),+ ),+ "k",+ ),+ ],+ // Unqualified, the pass can't resolve which relation it reads.+ [+ "unqualified columns",+ () =>+ metadataValue(+ eb.ref("metadata_names"),+ eb.ref("metadata_values"),+ "k",+ ),+ ],+ // Swapped, the map's keys would be the values.+ [+ "swapped columns",+ () =>+ metadataValue(+ eb.ref("e.metadata_values"),+ eb.ref("e.metadata_names"),+ "k",+ ),+ ],+ ])("%s", (_, build) => {+ expect(build).toThrow(QueryCompileError);+ });++ it("rejects names and values from two relations", () => {+ const joined = expressionBuilder<+ {+ e: ClickHouseDatabase["events_core"];+ f: ClickHouseDatabase["events_core"];+ },+ "e" | "f"+ >();+ expect(() =>+ metadataValue(+ joined.ref("e.metadata_names"),+ joined.ref("f.metadata_values"),+ "k",+ ),+ ).toThrow(QueryCompileError);+ });+ });});describe("virtual view (condition 8)", () => {@@ -245,9 +316,11 @@ describe("renderClickhouseExpression", () => {expect(renderClickhouseExpression(metadata)).toBe("mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))",);- expect(renderClickhouseExpression(traceSessionId())).toBe(- "argMaxIf(session_id, event_ts, session_id != '')",- );+ expect(+ renderClickhouseExpression(+ latestNonEmpty(eb.ref("session_id"), eb.ref("event_ts")),+ ),+ ).toBe("argMaxIf(session_id, event_ts, session_id != '')");});it("rejects a subquery", () => {diff --git a/packages/shared/src/server/query-ast/extensions.ts b/packages/shared/src/server/query-ast/extensions.tsindex 30b32a92e5..df19aca9b6 100644--- a/packages/shared/src/server/query-ast/extensions.ts+++ b/packages/shared/src/server/query-ast/extensions.ts@@ -272,6 +272,13 @@ export function sumMapValuesWhereKeyContains();}+/**+ * A helper's result. Its table scope is every table: Kysely doesn't check an+ * expression's scope where it is used (`ExpressionWrapper<DB, "e", T>` is+ * assignable to `ExpressionWrapper<DB, "t", T>`), so a narrower type would+ * catch nothing. Scope is checked where references are written, so shared+ * expressions take the calling query's `eb` (README, "Using it").+ */type ChExpression<T> = ExpressionWrapper<ClickHouseDatabase,keyof ClickHouseDatabase,@@ -625,34 +632,60 @@ export function argMaxIf<T>(}/**- * A trace's session id from its events: the latest non-empty `session_id`,- * `argMaxIf(session_id, event_ts, session_id != '')`. Reads unqualified- * columns, for a trace-level aggregate over one events table.+ * The latest non-empty `value` by `by` across the group:+ * `argMaxIf(value, by, value != '')`.*/-export function traceSessionId(): ChExpression<string> {- const eb = expressionBuilder<ClickHouseDatabase, "events_core">();- const sessionId = eb.ref("session_id");- return argMaxIf(sessionId, eb.ref("event_ts"), nonEmpty(sessionId));+export function latestNonEmpty(+ value: Expression<string>,+ by: Expression<unknown>,+): ChExpression<string> {+ return argMaxIf(value, by, nonEmpty(value));}/**- * Lower `metadata[key]` to a subscript of {@link metadataMap}, so the last- * entry of a duplicated key wins. Keys follow {@link mapValue}: a string is- * bound, a `literal(...)` is inlined.+ * Lower `metadata[key]` to a subscript of {@link metadataMap} over the events+ * `metadata_names` / `metadata_values` columns, so the last entry of a+ * duplicated key wins. Keys follow {@link mapValue}: a string is bound, a+ * `literal(...)` is inlined. `names` / `values` must be one relation's+ * `metadata_names` / `metadata_values` refs; anything else throws.*/export function metadataValue(- tableAlias: string,+ names: Expression<string[]>,+ values: Expression<string[]>,key: string | Expression<string>,): ChExpression<string> {- const column = (name: string) =>- new ExpressionWrapper<ClickHouseDatabase, "events_core", string[]>(- qualifiedColumn(tableAlias, name),+ assertMetadataColumns(names, values);+ return subscript<string>(metadataMap(names, values), key, true);+}++/**+ * Throws unless `names` / `values` are `q.metadata_names` / `q.metadata_values`+ * of one qualifier `q`. The routing pass trusts the lookup's marker and does+ * not walk these arguments, so they may read nothing else.+ */+function assertMetadataColumns(+ names: Expression<string[]>,+ values: Expression<string[]>,+): void {+ const namesOf = qualifierOf(names.toOperationNode(), "metadata_names");+ const valuesOf = qualifierOf(values.toOperationNode(), "metadata_values");+ if (namesOf === undefined || namesOf !== valuesOf) {+ throw new QueryCompileError(+ 'metadataValue reads the metadata columns of one relation: pass eb.ref("e.metadata_names") and eb.ref("e.metadata_values").',);- return subscript<string>(- metadataMap(column("metadata_names"), column("metadata_values")),- key,- true,- );+ }+}++/** `e` for the reference `e.<column>`; undefined for anything else. */+function qualifierOf(node: OperationNode, column: string): string | undefined {+ if (+ !ReferenceNode.is(node) ||+ !ColumnNode.is(node.column) ||+ node.column.column.name !== column+ ) {+ return undefined;+ }+ return node.table?.table.identifier.name;}// Extract the element type `T` from an array-typed `Expression<T[]>`. `arrayJoin`diff --git a/packages/shared/src/server/query-ast/fixtures/catalog.ts b/packages/shared/src/server/query-ast/fixtures/catalog.tsindex bd0dfa9fe8..2558c50527 100644--- a/packages/shared/src/server/query-ast/fixtures/catalog.ts+++ b/packages/shared/src/server/query-ast/fixtures/catalog.ts@@ -353,7 +353,11 @@ export const CATALOG: CatalogEntry[] = [db().selectFrom("events_core as e").select((eb) => [- metadataValue("e", "model").as("model"),+ metadataValue(+ eb.ref("e.metadata_names"),+ eb.ref("e.metadata_values"),+ "model",+ ).as("model"),eb.fn.countAll().as("n"),]).groupBy("model"),diff --git a/packages/shared/src/server/query-ast/schema.test.ts b/packages/shared/src/server/query-ast/schema.test.tsindex 340d388604..58e61f2e4e 100644--- a/packages/shared/src/server/query-ast/schema.test.ts+++ b/packages/shared/src/server/query-ast/schema.test.ts@@ -7,7 +7,6 @@ import {DEDUP_SPECS,TENANTED_TABLES,} from "./schema";-import { schemaTypeAssertions, extensionTypeAssertions } from "./types.assert";describe("table registry derivation", () => {it("derives the tenanted table set from the registry", () => {@@ -52,11 +51,4 @@ describe("table registry derivation", () => {expect(COLUMN_BIND_TYPES.metadata).toBe("Map(String, String)");expect(COLUMN_BIND_TYPES.cost_details).toBe("Map(String, Decimal(18, 12))");});-- // The assertions themselves are compile-time (`tsc` is the test); this only- // anchors the file so its `@ts-expect-error` checks stay in the build graph.- it("keeps the compile-time type assertions in the build graph", () => {- expect(typeof schemaTypeAssertions).toBe("function");- expect(typeof extensionTypeAssertions).toBe("function");- });});diff --git a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snapindex ee2ad2d9d5..4578dc3c3c 100644--- a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap+++ b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap@@ -2471,8 +2471,8 @@ exports[`golden: events point-reads & existence family > getTraceByIdFromEventsT"sql": "WITH traces AS(SELECT- trace_id AS id,- project_id,+ e.trace_id AS id,+ e.project_id AS project_id,coalesce(nullIf(argMaxIf(e.trace_name, e.event_ts, e.trace_name != ''), ''),nullIf(@@ -2480,53 +2480,53 @@ exports[`golden: events point-reads & existence family > getTraceByIdFromEventsT'')) AS name,- min(start_time) AS timestamp,- argMaxIf(environment, event_ts, environment != '') AS environment,- argMaxIf(version, event_ts, version != '') AS version,- argMaxIf(session_id, event_ts, session_id != '') AS session_id,- argMaxIf(user_id, event_ts, user_id != '') AS user_id,- argMaxIf(input, event_ts, parent_span_id = '') AS input,- argMaxIf(output, event_ts, parent_span_id = '') AS output,+ min(e.start_time) AS timestamp,+ argMaxIf(e.environment, e.event_ts, e.environment != '') AS environment,+ argMaxIf(e.version, e.event_ts, e.version != '') AS version,+ argMaxIf(e.session_id, e.event_ts, e.session_id != '') AS session_id,+ argMaxIf(e.user_id, e.event_ts, e.user_id != '') AS user_id,+ argMaxIf(e.input, e.event_ts, e.parent_span_id = '') AS input,+ argMaxIf(e.output, e.event_ts, e.parent_span_id = '') AS output,argMaxIf(mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)),- event_ts,- parent_span_id = ''+ e.event_ts,+ e.parent_span_id = '') AS metadata,- min(created_at) AS created_at,- max(updated_at) AS updated_at,- sum(total_cost) AS total_cost,+ min(e.created_at) AS created_at,+ max(e.updated_at) AS updated_at,+ sum(e.total_cost) AS total_cost,if(- max(end_time) IS NULL,+ max(e.end_time) IS NULL,NULL,- dateDiff('millisecond', min(start_time), greatest(max(start_time), max(end_time)))+ dateDiff('millisecond', min(e.start_time), greatest(max(e.start_time), max(e.end_time)))) AS latency_milliseconds,- groupUniqArrayIf(span_id, span_id != '') AS observation_ids,+ groupUniqArrayIf(e.span_id, e.span_id != '') AS observation_ids,length(- groupUniqArrayIf(span_id, (span_id != '') AND (span_id != concat('t-', trace_id)))+ groupUniqArrayIf(e.span_id, (e.span_id != '') AND (e.span_id != concat('t-', e.trace_id)))) AS observation_count,- argMaxIf(bookmarked, event_ts, parent_span_id = '') AS bookmarked,- max(public) AS public,- argMaxIf(experiment_item_id, event_ts, experiment_item_id != '') AS experiment_item_id,- sumMap(usage_details) AS usage_details,- sumMap(cost_details) AS cost_details,+ argMaxIf(e.bookmarked, e.event_ts, e.parent_span_id = '') AS bookmarked,+ max(e.public) AS public,+ argMaxIf(e.experiment_item_id, e.event_ts, e.experiment_item_id != '') AS experiment_item_id,+ sumMap(e.usage_details) AS usage_details,+ sumMap(e.cost_details) AS cost_details,multiIf(- arrayExists(x -> (x = 'ERROR'), groupArray(level)),+ arrayExists(x -> (x = 'ERROR'), groupArray(e.level)),'ERROR',- arrayExists(x -> (x = 'WARNING'), groupArray(level)),+ arrayExists(x -> (x = 'WARNING'), groupArray(e.level)),'WARNING',- arrayExists(x -> (x = 'DEFAULT'), groupArray(level)),+ arrayExists(x -> (x = 'DEFAULT'), groupArray(e.level)),'DEFAULT','DEBUG') AS aggregated_level,- countIf(level = 'WARNING') AS warning_count,- countIf(level = 'ERROR') AS error_count,- countIf(level = 'DEFAULT') AS default_count,- countIf(level = 'DEBUG') AS debug_count,- argMaxIf(tags, event_ts, notEmpty(tags)) AS tags,- argMaxIf(release, event_ts, release != '') AS release,- argMaxIf(evaluator_id, event_ts, evaluator_id != '') AS evaluator_id,- argMaxIf(evaluation_rule_id, event_ts, evaluation_rule_id != '') AS evaluation_rule_id,- any(experiment_id) AS experiment_id+ countIf(e.level = 'WARNING') AS warning_count,+ countIf(e.level = 'ERROR') AS error_count,+ countIf(e.level = 'DEFAULT') AS default_count,+ countIf(e.level = 'DEBUG') AS debug_count,+ argMaxIf(e.tags, e.event_ts, notEmpty(e.tags)) AS tags,+ argMaxIf(e.release, e.event_ts, e.release != '') AS release,+ argMaxIf(e.evaluator_id, e.event_ts, e.evaluator_id != '') AS evaluator_id,+ argMaxIf(e.evaluation_rule_id, e.event_ts, e.evaluation_rule_id != '') AS evaluation_rule_id,+ any(e.experiment_id) AS experiment_idFROM events_full AS eWHERE (e.project_id = {param1:String})AND (e.is_deleted = {param2:UInt8})@@ -2578,8 +2578,8 @@ exports[`golden: events point-reads & existence family > getTraceByIdFromEventsT"sql": "WITH traces AS(SELECT- trace_id AS id,- project_id,+ e.trace_id AS id,+ e.project_id AS project_id,coalesce(nullIf(argMaxIf(e.trace_name, e.event_ts, e.trace_name != ''), ''),nullIf(@@ -2587,53 +2587,53 @@ exports[`golden: events point-reads & existence family > getTraceByIdFromEventsT'')) AS name,- min(start_time) AS timestamp,- argMaxIf(environment, event_ts, environment != '') AS environment,- argMaxIf(version, event_ts, version != '') AS version,- argMaxIf(session_id, event_ts, session_id != '') AS session_id,- argMaxIf(user_id, event_ts, user_id != '') AS user_id,- argMaxIf(input, event_ts, parent_span_id = '') AS input,- argMaxIf(output, event_ts, parent_span_id = '') AS output,+ min(e.start_time) AS timestamp,+ argMaxIf(e.environment, e.event_ts, e.environment != '') AS environment,+ argMaxIf(e.version, e.event_ts, e.version != '') AS version,+ argMaxIf(e.session_id, e.event_ts, e.session_id != '') AS session_id,+ argMaxIf(e.user_id, e.event_ts, e.user_id != '') AS user_id,+ argMaxIf(e.input, e.event_ts, e.parent_span_id = '') AS input,+ argMaxIf(e.output, e.event_ts, e.parent_span_id = '') AS output,argMaxIf(mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)),- event_ts,- parent_span_id = ''+ e.event_ts,+ e.parent_span_id = '') AS metadata,- min(created_at) AS created_at,- max(updated_at) AS updated_at,- sum(total_cost) AS total_cost,+ min(e.created_at) AS created_at,+ max(e.updated_at) AS updated_at,+ sum(e.total_cost) AS total_cost,if(- max(end_time) IS NULL,+ max(e.end_time) IS NULL,NULL,- dateDiff('millisecond', min(start_time), greatest(max(start_time), max(end_time)))+ dateDiff('millisecond', min(e.start_time), greatest(max(e.start_time), max(e.end_time)))) AS latency_milliseconds,- groupUniqArrayIf(span_id, span_id != '') AS observation_ids,+ groupUniqArrayIf(e.span_id, e.span_id != '') AS observation_ids,length(- groupUniqArrayIf(span_id, (span_id != '') AND (span_id != concat('t-', trace_id)))+ groupUniqArrayIf(e.span_id, (e.span_id != '') AND (e.span_id != concat('t-', e.trace_id)))) AS observation_count,- argMaxIf(bookmarked, event_ts, parent_span_id = '') AS bookmarked,- max(public) AS public,- argMaxIf(experiment_item_id, event_ts, experiment_item_id != '') AS experiment_item_id,- sumMap(usage_details) AS usage_details,- sumMap(cost_details) AS cost_details,+ argMaxIf(e.bookmarked, e.event_ts, e.parent_span_id = '') AS bookmarked,+ max(e.public) AS public,+ argMaxIf(e.experiment_item_id, e.event_ts, e.experiment_item_id != '') AS experiment_item_id,+ sumMap(e.usage_details) AS usage_details,+ sumMap(e.cost_details) AS cost_details,multiIf(- arrayExists(x -> (x = 'ERROR'), groupArray(level)),+ arrayExists(x -> (x = 'ERROR'), groupArray(e.level)),'ERROR',- arrayExists(x -> (x = 'WARNING'), groupArray(level)),+ arrayExists(x -> (x = 'WARNING'), groupArray(e.level)),'WARNING',- arrayExists(x -> (x = 'DEFAULT'), groupArray(level)),+ arrayExists(x -> (x = 'DEFAULT'), groupArray(e.level)),'DEFAULT','DEBUG') AS aggregated_level,- countIf(level = 'WARNING') AS warning_count,- countIf(level = 'ERROR') AS error_count,- countIf(level = 'DEFAULT') AS default_count,- countIf(level = 'DEBUG') AS debug_count,- argMaxIf(tags, event_ts, notEmpty(tags)) AS tags,- argMaxIf(release, event_ts, release != '') AS release,- argMaxIf(evaluator_id, event_ts, evaluator_id != '') AS evaluator_id,- argMaxIf(evaluation_rule_id, event_ts, evaluation_rule_id != '') AS evaluation_rule_id,- any(experiment_id) AS experiment_id+ countIf(e.level = 'WARNING') AS warning_count,+ countIf(e.level = 'ERROR') AS error_count,+ countIf(e.level = 'DEFAULT') AS default_count,+ countIf(e.level = 'DEBUG') AS debug_count,+ argMaxIf(e.tags, e.event_ts, notEmpty(e.tags)) AS tags,+ argMaxIf(e.release, e.event_ts, e.release != '') AS release,+ argMaxIf(e.evaluator_id, e.event_ts, e.evaluator_id != '') AS evaluator_id,+ argMaxIf(e.evaluation_rule_id, e.event_ts, e.evaluation_rule_id != '') AS evaluation_rule_id,+ any(e.experiment_id) AS experiment_idFROM events_full AS eWHERE (e.project_id = {param1:String})AND (e.is_deleted = {param2:UInt8})@@ -2687,8 +2687,8 @@ exports[`golden: events point-reads & existence family > getTraceByIdFromEventsT"sql": "WITH traces AS(SELECT- trace_id AS id,- project_id,+ e.trace_id AS id,+ e.project_id AS project_id,coalesce(nullIf(argMaxIf(e.trace_name, e.event_ts, e.trace_name != ''), ''),nullIf(@@ -2696,53 +2696,53 @@ exports[`golden: events point-reads & existence family > getTraceByIdFromEventsT'')) AS name,- min(start_time) AS timestamp,- argMaxIf(environment, event_ts, environment != '') AS environment,- argMaxIf(version, event_ts, version != '') AS version,- argMaxIf(session_id, event_ts, session_id != '') AS session_id,- argMaxIf(user_id, event_ts, user_id != '') AS user_id,- argMaxIf(input, event_ts, parent_span_id = '') AS input,- argMaxIf(output, event_ts, parent_span_id = '') AS output,+ min(e.start_time) AS timestamp,+ argMaxIf(e.environment, e.event_ts, e.environment != '') AS environment,+ argMaxIf(e.version, e.event_ts, e.version != '') AS version,+ argMaxIf(e.session_id, e.event_ts, e.session_id != '') AS session_id,+ argMaxIf(e.user_id, e.event_ts, e.user_id != '') AS user_id,+ argMaxIf(e.input, e.event_ts, e.parent_span_id = '') AS input,+ argMaxIf(e.output, e.event_ts, e.parent_span_id = '') AS output,argMaxIf(mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)),- event_ts,- parent_span_id = ''+ e.event_ts,+ e.parent_span_id = '') AS metadata,- min(created_at) AS created_at,- max(updated_at) AS updated_at,- sum(total_cost) AS total_cost,+ min(e.created_at) AS created_at,+ max(e.updated_at) AS updated_at,+ sum(e.total_cost) AS total_cost,if(- max(end_time) IS NULL,+ max(e.end_time) IS NULL,NULL,- dateDiff('millisecond', min(start_time), greatest(max(start_time), max(end_time)))+ dateDiff('millisecond', min(e.start_time), greatest(max(e.start_time), max(e.end_time)))) AS latency_milliseconds,- groupUniqArrayIf(span_id, span_id != '') AS observation_ids,+ groupUniqArrayIf(e.span_id, e.span_id != '') AS observation_ids,length(- groupUniqArrayIf(span_id, (span_id != '') AND (span_id != concat('t-', trace_id)))+ groupUniqArrayIf(e.span_id, (e.span_id != '') AND (e.span_id != concat('t-', e.trace_id)))) AS observation_count,- argMaxIf(bookmarked, event_ts, parent_span_id = '') AS bookmarked,- max(public) AS public,- argMaxIf(experiment_item_id, event_ts, experiment_item_id != '') AS experiment_item_id,- sumMap(usage_details) AS usage_details,- sumMap(cost_details) AS cost_details,+ argMaxIf(e.bookmarked, e.event_ts, e.parent_span_id = '') AS bookmarked,+ max(e.public) AS public,+ argMaxIf(e.experiment_item_id, e.event_ts, e.experiment_item_id != '') AS experiment_item_id,+ sumMap(e.usage_details) AS usage_details,+ sumMap(e.cost_details) AS cost_details,multiIf(- arrayExists(x -> (x = 'ERROR'), groupArray(level)),+ arrayExists(x -> (x = 'ERROR'), groupArray(e.level)),'ERROR',- arrayExists(x -> (x = 'WARNING'), groupArray(level)),+ arrayExists(x -> (x = 'WARNING'), groupArray(e.level)),'WARNING',- arrayExists(x -> (x = 'DEFAULT'), groupArray(level)),+ arrayExists(x -> (x = 'DEFAULT'), groupArray(e.level)),'DEFAULT','DEBUG') AS aggregated_level,- countIf(level = 'WARNING') AS warning_count,- countIf(level = 'ERROR') AS error_count,- countIf(level = 'DEFAULT') AS default_count,- countIf(level = 'DEBUG') AS debug_count,- argMaxIf(tags, event_ts, notEmpty(tags)) AS tags,- argMaxIf(release, event_ts, release != '') AS release,- argMaxIf(evaluator_id, event_ts, evaluator_id != '') AS evaluator_id,- argMaxIf(evaluation_rule_id, event_ts, evaluation_rule_id != '') AS evaluation_rule_id,- any(experiment_id) AS experiment_id+ countIf(e.level = 'WARNING') AS warning_count,+ countIf(e.level = 'ERROR') AS error_count,+ countIf(e.level = 'DEFAULT') AS default_count,+ countIf(e.level = 'DEBUG') AS debug_count,+ argMaxIf(e.tags, e.event_ts, notEmpty(e.tags)) AS tags,+ argMaxIf(e.release, e.event_ts, e.release != '') AS release,+ argMaxIf(e.evaluator_id, e.event_ts, e.evaluator_id != '') AS evaluator_id,+ argMaxIf(e.evaluation_rule_id, e.event_ts, e.evaluation_rule_id != '') AS evaluation_rule_id,+ any(e.experiment_id) AS experiment_idFROM events_full AS eWHERE (e.project_id = {param1:String})AND (e.is_deleted = {param2:UInt8})@@ -2796,8 +2796,8 @@ exports[`golden: events point-reads & existence family > getTraceByIdFromEventsT"sql": "WITH traces AS(SELECT- trace_id AS id,- project_id,+ e.trace_id AS id,+ e.project_id AS project_id,coalesce(nullIf(argMaxIf(e.trace_name, e.event_ts, e.trace_name != ''), ''),nullIf(@@ -2805,53 +2805,53 @@ exports[`golden: events point-reads & existence family > getTraceByIdFromEventsT'')) AS name,- min(start_time) AS timestamp,- argMaxIf(environment, event_ts, environment != '') AS environment,- argMaxIf(version, event_ts, version != '') AS version,- argMaxIf(session_id, event_ts, session_id != '') AS session_id,- argMaxIf(user_id, event_ts, user_id != '') AS user_id,- argMaxIf(input, event_ts, parent_span_id = '') AS input,- argMaxIf(output, event_ts, parent_span_id = '') AS output,+ min(e.start_time) AS timestamp,+ argMaxIf(e.environment, e.event_ts, e.environment != '') AS environment,+ argMaxIf(e.version, e.event_ts, e.version != '') AS version,+ argMaxIf(e.session_id, e.event_ts, e.session_id != '') AS session_id,+ argMaxIf(e.user_id, e.event_ts, e.user_id != '') AS user_id,+ argMaxIf(e.input, e.event_ts, e.parent_span_id = '') AS input,+ argMaxIf(e.output, e.event_ts, e.parent_span_id = '') AS output,argMaxIf(mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)),- event_ts,- parent_span_id = ''+ e.event_ts,+ e.parent_span_id = '') AS metadata,- min(created_at) AS created_at,- max(updated_at) AS updated_at,- sum(total_cost) AS total_cost,+ min(e.created_at) AS created_at,+ max(e.updated_at) AS updated_at,+ sum(e.total_cost) AS total_cost,if(- max(end_time) IS NULL,+ max(e.end_time) IS NULL,NULL,- dateDiff('millisecond', min(start_time), greatest(max(start_time), max(end_time)))+ dateDiff('millisecond', min(e.start_time), greatest(max(e.start_time), max(e.end_time)))) AS latency_milliseconds,- groupUniqArrayIf(span_id, span_id != '') AS observation_ids,+ groupUniqArrayIf(e.span_id, e.span_id != '') AS observation_ids,length(- groupUniqArrayIf(span_id, (span_id != '') AND (span_id != concat('t-', trace_id)))+ groupUniqArrayIf(e.span_id, (e.span_id != '') AND (e.span_id != concat('t-', e.trace_id)))) AS observation_count,- argMaxIf(bookmarked, event_ts, parent_span_id = '') AS bookmarked,- max(public) AS public,- argMaxIf(experiment_item_id, event_ts, experiment_item_id != '') AS experiment_item_id,- sumMap(usage_details) AS usage_details,- sumMap(cost_details) AS cost_details,+ argMaxIf(e.bookmarked, e.event_ts, e.parent_span_id = '') AS bookmarked,+ max(e.public) AS public,+ argMaxIf(e.experiment_item_id, e.event_ts, e.experiment_item_id != '') AS experiment_item_id,+ sumMap(e.usage_details) AS usage_details,+ sumMap(e.cost_details) AS cost_details,multiIf(- arrayExists(x -> (x = 'ERROR'), groupArray(level)),+ arrayExists(x -> (x = 'ERROR'), groupArray(e.level)),'ERROR',- arrayExists(x -> (x = 'WARNING'), groupArray(level)),+ arrayExists(x -> (x = 'WARNING'), groupArray(e.level)),'WARNING',- arrayExists(x -> (x = 'DEFAULT'), groupArray(level)),+ arrayExists(x -> (x = 'DEFAULT'), groupArray(e.level)),'DEFAULT','DEBUG') AS aggregated_level,- countIf(level = 'WARNING') AS warning_count,- countIf(level = 'ERROR') AS error_count,- countIf(level = 'DEFAULT') AS default_count,- countIf(level = 'DEBUG') AS debug_count,- argMaxIf(tags, event_ts, notEmpty(tags)) AS tags,- argMaxIf(release, event_ts, release != '') AS release,- argMaxIf(evaluator_id, event_ts, evaluator_id != '') AS evaluator_id,- argMaxIf(evaluation_rule_id, event_ts, evaluation_rule_id != '') AS evaluation_rule_id,- any(experiment_id) AS experiment_id+ countIf(e.level = 'WARNING') AS warning_count,+ countIf(e.level = 'ERROR') AS error_count,+ countIf(e.level = 'DEFAULT') AS default_count,+ countIf(e.level = 'DEBUG') AS debug_count,+ argMaxIf(e.tags, e.event_ts, notEmpty(e.tags)) AS tags,+ argMaxIf(e.release, e.event_ts, e.release != '') AS release,+ argMaxIf(e.evaluator_id, e.event_ts, e.evaluator_id != '') AS evaluator_id,+ argMaxIf(e.evaluation_rule_id, e.event_ts, e.evaluation_rule_id != '') AS evaluation_rule_id,+ any(e.experiment_id) AS experiment_idFROM events_core AS eWHERE (e.project_id = {param1:String})AND (e.is_deleted = {param2:UInt8})@@ -3560,32 +3560,32 @@ exports[`golden: events simple aggregates family > getSessionMetricsFromEvents f},"route": "unknown","sql": "SELECT- session_id,- max(start_time) AS max_timestamp,- min(start_time) AS min_timestamp,- groupUniqArray(trace_id) AS trace_ids,- groupUniqArrayIf(user_id, (user_id IS NOT NULL) AND (user_id != '')) AS user_ids,- uniq(trace_id) AS trace_count,- groupUniqArrayArrayIf(tags, notEmpty(tags)) AS trace_tags,- argMaxIf(environment, event_ts, environment != '') AS environment,- uniqIf(span_id, parent_span_id != '') AS total_observations,- dateDiff('second', min(start_time), max(if(isNull(end_time), start_time, end_time))) AS duration,- sumMap(usage_details) AS session_usage_details,- sumMap(cost_details) AS session_cost_details,+ e.session_id AS session_id,+ max(e.start_time) AS max_timestamp,+ min(e.start_time) AS min_timestamp,+ groupUniqArray(e.trace_id) AS trace_ids,+ groupUniqArrayIf(e.user_id, (e.user_id IS NOT NULL) AND (e.user_id != '')) AS user_ids,+ uniq(e.trace_id) AS trace_count,+ groupUniqArrayArrayIf(e.tags, notEmpty(e.tags)) AS trace_tags,+ argMaxIf(e.environment, e.event_ts, e.environment != '') AS environment,+ uniqIf(e.span_id, e.parent_span_id != '') AS total_observations,+ dateDiff('second', min(e.start_time), max(if(isNull(e.end_time), e.start_time, e.end_time))) AS duration,+ sumMap(e.usage_details) AS session_usage_details,+ sumMap(e.cost_details) AS session_cost_details,arraySum(- mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param1:String}) > 0), sumMap(cost_details)))+ mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param1:String}) > 0), sumMap(e.cost_details)))) AS session_input_cost,arraySum(- mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param2:String}) > 0), sumMap(cost_details)))+ mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param2:String}) > 0), sumMap(e.cost_details)))) AS session_output_cost,- sumMap(cost_details)['total'] AS session_total_cost,+ sumMap(e.cost_details)['total'] AS session_total_cost,arraySum(- mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param1:String}) > 0), sumMap(usage_details)))+ mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param1:String}) > 0), sumMap(e.usage_details)))) AS session_input_usage,arraySum(- mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param2:String}) > 0), sumMap(usage_details)))+ mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param2:String}) > 0), sumMap(e.usage_details)))) AS session_output_usage,- sumMap(usage_details)['total'] AS session_total_usage+ sumMap(e.usage_details)['total'] AS session_total_usageFROM events_core AS eWHERE (e.project_id = {param3:String})AND (e.is_deleted = {param4:UInt8})@@ -3615,8 +3615,8 @@ exports[`golden: events simple aggregates family > getSessionMetricsFromEvents s},"route": "unknown","sql": "SELECT- session_id,- uniq(trace_id) AS trace_count+ e.session_id AS session_id,+ uniq(e.trace_id) AS trace_countFROM events_core AS eWHERE (e.project_id = {param1:String})AND (e.is_deleted = {param2:UInt8})@@ -3645,8 +3645,8 @@ exports[`golden: events simple aggregates family > getSessionMetricsFromEvents s},"route": "unknown","sql": "SELECT- session_id,- uniq(trace_id) AS trace_count+ e.session_id AS session_id,+ uniq(e.trace_id) AS trace_countFROM events_core AS eWHERE (e.project_id = {param1:String})AND (e.is_deleted = {param2:UInt8})diff --git a/packages/shared/src/server/repositories/events-selects.ts b/packages/shared/src/server/repositories/events-selects.tsindex 96c2b52594..633c077fc3 100644--- a/packages/shared/src/server/repositories/events-selects.ts+++ b/packages/shared/src/server/repositories/events-selects.ts@@ -1,4 +1,9 @@-import { expressionBuilder, sql, type SelectExpression } from "kysely";+import {+ sql,+ type Expression,+ type ExpressionBuilder,+ type SelectExpression,+} from "kysely";import type {SkillsAvailable,@@ -21,8 +26,10 @@ import {ifElse,ifNull,isNull,+ latestNonEmpty,literal,mapValue,+ metadataValue,metadataMap,multiIf,notEmpty,@@ -30,14 +37,13 @@ import {sumMap,sumMapValuesWhereKeyContains,toBool,- traceSessionId,uniq,uniqIf,} from "../query-ast/extensions";import type { ClickHouseDatabase } from "../query-ast/schema";import type { FieldSetFieldKey } from "./events-field-sets";-/** One row of the session aggregation (`SESSION_AGGREGATION_SELECTS`). */+/** One row of the session aggregation (`sessionAggregationSelects`). */export type SessionEventsMetricsRow = {session_id: string;max_timestamp: string;@@ -69,29 +75,40 @@ type StringColumn = {}[EventsColumn];type EventsSelection = SelectExpression<EventsDb, "e">;-const eb = expressionBuilder<EventsDb, "e">();+type EventsExpressionBuilder = ExpressionBuilder<EventsDb, "e">;++const eventsMetadataMap = (eb: EventsExpressionBuilder) =>+ metadataMap(eb.ref("e.metadata_names"), eb.ref("e.metadata_values"));-const eventsMetadataMap = metadataMap(- eb.ref("e.metadata_names"),- eb.ref("e.metadata_values"),+/**+ * The event's metadata value for `key`; the last entry of a duplicated key+ * wins. Keys follow `mapValue`: a string is bound, a `literal(...)` is inlined.+ */+export const eventsMetadataValue = scopedToEvents(+ (eb, key: string | Expression<string>) =>+ metadataValue(eb.ref("e.metadata_names"), eb.ref("e.metadata_values"), key),);/** The event is a root observation: no parent span, or an app root. */-export const eventsIsRootObservation = eb.or([- eb("e.parent_span_id", "=", literal("")),- eb("e.is_app_root", "=", literal(true)),-]);+export const eventsIsRootObservation = scopedToEvents((eb) =>+ eb.or([+ eb("e.parent_span_id", "=", literal("")),+ eb("e.is_app_root", "=", literal(true)),+ ]),+);/*** An event's trace name: the stored `trace_name`, else the name of a root* observation, else NULL.*/-export const eventsTraceName = eb.fn.coalesce(- nullIf(eb.ref("e.trace_name"), literal("")),- ifElse(- eventsIsRootObservation,- nullIf(eb.ref("e.name"), literal("")),- literal(null),+export const eventsTraceName = scopedToEvents((eb) =>+ eb.fn.coalesce(+ nullIf(eb.ref("e.trace_name"), literal("")),+ ifElse(+ eventsIsRootObservation(eb),+ nullIf(eb.ref("e.name"), literal("")),+ literal(null),+ ),),);@@ -101,241 +118,278 @@ export const eventsTraceName = eb.fn.coalesce(* physical String column of the same name once a query also filters and sorts* on it (ClickHouse AMBIGUOUS_COLUMN_NAME).*/-export const eventsTraceNameOrEmpty = ifNull(eventsTraceName, literal(""));+export const eventsTraceNameOrEmpty = scopedToEvents((eb) =>+ ifNull(eventsTraceName(eb), literal("")),+);-const traceNameSelect = eventsTraceNameOrEmpty.as("trace_name");+const traceNameSelect = (eb: EventsExpressionBuilder) =>+ eventsTraceNameOrEmpty(eb).as("trace_name");/** The latest non-empty value of `column` across a trace's events. */-const latestNonEmpty = (column: StringColumn) =>- argMaxIf(eb.ref(column), eb.ref("event_ts"), eb(column, "<>", literal("")));+const latestNonEmptyOf = (+ eb: EventsExpressionBuilder,+ column: `e.${StringColumn}`,+) => latestNonEmpty(eb.ref(column), eb.ref("e.event_ts"));/** `column` of the trace's root observation. */-const ofRootSpan = <C extends EventsColumn>(column: C) =>+const ofRootSpan = (eb: EventsExpressionBuilder, column: `e.${EventsColumn}`) =>argMaxIf(eb.ref(column),- eb.ref("event_ts"),- eb("parent_span_id", "=", literal("")),+ eb.ref("e.event_ts"),+ eb("e.parent_span_id", "=", literal("")),);-const countLevel = (level: string) => countIf(eb("level", "=", literal(level)));+const countLevel = (eb: EventsExpressionBuilder, level: string) =>+ countIf(eb("e.level", "=", literal(level)));/*** One row per trace, aggregated from its events. Grouped by* `trace_id, project_id`.*/-export const TRACE_AGGREGATION_SELECTS = [- eb.ref("trace_id").as("id"),- "project_id",- eb.fn- .coalesce(- nullIf(- argMaxIf(- eb.ref("e.trace_name"),- eb.ref("e.event_ts"),- eb("e.trace_name", "<>", literal("")),+export const traceAggregationSelects = scopedToEvents(+ (eb) =>+ [+ eb.ref("e.trace_id").as("id"),+ "e.project_id as project_id",+ eb.fn+ .coalesce(+ nullIf(+ argMaxIf(+ eb.ref("e.trace_name"),+ eb.ref("e.event_ts"),+ eb("e.trace_name", "<>", literal("")),+ ),+ literal(""),+ ),+ nullIf(+ argMaxIf(+ eb.ref("e.name"),+ eb.ref("e.event_ts"),+ eb.and([+ eventsIsRootObservation(eb),+ eb("e.name", "<>", literal("")),+ ]),+ ),+ literal(""),+ ),+ )+ .as("name"),+ eb.fn.min("e.start_time").as("timestamp"),+ latestNonEmptyOf(eb, "e.environment").as("environment"),+ latestNonEmptyOf(eb, "e.version").as("version"),+ latestNonEmptyOf(eb, "e.session_id").as("session_id"),+ latestNonEmptyOf(eb, "e.user_id").as("user_id"),+ ofRootSpan(eb, "e.input").as("input"),+ ofRootSpan(eb, "e.output").as("output"),+ argMaxIf(+ eventsMetadataMap(eb),+ eb.ref("e.event_ts"),+ eb("e.parent_span_id", "=", literal("")),+ ).as("metadata"),+ eb.fn.min("e.created_at").as("created_at"),+ eb.fn.max("e.updated_at").as("updated_at"),+ eb.fn.sum("e.total_cost").as("total_cost"),+ ifElse(+ eb(eb.fn.max("e.end_time"), "is", null),+ literal(null),+ dateDiff(+ "millisecond",+ eb.fn.min("e.start_time"),+ greatest(eb.fn.max("e.start_time"), eb.fn.max("e.end_time")),),- literal(""),- ),- nullIf(- argMaxIf(- eb.ref("e.name"),- eb.ref("e.event_ts"),- eb.and([eventsIsRootObservation, eb("e.name", "<>", literal(""))]),+ ).as("latency_milliseconds"),+ groupUniqArrayIf(+ eb.ref("e.span_id"),+ eb("e.span_id", "<>", literal("")),+ ).as("observation_ids"),+ arrayLength(+ groupUniqArrayIf(+ eb.ref("e.span_id"),+ eb.and([+ eb("e.span_id", "<>", literal("")),+ eb("e.span_id", "<>", concat(literal("t-"), eb.ref("e.trace_id"))),+ ]),),- literal(""),- ),- )- .as("name"),- eb.fn.min("start_time").as("timestamp"),- latestNonEmpty("environment").as("environment"),- latestNonEmpty("version").as("version"),- traceSessionId().as("session_id"),- latestNonEmpty("user_id").as("user_id"),- ofRootSpan("input").as("input"),- ofRootSpan("output").as("output"),- argMaxIf(- eventsMetadataMap,- eb.ref("event_ts"),- eb("parent_span_id", "=", literal("")),- ).as("metadata"),- eb.fn.min("created_at").as("created_at"),- eb.fn.max("updated_at").as("updated_at"),- eb.fn.sum("total_cost").as("total_cost"),- ifElse(- eb(eb.fn.max("end_time"), "is", null),- literal(null),- dateDiff(- "millisecond",- eb.fn.min("start_time"),- greatest(eb.fn.max("start_time"), eb.fn.max("end_time")),- ),- ).as("latency_milliseconds"),- groupUniqArrayIf(eb.ref("span_id"), eb("span_id", "<>", literal(""))).as(- "observation_ids",- ),- arrayLength(- groupUniqArrayIf(- eb.ref("span_id"),+ ).as("observation_count"),+ ofRootSpan(eb, "e.bookmarked").as("bookmarked"),+ eb.fn.max("e.public").as("public"),+ latestNonEmptyOf(eb, "e.experiment_item_id").as("experiment_item_id"),+ sumMap(eb.ref("e.usage_details")).as("usage_details"),+ sumMap(eb.ref("e.cost_details")).as("cost_details"),+ multiIf(+ ["ERROR", "WARNING", "DEFAULT"].map(+ (level) =>+ [+ arrayExists(+ (x) => eb(x, "=", literal(level)),+ groupArray(eb.ref("e.level")),+ ),+ literal(level),+ ] as const,+ ),+ literal("DEBUG"),+ ).as("aggregated_level"),+ countLevel(eb, "WARNING").as("warning_count"),+ countLevel(eb, "ERROR").as("error_count"),+ countLevel(eb, "DEFAULT").as("default_count"),+ countLevel(eb, "DEBUG").as("debug_count"),+ argMaxIf(+ eb.ref("e.tags"),+ eb.ref("e.event_ts"),+ notEmpty(eb.ref("e.tags")),+ ).as("tags"),+ latestNonEmptyOf(eb, "e.release").as("release"),+ latestNonEmptyOf(eb, "e.evaluator_id").as("evaluator_id"),+ latestNonEmptyOf(eb, "e.evaluation_rule_id").as("evaluation_rule_id"),+ anyValue(eb.ref("e.experiment_id")).as("experiment_id"),+ ] satisfies EventsSelection[],+);++/**+ * Session-level aggregates over events for `fields`, in that order. Grouped by+ * `session_id`.+ */+export const sessionAggregationSelects = pickSelects(+ sessionAggregationSelectMap,+);++/** Session-level aggregates, keyed by field. */+function sessionAggregationSelectMap(eb: EventsExpressionBuilder) {+ return {+ session_id: "e.session_id as session_id",+ max_timestamp: eb.fn.max("e.start_time").as("max_timestamp"),+ min_timestamp: eb.fn.min("e.start_time").as("min_timestamp"),+ trace_ids: groupUniqArray(eb.ref("e.trace_id")).as("trace_ids"),+ user_ids: groupUniqArrayIf(+ eb.ref("e.user_id"),eb.and([- eb("span_id", "<>", literal("")),- eb("span_id", "<>", concat(literal("t-"), eb.ref("trace_id"))),+ eb("e.user_id", "is not", null),+ eb("e.user_id", "!=", literal("")),]),- ),- ).as("observation_count"),- ofRootSpan("bookmarked").as("bookmarked"),- eb.fn.max("public").as("public"),- latestNonEmpty("experiment_item_id").as("experiment_item_id"),- sumMap(eb.ref("usage_details")).as("usage_details"),- sumMap(eb.ref("cost_details")).as("cost_details"),- multiIf(- ["ERROR", "WARNING", "DEFAULT"].map(- (level) =>- [- arrayExists(- (x) => eb(x, "=", literal(level)),- groupArray(eb.ref("level")),- ),- literal(level),- ] as const,- ),- literal("DEBUG"),- ).as("aggregated_level"),- countLevel("WARNING").as("warning_count"),- countLevel("ERROR").as("error_count"),- countLevel("DEFAULT").as("default_count"),- countLevel("DEBUG").as("debug_count"),- argMaxIf(eb.ref("tags"), eb.ref("event_ts"), notEmpty(eb.ref("tags"))).as(- "tags",- ),- latestNonEmpty("release").as("release"),- latestNonEmpty("evaluator_id").as("evaluator_id"),- latestNonEmpty("evaluation_rule_id").as("evaluation_rule_id"),- anyValue(eb.ref("experiment_id")).as("experiment_id"),-] satisfies EventsSelection[];--/** Session-level aggregates over events, keyed by field. Grouped by `session_id`. */-export const SESSION_AGGREGATION_SELECTS = {- session_id: "session_id",- max_timestamp: eb.fn.max("start_time").as("max_timestamp"),- min_timestamp: eb.fn.min("start_time").as("min_timestamp"),- trace_ids: groupUniqArray(eb.ref("trace_id")).as("trace_ids"),- user_ids: groupUniqArrayIf(- eb.ref("user_id"),- eb.and([eb("user_id", "is not", null), eb("user_id", "!=", literal(""))]),- ).as("user_ids"),- trace_count: uniq(eb.ref("trace_id")).as("trace_count"),- trace_tags: groupUniqArrayArrayIf(- eb.ref("tags"),- notEmpty(eb.ref("tags")),- ).as("trace_tags"),- environment: latestNonEmpty("environment").as("environment"),- total_observations: uniqIf(- eb.ref("span_id"),- eb("parent_span_id", "!=", literal("")),- ).as("total_observations"),- duration: dateDiff(- "second",- eb.fn.min("start_time"),- eb.fn.max(- ifElse(- isNull(eb.ref("end_time")),- eb.ref("start_time"),- eb.ref("end_time"),+ ).as("user_ids"),+ trace_count: uniq(eb.ref("e.trace_id")).as("trace_count"),+ trace_tags: groupUniqArrayArrayIf(+ eb.ref("e.tags"),+ notEmpty(eb.ref("e.tags")),+ ).as("trace_tags"),+ environment: latestNonEmptyOf(eb, "e.environment").as("environment"),+ total_observations: uniqIf(+ eb.ref("e.span_id"),+ eb("e.parent_span_id", "!=", literal("")),+ ).as("total_observations"),+ duration: dateDiff(+ "second",+ eb.fn.min("e.start_time"),+ eb.fn.max(+ ifElse(+ isNull(eb.ref("e.end_time")),+ eb.ref("e.start_time"),+ eb.ref("e.end_time"),+ ),),+ ).as("duration"),+ session_usage_details: sumMap(eb.ref("e.usage_details")).as(+ "session_usage_details",),- ).as("duration"),- session_usage_details: sumMap(eb.ref("usage_details")).as(- "session_usage_details",- ),- session_cost_details: sumMap(eb.ref("cost_details")).as(- "session_cost_details",- ),- session_input_cost: sumMapValuesWhereKeyContains(- sumMap(eb.ref("cost_details")),- "input",- ).as("session_input_cost"),- session_output_cost: sumMapValuesWhereKeyContains(- sumMap(eb.ref("cost_details")),- "output",- ).as("session_output_cost"),- session_total_cost: mapValue(- sumMap(eb.ref("cost_details")),- literal("total"),- ).as("session_total_cost"),- session_input_usage: sumMapValuesWhereKeyContains(- sumMap(eb.ref("usage_details")),- "input",- ).as("session_input_usage"),- session_output_usage: sumMapValuesWhereKeyContains(- sumMap(eb.ref("usage_details")),- "output",- ).as("session_output_usage"),- session_total_usage: mapValue(- sumMap(eb.ref("usage_details")),- literal("total"),- ).as("session_total_usage"),-} satisfies Record<SessionEventsField, EventsSelection>;+ session_cost_details: sumMap(eb.ref("e.cost_details")).as(+ "session_cost_details",+ ),+ session_input_cost: sumMapValuesWhereKeyContains(+ sumMap(eb.ref("e.cost_details")),+ "input",+ ).as("session_input_cost"),+ session_output_cost: sumMapValuesWhereKeyContains(+ sumMap(eb.ref("e.cost_details")),+ "output",+ ).as("session_output_cost"),+ session_total_cost: mapValue(+ sumMap(eb.ref("e.cost_details")),+ literal("total"),+ ).as("session_total_cost"),+ session_input_usage: sumMapValuesWhereKeyContains(+ sumMap(eb.ref("e.usage_details")),+ "input",+ ).as("session_input_usage"),+ session_output_usage: sumMapValuesWhereKeyContains(+ sumMap(eb.ref("e.usage_details")),+ "output",+ ).as("session_output_usage"),+ session_total_usage: mapValue(+ sumMap(eb.ref("e.usage_details")),+ literal("total"),+ ).as("session_total_usage"),+ } satisfies Record<SessionEventsField, EventsSelection>;+}/** One observation's fields, without input/output. */-export const OBSERVATION_BY_ID_SELECTS = [- "e.span_id as id",- "e.trace_id as trace_id",- "e.project_id as project_id",- "e.environment as environment",- "e.type as type",- "e.parent_span_id as parent_observation_id",- "e.name as name",- "e.level as level",- "e.status_message as status_message",- "e.version as version",- "e.bookmarked as bookmarked",- "e.public as public",- "e.user_id as user_id",- "e.session_id as session_id",- traceNameSelect,- "e.start_time as start_time",- "e.end_time as end_time",- "e.completion_start_time as completion_start_time",- "e.created_at as created_at",- "e.updated_at as updated_at",- "e.event_ts",- "e.provided_model_name as provided_model_name",- "e.model_id as internal_model_id",- "e.model_parameters as model_parameters",- "e.provided_usage_details as provided_usage_details",- "e.usage_details as usage_details",- "e.provided_cost_details as provided_cost_details",- "e.cost_details as cost_details",- "e.total_cost as total_cost",- "e.prompt_id as prompt_id",- "e.prompt_name as prompt_name",- "e.prompt_version as prompt_version",- sql.ref<SkillsAvailable>("e.skills_available").as("skills_available"),- sql- .ref<SkillsResourceLoaded>("e.skills_resource_loaded")- .as("skills_resource_loaded"),- "e.tool_definitions as tool_definitions",- "e.tool_calls as tool_calls",- "e.tool_call_names as tool_call_names",- "e.usage_pricing_tier_id as usage_pricing_tier_id",- "e.usage_pricing_tier_name as usage_pricing_tier_name",- eventsMetadataMap.as("metadata"),- "e.tags as tags",- "e.release as release",-] satisfies EventsSelection[];+export const observationByIdSelects = scopedToEvents(+ (eb) =>+ [+ "e.span_id as id",+ "e.trace_id as trace_id",+ "e.project_id as project_id",+ "e.environment as environment",+ "e.type as type",+ "e.parent_span_id as parent_observation_id",+ "e.name as name",+ "e.level as level",+ "e.status_message as status_message",+ "e.version as version",+ "e.bookmarked as bookmarked",+ "e.public as public",+ "e.user_id as user_id",+ "e.session_id as session_id",+ traceNameSelect(eb),+ "e.start_time as start_time",+ "e.end_time as end_time",+ "e.completion_start_time as completion_start_time",+ "e.created_at as created_at",+ "e.updated_at as updated_at",+ "e.event_ts",+ "e.provided_model_name as provided_model_name",+ "e.model_id as internal_model_id",+ "e.model_parameters as model_parameters",+ "e.provided_usage_details as provided_usage_details",+ "e.usage_details as usage_details",+ "e.provided_cost_details as provided_cost_details",+ "e.cost_details as cost_details",+ "e.total_cost as total_cost",+ "e.prompt_id as prompt_id",+ "e.prompt_name as prompt_name",+ "e.prompt_version as prompt_version",+ sql.ref<SkillsAvailable>("e.skills_available").as("skills_available"),+ sql+ .ref<SkillsResourceLoaded>("e.skills_resource_loaded")+ .as("skills_resource_loaded"),+ "e.tool_definitions as tool_definitions",+ "e.tool_calls as tool_calls",+ "e.tool_call_names as tool_call_names",+ "e.usage_pricing_tier_id as usage_pricing_tier_id",+ "e.usage_pricing_tier_name as usage_pricing_tier_name",+ eventsMetadataMap(eb).as("metadata"),+ "e.tags as tags",+ "e.release as release",+ ] satisfies EventsSelection[],+);/** Milliseconds from an event's start to `end`, NULL while `end` is unset. */-export const millisecondsSinceStart = (- end: "e.end_time" | "e.completion_start_time",-) =>- ifElse(- isNull(eb.ref(end)),- literal(null),- dateDiff("millisecond", eb.ref("e.start_time"), eb.ref(end)),- );+export const millisecondsSinceStart = scopedToEvents(+ (+ eb: EventsExpressionBuilder,+ end: "e.end_time" | "e.completion_start_time",+ ) =>+ ifElse(+ isNull(eb.ref(end)),+ literal(null),+ dateDiff("millisecond", eb.ref("e.start_time"), eb.ref(end)),+ ),+);/** {@link millisecondsSinceStart} in seconds. */-const secondsSinceStart = (end: "e.end_time" | "e.completion_start_time") =>+const secondsSinceStart = (+ eb: EventsExpressionBuilder,+ end: "e.end_time" | "e.completion_start_time",+) =>ifElse(isNull(eb.ref(end)),literal(null),@@ -359,58 +413,121 @@ type RowFieldSet =| "tools"| "export";+/**+ * Row-level event fields for `keys`, in that order. Pass keys in+ * `eventsFieldSetKeys` order: clustered reads align result blocks by column+ * position.+ */+export const eventsRowFieldSelects = pickSelects(rowFieldSelectMap);+/*** Row-level event fields, keyed like the legacy builder's field sets. Select* them in `eventsFieldSetKeys` order: clustered reads align result blocks by* column position.*/-export const EVENTS_ROW_FIELD_SELECTS = {- id: "e.span_id as id",- traceId: "e.trace_id as trace_id",- projectId: "e.project_id as project_id",- environment: "e.environment as environment",- type: "e.type as type",- parentObservationId: "e.parent_span_id as parent_observation_id",- isRootObservation: toBool(eventsIsRootObservation).as("is_root_observation"),- name: "e.name as name",- level: "e.level as level",- statusMessage: "e.status_message as status_message",- version: "e.version as version",- bookmarked: "e.bookmarked as bookmarked",- public: "e.public as public",- userId: "e.user_id as user_id",- sessionId: "e.session_id as session_id",- traceName: traceNameSelect,- startTime: "e.start_time as start_time",- endTime: "e.end_time as end_time",- completionStartTime: "e.completion_start_time as completion_start_time",- createdAt: "e.created_at as created_at",- updatedAt: "e.updated_at as updated_at",- providedModelName: "e.provided_model_name as provided_model_name",- modelParameters: "e.model_parameters as model_parameters",- usageDetails: "e.usage_details as usage_details",- costDetails: "e.cost_details as cost_details",- totalCost: "e.total_cost as total_cost",- promptId: "e.prompt_id as prompt_id",- promptName: "e.prompt_name as prompt_name",- promptVersion: "e.prompt_version as prompt_version",- toolDefinitions: "e.tool_definitions as tool_definitions",- toolCalls: "e.tool_calls as tool_calls",- toolCallNames: "e.tool_call_names as tool_call_names",- usagePricingTierId: "e.usage_pricing_tier_id as usage_pricing_tier_id",- usagePricingTierName: "e.usage_pricing_tier_name as usage_pricing_tier_name",- metadata: eventsMetadataMap.as("metadata"),- tags: "e.tags as tags",- release: "e.release as release",- modelId: "e.model_id as model_id",- latency: millisecondsSinceStart("e.end_time").as("latency"),- timeToFirstToken: millisecondsSinceStart("e.completion_start_time").as(- "time_to_first_token",- ),-} satisfies Record<FieldSetFieldKey<RowFieldSet>, EventsSelection>;+function rowFieldSelectMap(eb: EventsExpressionBuilder) {+ return {+ id: "e.span_id as id",+ traceId: "e.trace_id as trace_id",+ projectId: "e.project_id as project_id",+ environment: "e.environment as environment",+ type: "e.type as type",+ parentObservationId: "e.parent_span_id as parent_observation_id",+ isRootObservation: toBool(eventsIsRootObservation(eb)).as(+ "is_root_observation",+ ),+ name: "e.name as name",+ level: "e.level as level",+ statusMessage: "e.status_message as status_message",+ version: "e.version as version",+ bookmarked: "e.bookmarked as bookmarked",+ public: "e.public as public",+ userId: "e.user_id as user_id",+ sessionId: "e.session_id as session_id",+ traceName: traceNameSelect(eb),+ startTime: "e.start_time as start_time",+ endTime: "e.end_time as end_time",+ completionStartTime: "e.completion_start_time as completion_start_time",+ createdAt: "e.created_at as created_at",+ updatedAt: "e.updated_at as updated_at",+ providedModelName: "e.provided_model_name as provided_model_name",+ modelParameters: "e.model_parameters as model_parameters",+ usageDetails: "e.usage_details as usage_details",+ costDetails: "e.cost_details as cost_details",+ totalCost: "e.total_cost as total_cost",+ promptId: "e.prompt_id as prompt_id",+ promptName: "e.prompt_name as prompt_name",+ promptVersion: "e.prompt_version as prompt_version",+ toolDefinitions: "e.tool_definitions as tool_definitions",+ toolCalls: "e.tool_calls as tool_calls",+ toolCallNames: "e.tool_call_names as tool_call_names",+ usagePricingTierId: "e.usage_pricing_tier_id as usage_pricing_tier_id",+ usagePricingTierName:+ "e.usage_pricing_tier_name as usage_pricing_tier_name",+ metadata: eventsMetadataMap(eb).as("metadata"),+ tags: "e.tags as tags",+ release: "e.release as release",+ modelId: "e.model_id as model_id",+ latency: millisecondsSinceStart(eb, "e.end_time").as("latency"),+ timeToFirstToken: millisecondsSinceStart(eb, "e.completion_start_time").as(+ "time_to_first_token",+ ),+ } satisfies Record<FieldSetFieldKey<RowFieldSet>, EventsSelection>;+}/** The `metrics` field set with latencies in seconds instead of milliseconds. */-export const EVENTS_METRICS_SECONDS_SELECTS = [- secondsSinceStart("e.end_time").as("latency"),- secondsSinceStart("e.completion_start_time").as("time_to_first_token"),-] satisfies EventsSelection[];+export const eventsMetricsSecondsSelects = scopedToEvents(+ (eb) =>+ [+ secondsSinceStart(eb, "e.end_time").as("latency"),+ secondsSinceStart(eb, "e.completion_start_time").as(+ "time_to_first_token",+ ),+ ] satisfies EventsSelection[],+);++/**+ * Makes `build` callable with the calling query's expression builder, which+ * must satisfy {@link ReadsEvents}. Every export of this module goes through+ * it, and so must any helper that composes them: called with the query's own+ * builder, using one outside a query that reads only events as `e` is a type+ * error. Never pass a query's own builder to a helper typed+ * `ExpressionBuilder<{ e: … }, "e">`: that accepts any query's builder and the+ * check is lost. Wrap the helper in `scopedToEvents` instead.+ */+export function scopedToEvents<A extends unknown[], R>(+ build: (eb: EventsExpressionBuilder, ...args: A) => R,+) {+ return <DB, TB extends keyof DB>(eb: ReadsEvents<DB, TB>, ...args: A): R =>+ // Sound when `eb` is the query's own builder: `ReadsEvents` then proves the+ // query reads only a full events row as `e`.+ build(eb as unknown as EventsExpressionBuilder, ...args);+}++/** For a keyed select map: the selects of the given keys, in their order. */+function pickSelects<M extends Record<string, EventsSelection>>(+ map: (eb: EventsExpressionBuilder) => M,+) {+ return (keys: ReadonlyArray<keyof M>) =>+ scopedToEvents((eb) => {+ const selects = map(eb);+ return keys.map((key) => selects[key]);+ });+}++/**+ * The expression builder of a query that reads only a full events row, as+ * `e`. Anything else resolves to `never`: no `e`, no FROM (`selectNoFrom`), a+ * join (no caller joins; with one, an unqualified column added here later+ * would silently read the other table), or an `e` missing an events column+ * (`observations as e`, a narrower subquery). `ExpressionBuilder` alone would+ * accept the narrower `e`, because it is contravariant in `DB`.+ */+type ReadsEvents<DB, TB extends keyof DB> = ExpressionBuilder<DB, TB> &+ ([DB] extends [EventsDb]+ ? [TB] extends ["e"]+ ? ["e"] extends [TB]+ ? unknown+ : never+ : never+ : never);diff --git a/packages/shared/src/server/repositories/events-selects.types.assert.ts b/packages/shared/src/server/repositories/events-selects.types.assert.tsnew file mode 100644index 0000000000..0cbc0902fc--- /dev/null+++ b/packages/shared/src/server/repositories/events-selects.types.assert.ts@@ -0,0 +1,100 @@+import { expressionBuilder, type ExpressionBuilder } from "kysely";++import { getClickhouseKysely } from "../query-ast/dialect";+import { ifNull, literal } from "../query-ast/extensions";+import type { ClickHouseDatabase } from "../query-ast/schema";+import {+ eventsMetricsSecondsSelects,+ eventsTraceName,+ scopedToEvents,+ traceAggregationSelects,+} from "./events-selects";++/**+ * Compile-time assertions that the shared events expressions keep their table+ * scope: they type-check only in a query that reads a full events row as `e`+ * and nothing else. Never called at runtime; `tsc` is the test, and every+ * `@ts-expect-error` must stay a real error. knip treats `*types.assert.ts` as+ * an entry, so nothing needs to import it.+ */+export function eventsSelectsScopeTypeAssertions(): void {+ const db = getClickhouseKysely();++ db.selectFrom("events_core as e").select(traceAggregationSelects);+ db.selectFrom("events_full as e").select(traceAggregationSelects);+ db.selectFrom("events as e").select(traceAggregationSelects);++ db.selectFrom("events as e")+ .innerJoin("traces as t", "t.id", "e.trace_id")+ // @ts-expect-error a join+ .select(traceAggregationSelects);+ db.with("e", (qb) => qb.selectFrom("events_core").selectAll())+ .selectFrom("traces as t")+ // @ts-expect-error a CTE named `e` is not read in FROM+ .select(traceAggregationSelects);++ db.selectFrom("traces as t")+ // @ts-expect-error no `e` in scope+ .select(traceAggregationSelects);++ db.selectFrom("events_core")+ // @ts-expect-error the events table is not aliased `e`+ .select(traceAggregationSelects);++ db.selectFrom("traces as t")+ .selectAll()+ // @ts-expect-error no `e` in scope+ .where((eb) => eb(eventsTraceName(eb), "is not", null));++ db.selectFrom("observations as e")+ .selectAll()+ // @ts-expect-error `e` lacks events columns (`trace_name`, `is_app_root`)+ .where((eb) => eb(eventsTraceName(eb), "is not", null));++ db.selectFrom(+ db.selectFrom("events as x").select(["x.start_time", "x.end_time"]).as("e"),+ )+ // @ts-expect-error the subquery `e` lacks `completion_start_time`+ .select(eventsMetricsSecondsSelects);++ db.with("e", (qb) => qb.selectFrom("events_core").selectAll())+ // @ts-expect-error no FROM: `e` is never read+ .selectNoFrom((eb) => [eventsTraceName(eb).as("name")]);++ // A helper that composes the exports keeps the check through scopedToEvents.+ const traceNameSelect = scopedToEvents((eb) =>+ eventsTraceName(eb).as("name"),+ );+ db.selectFrom("events_core as e").select(traceNameSelect);+ db.selectFrom("events_core as e")+ .innerJoin("traces as t", "t.id", "e.trace_id")+ // @ts-expect-error a join, through a composed helper+ .select(traceNameSelect);+ db.selectFrom("observations as e")+ // @ts-expect-error `e` lacks events columns, through a composed helper+ .select(traceNameSelect);++ // Intentionally valid at the TS layer: a helper result does not carry the+ // scope of the builder it was made from, and Kysely's `select()` accepts any+ // aliased expression. Build shared expressions from the query's builder.+ const detached = expressionBuilder<+ { e: ClickHouseDatabase["events_core"] },+ "e"+ >();+ db.selectFrom("traces as t").select(+ ifNull(detached.ref("e.trace_name"), literal("")).as("name"),+ );++ // Intentionally valid too: upcasting the query's builder to the plain events+ // builder (here, under a join) drops what the guard checks. Use+ // scopedToEvents instead of typing a parameter as `ExpressionBuilder`.+ db.selectFrom("events_core as e")+ .innerJoin("traces as t", "t.id", "e.trace_id")+ .select((eb) => {+ const events: ExpressionBuilder<+ { e: ClickHouseDatabase["events_core"] },+ "e"+ > = eb;+ return [eventsTraceName(events).as("name")];+ });+}diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex 42220b0de7..fd3f976398 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -41,16 +41,15 @@ import {softDeleteAfterLimit,leftUTF8,limitBy,+ latestNonEmpty,literal,mapValue,medianExact,metadataMap,- metadataValue,sumMapValuesWhereKeyContains,toDate,toJSONString,toStartOfMinute,- traceSessionId,tupleIn,uniq,xxHash32,@@ -137,12 +136,13 @@ import {} from "./observations";import { convertEventsObservation } from "./observations_converters";import {- EVENTS_METRICS_SECONDS_SELECTS,- EVENTS_ROW_FIELD_SELECTS,+ eventsMetadataValue,+ eventsMetricsSecondsSelects,+ eventsRowFieldSelects,eventsTraceName,- OBSERVATION_BY_ID_SELECTS,- SESSION_AGGREGATION_SELECTS,- TRACE_AGGREGATION_SELECTS,+ observationByIdSelects,+ sessionAggregationSelects,+ traceAggregationSelects,type SessionEventsField,type SessionEventsMetricsRow,} from "./events-selects";@@ -1113,7 +1113,7 @@ async function getObservationByIdFromEventsTableInternal({: db.selectFrom("events_core as e")).$call(softDeleteAfterLimit())- .select(OBSERVATION_BY_ID_SELECTS)+ .select(observationByIdSelects).$if(fetchWithInputOutput, (qb) =>qb.select((eb) => observationIoSelects(eb, renderingProps.truncated)),)@@ -1199,7 +1199,7 @@ export const getTraceByIdFromEventsTable = async ({? qb.selectFrom("events_core as e"): qb.selectFrom("events_full as e");return from- .select(TRACE_AGGREGATION_SELECTS)+ .select(traceAggregationSelects).where("trace_id", "in", [traceId]).$if(fromTimestamp != null, (inner) =>inner.where(@@ -2863,7 +2863,13 @@ export const getObservationsBatchIOFromEventsTable = async <.select("trace_id").where("trace_id", "in", traceIds).groupBy("trace_id")- .having(traceSessionId(), "=", opts.sessionId!),+ .having((eb) =>+ eb(+ latestNonEmpty(eb.ref("session_id"), eb.ref("event_ts")),+ "=",+ opts.sessionId!,+ ),+ ),),).where("e.start_time", ">=", minTimestamp)@@ -3422,13 +3428,9 @@ const buildEventsForBlobStorageExportQuery = (const builder = getClickhouseKysely().selectFrom(includeIO ? "events_full as e" : "events_core as e")- .select(- eventsFieldSetKeys(...fieldSets).map(- (key) => EVENTS_ROW_FIELD_SELECTS[key],- ),- )+ .select(eventsRowFieldSelects(eventsFieldSetKeys(...fieldSets))).$if(includeIO, (qb) => qb.select(["input", "output"]))- .$if(metricsInSeconds, (qb) => qb.select(EVENTS_METRICS_SECONDS_SELECTS))+ .$if(metricsInSeconds, (qb) => qb.select(eventsMetricsSecondsSelects)).where("e.start_time", ">=", minTimestamp).where("e.start_time", "<", maxTimestamp).$call(@@ -3529,9 +3531,7 @@ export const getEventsForAnalyticsIntegrations = async function* (.selectFrom("events_core as e")// Use export field set for most fields (id, traceId, name, type, level, version,// environment, userId, sessionId, tags, release, traceName, totalCost, latency, etc.)- .select(- eventsFieldSetKeys("export").map((key) => EVENTS_ROW_FIELD_SELECTS[key]),- )+ .select(eventsRowFieldSelects(eventsFieldSetKeys("export")))// Add analytics-specific computed fields.select((eb) => [// Token counts from usage/cost details@@ -3541,10 +3541,10 @@ export const getEventsForAnalyticsIntegrations = async function* (),mapValue(eb.ref("e.usage_details"), literal("total")).as("total_tokens"),// Analytics integration session IDs from metadata (constructed from array columns)- metadataValue("e", literal("$posthog_session_id")).as(+ eventsMetadataValue(eb, literal("$posthog_session_id")).as("posthog_session_id",),- metadataValue("e", literal("$mixpanel_session_id")).as(+ eventsMetadataValue(eb, literal("$mixpanel_session_id")).as("mixpanel_session_id",),])@@ -3647,13 +3647,13 @@ export const getTraceMetadataByIdsFromEvents = async (props: {const builder = db.selectFrom("events as e")- .select([+ .select((eb) => ["e.trace_id as id",- eventsTraceName.as("name"),+ eventsTraceName(eb).as("name"),"e.user_id as user_id","e.tags as tags",])- .where(eventsTraceName, "is not", null)+ .where((eb) => eb(eventsTraceName(eb), "is not", null)).where("e.trace_id", "in", props.traceIds).$call(limitBy({ count: 1, columns: (eb) => [eb.ref("e.trace_id")] }));@@ -3931,7 +3931,7 @@ export const getSessionMetricsFromEvents = async <const { projectId, sessionIds, fields, queryFromTimestamp } = props;const builder = getClickhouseKysely().selectFrom("events as e")- .select(fields.map((field) => SESSION_AGGREGATION_SELECTS[field]))+ .select(sessionAggregationSelects(fields)).where("session_id", "in", sessionIds).$if(queryFromTimestamp != null, (qb) =>qb.where(diff --git a/packages/shared/src/server/services/__snapshots__/sessions-ui-table-events-service.golden.test.ts.snap b/packages/shared/src/server/services/__snapshots__/sessions-ui-table-events-service.golden.test.ts.snapindex 9e9b173608..c619d56191 100644--- a/packages/shared/src/server/services/__snapshots__/sessions-ui-table-events-service.golden.test.ts.snap+++ b/packages/shared/src/server/services/__snapshots__/sessions-ui-table-events-service.golden.test.ts.snap@@ -11,57 +11,59 @@ exports[`golden: sessions simple aggregates family > getSessionTracesFromEvents},"route": "unknown","sql": "SELECT- trace_id AS id,- project_id,+ e.trace_id AS id,+ e.project_id AS project_id,coalesce(nullIf(argMaxIf(e.trace_name, e.event_ts, e.trace_name != ''), ''),nullIf(argMaxIf(e.name, e.event_ts, ((e.parent_span_id = '') OR (e.is_app_root = true)) AND (e.name != '')), '')) AS name,- min(start_time) AS timestamp,- argMaxIf(environment, event_ts, environment != '') AS environment,- argMaxIf(version, event_ts, version != '') AS version,- argMaxIf(session_id, event_ts, session_id != '') AS session_id,- argMaxIf(user_id, event_ts, user_id != '') AS user_id,- argMaxIf(input, event_ts, parent_span_id = '') AS input,- argMaxIf(output, event_ts, parent_span_id = '') AS output,+ min(e.start_time) AS timestamp,+ argMaxIf(e.environment, e.event_ts, e.environment != '') AS environment,+ argMaxIf(e.version, e.event_ts, e.version != '') AS version,+ argMaxIf(e.session_id, e.event_ts, e.session_id != '') AS session_id,+ argMaxIf(e.user_id, e.event_ts, e.user_id != '') AS user_id,+ argMaxIf(e.input, e.event_ts, e.parent_span_id = '') AS input,+ argMaxIf(e.output, e.event_ts, e.parent_span_id = '') AS output,argMaxIf(mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)),- event_ts,- parent_span_id = ''+ e.event_ts,+ e.parent_span_id = '') AS metadata,- min(created_at) AS created_at,- max(updated_at) AS updated_at,- sum(total_cost) AS total_cost,+ min(e.created_at) AS created_at,+ max(e.updated_at) AS updated_at,+ sum(e.total_cost) AS total_cost,if(- max(end_time) IS NULL,+ max(e.end_time) IS NULL,NULL,- dateDiff('millisecond', min(start_time), greatest(max(start_time), max(end_time)))+ dateDiff('millisecond', min(e.start_time), greatest(max(e.start_time), max(e.end_time)))) AS latency_milliseconds,- groupUniqArrayIf(span_id, span_id != '') AS observation_ids,- length(groupUniqArrayIf(span_id, (span_id != '') AND (span_id != concat('t-', trace_id)))) AS observation_count,- argMaxIf(bookmarked, event_ts, parent_span_id = '') AS bookmarked,- max(public) AS public,- argMaxIf(experiment_item_id, event_ts, experiment_item_id != '') AS experiment_item_id,- sumMap(usage_details) AS usage_details,- sumMap(cost_details) AS cost_details,+ groupUniqArrayIf(e.span_id, e.span_id != '') AS observation_ids,+ length(+ groupUniqArrayIf(e.span_id, (e.span_id != '') AND (e.span_id != concat('t-', e.trace_id)))+ ) AS observation_count,+ argMaxIf(e.bookmarked, e.event_ts, e.parent_span_id = '') AS bookmarked,+ max(e.public) AS public,+ argMaxIf(e.experiment_item_id, e.event_ts, e.experiment_item_id != '') AS experiment_item_id,+ sumMap(e.usage_details) AS usage_details,+ sumMap(e.cost_details) AS cost_details,multiIf(- arrayExists(x -> (x = 'ERROR'), groupArray(level)),+ arrayExists(x -> (x = 'ERROR'), groupArray(e.level)),'ERROR',- arrayExists(x -> (x = 'WARNING'), groupArray(level)),+ arrayExists(x -> (x = 'WARNING'), groupArray(e.level)),'WARNING',- arrayExists(x -> (x = 'DEFAULT'), groupArray(level)),+ arrayExists(x -> (x = 'DEFAULT'), groupArray(e.level)),'DEFAULT','DEBUG') AS aggregated_level,- countIf(level = 'WARNING') AS warning_count,- countIf(level = 'ERROR') AS error_count,- countIf(level = 'DEFAULT') AS default_count,- countIf(level = 'DEBUG') AS debug_count,- argMaxIf(tags, event_ts, notEmpty(tags)) AS tags,- argMaxIf(release, event_ts, release != '') AS release,- argMaxIf(evaluator_id, event_ts, evaluator_id != '') AS evaluator_id,- argMaxIf(evaluation_rule_id, event_ts, evaluation_rule_id != '') AS evaluation_rule_id,- any(experiment_id) AS experiment_id+ countIf(e.level = 'WARNING') AS warning_count,+ countIf(e.level = 'ERROR') AS error_count,+ countIf(e.level = 'DEFAULT') AS default_count,+ countIf(e.level = 'DEBUG') AS debug_count,+ argMaxIf(e.tags, e.event_ts, notEmpty(e.tags)) AS tags,+ argMaxIf(e.release, e.event_ts, e.release != '') AS release,+ argMaxIf(e.evaluator_id, e.event_ts, e.evaluator_id != '') AS evaluator_id,+ argMaxIf(e.evaluation_rule_id, e.event_ts, e.evaluation_rule_id != '') AS evaluation_rule_id,+ any(e.experiment_id) AS experiment_idFROM events_full AS eWHERE (e.project_id = {param1:String})AND (e.is_deleted = {param2:UInt8})diff --git a/packages/shared/src/server/services/sessions-ui-table-events-service.ts b/packages/shared/src/server/services/sessions-ui-table-events-service.tsindex 3cf8fc3727..4e11f0ffca 100644--- a/packages/shared/src/server/services/sessions-ui-table-events-service.ts+++ b/packages/shared/src/server/services/sessions-ui-table-events-service.ts@@ -25,7 +25,7 @@ import {eventsSessionScoresAggregation,} from "../queries/clickhouse-sql/query-fragments";import { queryClickhouse } from "../repositories";-import { TRACE_AGGREGATION_SELECTS } from "../repositories/events-selects";+import { traceAggregationSelects } from "../repositories/events-selects";import {sessionEventsCols,sessionEventsOrderByCols,@@ -51,7 +51,7 @@ export const getSessionTracesFromEvents = async (props: {const { projectId, sessionId } = props;const builder = getClickhouseKysely().selectFrom("events_full as e")- .select(TRACE_AGGREGATION_SELECTS)+ .select(traceAggregationSelects).where("e.session_id", "=", sessionId).groupBy(["trace_id", "project_id"]).orderBy("timestamp", "asc");