🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(query-ast): share the events metadata map and trace session id from the AST (#18798)
Niklas Semmler2728f90497e0
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.tsindex 91275a2479..d77454f689 100644--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts@@ -7,9 +7,28 @@ import {eventsTableTraceNameAggregationSql,eventsTableTraceNameSelectSql,} from "../../../eventsTable";+import { expressionBuilder } from "kysely";++import { renderClickhouseExpression } from "../../query-ast/compile";+import { metadataMap, traceSessionId } from "../../query-ast/extensions";+import type { ClickHouseDatabase } from "../../query-ast/schema";import { OBSERVATIONS_TO_TRACE_INTERVAL } from "../../repositories/constants";import { FilterList, StringFilter } from "./clickhouse-filter";+const eventsRef = expressionBuilder<+ { e: ClickHouseDatabase["events_core"] },+ "e"+>();++export const EVENTS_METADATA_MAP_SQL = renderClickhouseExpression(+ metadataMap(+ eventsRef.ref("e.metadata_names"),+ eventsRef.ref("e.metadata_values"),+ ),+);++const TRACE_SESSION_ID_SQL = renderClickhouseExpression(traceSessionId());+/*** Extract the output column alias from a field expression (unquoted).* E.g. "e.span_id as id" → "id", 'e.trace_id as "trace_id"' → "trace_id"@@ -173,8 +192,7 @@ const EVENTS_FIELDS = {// I/O & metadata fieldsinput: "e.input",output: "e.output",- metadata:- "mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) as metadata",+ metadata: `${EVENTS_METADATA_MAP_SQL} as metadata`,// Trace-level denormalized fieldstags: "e.tags as tags",release: "e.release as release",@@ -563,14 +581,13 @@ export const EVENTS_AGGREGATION_FIELDS = {environment:"argMaxIf(environment, event_ts, environment <> '') AS environment",version: "argMaxIf(version, event_ts, version <> '') AS version",- session_id: "argMaxIf(session_id, event_ts, session_id <> '') AS session_id",+ session_id: `${TRACE_SESSION_ID_SQL} AS session_id`,user_id: "argMaxIf(user_id, event_ts, user_id <> '') AS user_id",input: "argMaxIf(input, event_ts, parent_span_id = '') AS input",output: "argMaxIf(output, event_ts, parent_span_id = '') AS output",// Note: events_core/events_full tables don't have input_truncated/output_truncated columns.// Truncation is handled by the materialized view for events_core, or by leftUTF8() at query time.- metadata:- "argMaxIf(mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)), event_ts, parent_span_id = '') AS metadata",+ metadata: `argMaxIf(${EVENTS_METADATA_MAP_SQL}, event_ts, parent_span_id = '') AS metadata`,created_at: "min(created_at) AS created_at",updated_at: "max(updated_at) AS updated_at",total_cost: "sum(total_cost) AS total_cost",@@ -1263,9 +1280,7 @@ export class EventsQueryBuilder extends BaseEventsQueryBuilder<) {// For events_core/events_full, just use mapFromArrays with metadata_values directly// The caller should use events_full table if full metadata is needed- fieldExpressions.push(- `mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) as metadata`,- );+ fieldExpressions.push(`${EVENTS_METADATA_MAP_SQL} as metadata`);}// Add raw SELECT expressions (e.g., from CTE joins)@@ -2207,9 +2222,7 @@ export function buildEventsFullTableSplitQuery(opts: {ioSelectParts.push("e.input", "e.output");}if (opts.includeMetadata) {- ioSelectParts.push(- "mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) as metadata",- );+ ioSelectParts.push(`${EVENTS_METADATA_MAP_SQL} as metadata`);}// The tuple semi-join alone cannot prune events_full's primary key (raw// start_time/trace_id vs the toStartOfMinute/xxHash32 key expressions), sodiff --git a/packages/shared/src/server/query-ast/compile.ts b/packages/shared/src/server/query-ast/compile.tsindex 28a6bfbc67..02ed0668d9 100644--- a/packages/shared/src/server/query-ast/compile.ts+++ b/packages/shared/src/server/query-ast/compile.ts@@ -1,5 +1,6 @@import {createQueryId,+ type Expression,type KyselyPlugin,type RootOperationNode,} from "kysely";@@ -9,6 +10,7 @@ import { DedupLoweringPlugin } from "./dedup";import {type ExecutionContext,TenancyInjectionPlugin,+ assertRelationFree,requireExecutionContext,} from "./tenancy";import { validateTypeCompatibility } from "./typecheck";@@ -57,3 +59,18 @@ export function compileClickhouseQuery(params: { ...compiler.namedParameters },};}++/**+ * SQL text of a typed expression, for raw-SQL builders that share an AST+ * definition: `metadataMap(e.metadata_names,+ * e.metadata_values)` → `mapFromArrays(arrayReverse(e.metadata_names),+ * arrayReverse(e.metadata_values))`. Throws on a subquery or raw relation+ * (nothing scopes it to a project) and on a bound value.+ */+export function renderClickhouseExpression(+ expression: Expression<unknown>,+): string {+ const node = expression.toOperationNode();+ assertRelationFree(node);+ return new ClickHouseQueryCompiler().compileExpression(node);+}diff --git a/packages/shared/src/server/query-ast/compiler.ts b/packages/shared/src/server/query-ast/compiler.tsindex 8dcfc8c33c..6cfdaedb6e 100644--- a/packages/shared/src/server/query-ast/compiler.ts+++ b/packages/shared/src/server/query-ast/compiler.ts@@ -31,6 +31,7 @@ import {type ClickHouseSelectQueryNode,type LimitByNode,} from "./nodes";+import { QueryCompileError } from "./errors";import { COLUMN_BIND_TYPES, TABLE_BIND_TYPES } from "./schema";import { assertTenancyStamped } from "./tenancy";@@ -229,6 +230,21 @@ export class ClickHouseQueryCompiler extends DefaultQueryCompiler {return super.compileQuery(node, queryId);}+ /**+ * SQL text of one standalone expression, on a fresh compiler. The caller+ * checks the tree is relation-free. A bind throws: the text is spliced into+ * a query whose parameters this compiler does not own.+ */+ compileExpression(node: OperationNode): string {+ this.visitNode(node);+ if (Object.keys(this.namedParameters).length > 0) {+ throw new QueryCompileError(+ "A standalone expression cannot bind parameters; inline the value or build the whole query on the AST.",+ );+ }+ return this.getSql();+ }+// Emit identifiers unquoted. `clickhouse format` normalizes to unquoted// identifiers, so returning empty wrappers is what keeps raw compiler output// byte-comparable with the golden snapshots. (Column/table names in thisdiff --git a/packages/shared/src/server/query-ast/extensions.test.ts b/packages/shared/src/server/query-ast/extensions.test.tsindex b741be04f9..6ca2018da5 100644--- a/packages/shared/src/server/query-ast/extensions.test.ts+++ b/packages/shared/src/server/query-ast/extensions.test.ts@@ -1,10 +1,23 @@+import { expressionBuilder, sql } from "kysely";import { describe, expect, it } from "vitest";-import { compileClickhouseQuery } from "./compile";+import { compileClickhouseQuery, renderClickhouseExpression } from "./compile";import { getClickhouseKysely } from "./dialect";-import { QueryCompileError, TypeCompatibilityError } from "./errors";-import { arrayMap, leftUTF8, metadataValue } from "./extensions";+import {+ QueryCompileError,+ TypeCompatibilityError,+ UnscopedRelationError,+} from "./errors";+import {+ arrayMap,+ leftUTF8,+ mapValue,+ metadataMap,+ metadataValue,+ traceSessionId,+} from "./extensions";import { ArrayIndexNode } from "./nodes";+import type { ClickHouseDatabase } from "./schema";import { TenancyInjectionPlugin } from "./tenancy";import { defineView, fromView } from "./views";@@ -192,3 +205,41 @@ describe("arrayMap", () => {expect(compiled).toContain("arrayMap(v -> leftUTF8(v, 2), e.tags)");});});++describe("renderClickhouseExpression", () => {+ const eb = expressionBuilder<{ e: ClickHouseDatabase["events_core"] }, "e">();+ const metadata = metadataMap(+ eb.ref("e.metadata_names"),+ eb.ref("e.metadata_values"),+ );++ it("renders a typed expression as the SQL the compiler emits inline", () => {+ expect(renderClickhouseExpression(metadata)).toBe(+ "mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))",+ );+ expect(renderClickhouseExpression(traceSessionId())).toBe(+ "argMaxIf(session_id, event_ts, session_id != '')",+ );+ });++ it("rejects a subquery", () => {+ const subquery = getClickhouseKysely()+ .selectFrom("events_core")+ .select("trace_id");+ expect(() => renderClickhouseExpression(subquery)).toThrow(+ UnscopedRelationError,+ );+ });++ it("rejects a raw fragment that introduces a relation", () => {+ expect(() =>+ renderClickhouseExpression(sql`(select trace_id from events_core)`),+ ).toThrow(UnscopedRelationError);+ });++ it("rejects a bound value", () => {+ expect(() => renderClickhouseExpression(mapValue(metadata, "key"))).toThrow(+ /cannot bind parameters/,+ );+ });+});diff --git a/packages/shared/src/server/query-ast/extensions.ts b/packages/shared/src/server/query-ast/extensions.tsindex ddeff4047c..e389e1b050 100644--- a/packages/shared/src/server/query-ast/extensions.ts+++ b/packages/shared/src/server/query-ast/extensions.ts@@ -226,14 +226,14 @@ export function dateDiff();}-/** `map[key]` subscript with a bound key. */+/** `map[key]` subscript with a bound key, on a map column or expression. */export function mapValue<V = number>(- column: string,+ map: string | Expression<Record<string, V>>,key: string,-): Expression<V> {+): ChExpression<V> {return new ExpressionWrapper<ClickHouseDatabase, keyof ClickHouseDatabase, V>(ArrayIndexNode.create(- columnRef(column),+ typeof map === "string" ? columnRef(map) : map.toOperationNode(),ValueNode.create(key),) as unknown as OperationNode,);@@ -407,9 +407,7 @@ export function tupleIn(}/** `expression != ''`, with the empty string inlined rather than bound. */-export function nonEmpty(- expression: Expression<string>,-): ChExpression<SqlBool> {+function nonEmpty(expression: Expression<string>): ChExpression<SqlBool> {return new ExpressionWrapper(BinaryOperationNode.create(expression.toOperationNode(),@@ -420,7 +418,7 @@ export function nonEmpty(}/** ClickHouse `argMaxIf(value, by, condition)`. */-export function argMaxIf<T>(+function argMaxIf<T>(value: Expression<T>,by: Expression<unknown>,condition: Expression<SqlBool>,@@ -434,6 +432,24 @@ export function argMaxIf<T>();}+/**+ * A trace's session id from its events: the latest non-empty `session_id`,+ * `argMaxIf(session_id, event_ts, session_id != '')`. Reads unqualified+ * columns, for a trace-level aggregate over one events table.+ */+export function traceSessionId(): ChExpression<string> {+ const sessionId = new ExpressionWrapper<+ ClickHouseDatabase,+ keyof ClickHouseDatabase,+ string+ >(columnRef("session_id"));+ return argMaxIf(+ sessionId,+ new ExpressionWrapper(columnRef("event_ts")),+ nonEmpty(sessionId),+ );+}+/*** Lower `metadata[key]` to a traced array-subscript + `indexOf` node.* The key is a bound `ValueNode`, not a SQL literal.diff --git a/packages/shared/src/server/query-ast/tenancy.ts b/packages/shared/src/server/query-ast/tenancy.tsindex f393edd0ed..273a8aead7 100644--- a/packages/shared/src/server/query-ast/tenancy.ts+++ b/packages/shared/src/server/query-ast/tenancy.ts@@ -370,3 +370,29 @@ function rejectUnscopedRawSql(node: unknown): void {rejectUnscopedRawSql(value);}}++/**+ * Rejects a standalone expression that reads a relation: a subquery, or a raw+ * fragment that introduces one. Outside a query there is no tenancy scope to+ * inject into it.+ */+export function assertRelationFree(node: OperationNode): void {+ rejectUnscopedRawSql(node);+ rejectSubqueries(node);+}++function rejectSubqueries(node: unknown): void {+ if (!node || typeof node !== "object") return;+ if (Array.isArray(node)) {+ for (const item of node) rejectSubqueries(item);+ return;+ }+ if ("kind" in node && node.kind === "SelectQueryNode") {+ throw new UnscopedRelationError(+ "A standalone expression cannot contain a subquery: it has no tenancy scope. Compile the query through compileClickhouseQuery() instead.",+ );+ }+ for (const value of Object.values(node)) {+ rejectSubqueries(value);+ }+}diff --git a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snapindex 3eba719374..c845dc649d 100644--- a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap+++ b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap@@ -495,15 +495,235 @@ FORMAT RawBLOB",]`;-exports[`golden: events point-reads & existence family > getAgentGraphDataFromEventsTable 1`] = `+exports[`golden: events metadata map and session resolution > getEventsForAnalyticsIntegrations 1`] = `[{- "fn": "queryClickhouse",+ "fn": "queryClickhouseStream","params": {"param1": "golden-project",- "param2": "golden-trace",+ "param2": "2025-12-01 00:00:00.000","param3": "2026-01-01 00:00:00.000",- "param4": "2026-01-01 01:00:00.000",+ },+ "route": "analytics_integration",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.environment AS environment,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ e.name AS name,+ e.level AS level,+ e.status_message AS status_message,+ e.version AS version,+ e.bookmarked AS bookmarked,+ e.public AS public,+ e.user_id AS user_id,+ e.session_id AS session_id,+ ifNull(+ COALESCE(+ nullIf(e.trace_name, ''),+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)+ ),+ ''+ ) AS trace_name,+ e.start_time AS start_time,+ e.end_time AS end_time,+ e.completion_start_time AS completion_start_time,+ e.created_at AS created_at,+ e.updated_at AS updated_at,+ e.provided_model_name AS provided_model_name,+ e.model_parameters AS model_parameters,+ e.usage_details AS usage_details,+ e.cost_details AS cost_details,+ e.total_cost AS total_cost,+ e.prompt_id AS prompt_id,+ e.prompt_name AS prompt_name,+ e.prompt_version AS prompt_version,+ e.tool_definitions AS tool_definitions,+ e.tool_calls AS tool_calls,+ e.tool_call_names AS tool_call_names,+ e.usage_pricing_tier_id AS usage_pricing_tier_id,+ e.usage_pricing_tier_name AS usage_pricing_tier_name,+ e.tags AS tags,+ e.release AS release,+ e.model_id AS model_id,+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time)) AS latency,+ if(+ isNull(e.completion_start_time),+ NULL,+ dateDiff('millisecond', e.start_time, e.completion_start_time)+ ) AS time_to_first_token,+ e.usage_details['input'] AS input_tokens,+ e.usage_details['output'] AS output_tokens,+ e.usage_details['total'] AS total_tokens,+ mapFromArrays(+ arrayReverse(e.metadata_names),+ arrayReverse(e.metadata_values)+ )['$posthog_session_id'] AS posthog_session_id,+ mapFromArrays(+ arrayReverse(e.metadata_names),+ arrayReverse(e.metadata_values)+ )['$mixpanel_session_id'] AS mixpanel_session_id+FROM events_core AS e+WHERE (e.project_id = {param1:String}) AND (e.start_time >= {param2:DateTime64(3)}) AND (e.start_time < {param3:DateTime64(3)}) AND (e.is_deleted = 0)+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events metadata map and session resolution > getObservationsV2FromEventsTableForPublicApi fields=core+metadata expandMetadataKeys=set 1`] = `+[+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": "golden-project",+ "param2": 51,+ },+ "route": "unknown",+ "sql": "WITH+ base AS+ (+ SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ e.start_time AS start_time,+ e.end_time AS end_time+ FROM events_core AS e+ WHERE e.project_id = {param1:String}+ ORDER BY+ e.project_id DESC,+ toStartOfMinute(e.start_time) DESC,+ e.start_time DESC,+ xxHash32(e.trace_id) DESC,+ e.span_id DESC+ LIMIT {param2:Int32}+ ),+ io AS+ (+ SELECT+ e.span_id AS _io_id,+ e.trace_id AS _io_trace_id,+ e.start_time AS _io_start_time,+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata+ FROM events_full AS e+ WHERE (e.project_id = {param1:String}) AND (e.start_time >= (((+ SELECT (min(start_time), max(start_time))+ FROM base+ )).1)) AND (e.start_time <= (((+ SELECT (min(start_time), max(start_time))+ FROM base+ )).2)) AND ((e.start_time, e.trace_id, e.span_id) IN (+ SELECT+ start_time,+ trace_id,+ id+ FROM base+ ))+ )+SELECT+ b.id AS id,+ b.trace_id AS trace_id,+ b.start_time AS start_time,+ b.end_time AS end_time,+ b.project_id AS project_id,+ b.parent_observation_id AS parent_observation_id,+ b.type AS type,+ i.metadata AS metadata+FROM base AS b+ANY LEFT JOIN io AS i ON (b.start_time = i._io_start_time) AND (b.trace_id = i._io_trace_id) AND (b.id = i._io_id)+ORDER BY+ b.start_time DESC,+ xxHash32(b.trace_id) DESC,+ b.id DESC",+ },+]+`;++exports[`golden: events metadata map and session resolution > getTraceBatchEventStream 1`] = `+[+ {+ "fn": "queryClickhouseStream",+ "params": {+ "param1": [+ "golden-project",+ ],+ "param2": [+ "golden-trace",+ ],+ "param3": [+ TupleParam {+ "values": [+ "golden-project",+ "golden-trace",+ ],+ },+ ],+ "param4": 1764547080000,+ "param5": 1767225720000,+ "param6": [+ "golden-project",+ ],+ "param7": [+ [+ "golden-trace",+ ],+ ],+ "param8": [+ 1764547080000,+ ],+ "param9": [+ 1767225720000,+ ],+ },+ "route": "unknown",+ "sql": "SELECT+ e.tool_definitions AS tool_definitions,+ e.tool_calls AS tool_calls,+ e.tool_call_names AS tool_call_names,+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,+ input,+ output,+ e.project_id,+ e.trace_id,+ e.environment,+ e.span_id,+ e.parent_span_id,+ e.start_time,+ e.event_ts,+ e.type,+ e.name,+ e.level,+ e.status_message+FROM events_full AS e+WHERE (e.project_id IN ({param1:Array(String)})) AND (e.trace_id IN ({param2:Array(String)})) AND ((e.project_id, e.trace_id) IN ({param3:Array(Tuple(String, String))})) AND (xxHash32(e.trace_id) IN (+ SELECT arrayJoin(arrayMap(id -> xxHash32(id), {param2:Array(String)}))+)) AND (+ e.start_time >= fromUnixTimestamp64Milli({param4:Int64})+) AND (e.start_time <= fromUnixTimestamp64Milli({param5:Int64})) AND arrayExists((group_project_id, group_trace_ids, min_ms, max_ms) -> ((e.project_id = group_project_id) AND has(group_trace_ids, e.trace_id) AND ((e.start_time >= fromUnixTimestamp64Milli(min_ms)) AND (e.start_time <= fromUnixTimestamp64Milli(max_ms)))), {param6:Array(String)}, {param7:Array(Array(String))}, {param8:Array(Int64)}, {param9:Array(Int64)})+ORDER BY+ e.project_id ASC,+ e.trace_id ASC",+ },+]+`;++exports[`golden: events point-reads & existence family > getAgentGraphDataFromEventsTable 1`] = `+[+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": "langgraph_node",+ "param2": "langgraph_step",+ "param3": "golden-project",+ "param4": "golden-trace",+ "param5": "2026-01-01 00:00:00.000",+ "param6": "2026-01-01 01:00:00.000",},"route": "unknown","sql": "SELECT@@ -513,10 +733,10 @@ exports[`golden: events point-reads & existence family > getAgentGraphDataFromEve.name AS name,e.start_time AS start_time,e.end_time AS end_time,- mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['langgraph_node'] AS node,- mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['langgraph_step'] AS step+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))[{param1:String}] AS node,+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))[{param2:String}] AS stepFROM events_core AS e-WHERE (e.project_id = {param1:String}) AND (e.trace_id = {param2:String}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time <= {param4:DateTime64(6)})",+WHERE (e.project_id = {param3:String}) AND (e.trace_id = {param4:String}) AND (e.start_time >= {param5:DateTime64(6)}) AND (e.start_time <= {param6:DateTime64(6)})",},]`;diff --git a/packages/shared/src/server/repositories/events.golden.test.ts b/packages/shared/src/server/repositories/events.golden.test.tsindex a48fccad68..5c79475f3e 100644--- a/packages/shared/src/server/repositories/events.golden.test.ts+++ b/packages/shared/src/server/repositories/events.golden.test.ts@@ -28,6 +28,7 @@ vi.mock("./clickhouse", async (importOriginal) => {import {getAgentGraphDataFromEventsTable,+ getEventsForAnalyticsIntegrations,getLastTraceTimestampsByProjectsFromEventsTable,getLatestSdkVersionInfoFromEvents,getObservationByIdFromEventsTable,@@ -36,6 +37,7 @@ import {getObservationIOFieldByteLengthFromEventsTable,getObservationsBatchIOFromEventsTable,getObservationsTraceIdsFromEventsTable,+ getObservationsV2FromEventsTableForPublicApi,getObservationsWithPromptNameFromEvents,getSessionMetricsFromEvents,getTraceByIdFromEventsTable,@@ -45,6 +47,7 @@ import {hasAnyUserFromEventsTable,streamObservationIOFieldFromEventsTable,} from "./events";+import { getTraceBatchEventStream } from "./trace-batch";const FIXED_PROJECT_ID = "golden-project";const FIXED_PROJECT_IDS = ["golden-project", "golden-project-b"] as const;@@ -458,3 +461,51 @@ describeWithClickhouse("golden: events IO payload reads family", () => {expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();});});++describeWithClickhouse(+ "golden: events metadata map and session resolution",+ () => {+ beforeEach(() => {+ resetCaptures();+ });++ it("getObservationsV2FromEventsTableForPublicApi fields=core+metadata expandMetadataKeys=set", async () => {+ await getObservationsV2FromEventsTableForPublicApi({+ projectId: FIXED_PROJECT_ID,+ page: 1,+ limit: 50,+ fields: ["core", "metadata"],+ expandMetadataKeys: ["golden-key"],+ });+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();+ });++ it("getEventsForAnalyticsIntegrations", async () => {+ for await (const _ of getEventsForAnalyticsIntegrations(+ FIXED_PROJECT_ID,+ "golden-project-name",+ FIXED_FROM_TIMESTAMP,+ FIXED_START_TIME,+ )) {+ // The harness streams no rows.+ }+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();+ });++ it("getTraceBatchEventStream", async () => {+ for await (const _ of getTraceBatchEventStream({+ traces: [+ {+ projectId: FIXED_PROJECT_ID,+ traceId: FIXED_TRACE_ID,+ minStart: FIXED_FROM_TIMESTAMP.getTime(),+ maxStart: FIXED_START_TIME.getTime(),+ },+ ],+ })) {+ // The harness streams no rows.+ }+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();+ });+ },+);diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex eb574d06fe..d9cabf9dbb 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -36,7 +36,6 @@ import {import { getClickhouseKysely } from "../query-ast/dialect";import { promptGenerationEvents } from "./prompt-events";import {- argMaxIf,arrayMap,byteLength,count,@@ -46,9 +45,9 @@ import {mapValue,medianExact,metadataMap,- nonEmpty,sumMapValuesWhereKeyContains,toJSONString,+ traceSessionId,tupleIn,uniq,xxHash32,@@ -148,6 +147,7 @@ import {type SessionEventsFields,type SessionEventsMetricsRow,OrderByEntry,+ EVENTS_METADATA_MAP_SQL,} from "../queries/clickhouse-sql/event-query-builder";import {buildEventsFilterOptionColumnQuery,@@ -1055,9 +1055,6 @@ export const getObservationByIdFromEventsTable = async ({return mapped.shift();};-const EVENTS_METADATA_MAP_SQL =- "mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))";-/** ClickHouse DateTime binds stay strings; the column type is Date. */function dateTimeParam(value: string): Date {return value as unknown as Date;@@ -2653,16 +2650,22 @@ export async function getAgentGraphDataFromEventsTable(params: {const builder = db.selectFrom("events_core as e")- .select([- "e.span_id as id",- "e.parent_span_id as parent_observation_id",- "e.type as type",- "e.name as name",- "e.start_time as start_time",- "e.end_time as end_time",- sql.raw(`${EVENTS_METADATA_MAP_SQL}['langgraph_node']`).as("node"),- sql.raw(`${EVENTS_METADATA_MAP_SQL}['langgraph_step']`).as("step"),- ])+ .select((eb) => {+ const metadata = metadataMap(+ eb.ref("e.metadata_names"),+ eb.ref("e.metadata_values"),+ );+ return [+ "e.span_id as id",+ "e.parent_span_id as parent_observation_id",+ "e.type as type",+ "e.name as name",+ "e.start_time as start_time",+ "e.end_time as end_time",+ mapValue(metadata, "langgraph_node").as("node"),+ mapValue(metadata, "langgraph_step").as("step"),+ ];+ }).where("e.trace_id", "=", traceId).where("e.start_time", ">=", dateTimeParam(chMinStartTime)).where("e.start_time", "<=", dateTimeParam(chMaxStartTime));@@ -2855,16 +2858,7 @@ export const getObservationsBatchIOFromEventsTable = async <.select("trace_id").where("trace_id", "in", traceIds).groupBy("trace_id")- .having(- (eb) =>- argMaxIf(- eb.ref("session_id"),- eb.ref("event_ts"),- nonEmpty(eb.ref("session_id")),- ),- "=",- opts.sessionId!,- ),+ .having(traceSessionId(), "=", opts.sessionId!),),).where("e.start_time", ">=", chDateTime(minTimestamp))@@ -3539,8 +3533,8 @@ export const getEventsForAnalyticsIntegrations = async function* ("e.usage_details['output'] as output_tokens","e.usage_details['total'] as total_tokens",// Analytics integration session IDs from metadata (constructed from array columns)- "mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['$posthog_session_id'] as posthog_session_id",- "mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['$mixpanel_session_id'] as mixpanel_session_id",+ `${EVENTS_METADATA_MAP_SQL}['$posthog_session_id'] as posthog_session_id`,+ `${EVENTS_METADATA_MAP_SQL}['$mixpanel_session_id'] as mixpanel_session_id`,).whereRaw("e.start_time >= {minTimestamp: DateTime64(3)} AND e.start_time < {maxTimestamp: DateTime64(3)}",