Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

refactor(query-ast): share the events metadata map and trace session id from the AST (#18798)

Niklas Semmler2728f90497e0
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
index 91275a2479..d77454f689 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
@@ -7,9 +7,28 @@ import {
eventsTableTraceNameAggregationSql,
eventsTableTraceNameSelectSql,
} from "../../../eventsTable";
+import { expressionBuilder } from "kysely";
+
+import { renderClickhouseExpression } from "../../query-ast/compile";
+import { metadataMap, traceSessionId } from "../../query-ast/extensions";
+import type { ClickHouseDatabase } from "../../query-ast/schema";
import { OBSERVATIONS_TO_TRACE_INTERVAL } from "../../repositories/constants";
import { FilterList, StringFilter } from "./clickhouse-filter";
+const eventsRef = expressionBuilder<
+ { e: ClickHouseDatabase["events_core"] },
+ "e"
+>();
+
+export const EVENTS_METADATA_MAP_SQL = renderClickhouseExpression(
+ metadataMap(
+ eventsRef.ref("e.metadata_names"),
+ eventsRef.ref("e.metadata_values"),
+ ),
+);
+
+const TRACE_SESSION_ID_SQL = renderClickhouseExpression(traceSessionId());
+
/**
* Extract the output column alias from a field expression (unquoted).
* E.g. "e.span_id as id" → "id", 'e.trace_id as "trace_id"' → "trace_id"
@@ -173,8 +192,7 @@ const EVENTS_FIELDS = {
// I/O & metadata fields
input: "e.input",
output: "e.output",
- metadata:
- "mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) as metadata",
+ metadata: `${EVENTS_METADATA_MAP_SQL} as metadata`,
// Trace-level denormalized fields
tags: "e.tags as tags",
release: "e.release as release",
@@ -563,14 +581,13 @@ export const EVENTS_AGGREGATION_FIELDS = {
environment:
"argMaxIf(environment, event_ts, environment <> '') AS environment",
version: "argMaxIf(version, event_ts, version <> '') AS version",
- session_id: "argMaxIf(session_id, event_ts, session_id <> '') AS session_id",
+ session_id: `${TRACE_SESSION_ID_SQL} AS session_id`,
user_id: "argMaxIf(user_id, event_ts, user_id <> '') AS user_id",
input: "argMaxIf(input, event_ts, parent_span_id = '') AS input",
output: "argMaxIf(output, event_ts, parent_span_id = '') AS output",
// Note: events_core/events_full tables don't have input_truncated/output_truncated columns.
// Truncation is handled by the materialized view for events_core, or by leftUTF8() at query time.
- metadata:
- "argMaxIf(mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)), event_ts, parent_span_id = '') AS metadata",
+ metadata: `argMaxIf(${EVENTS_METADATA_MAP_SQL}, event_ts, parent_span_id = '') AS metadata`,
created_at: "min(created_at) AS created_at",
updated_at: "max(updated_at) AS updated_at",
total_cost: "sum(total_cost) AS total_cost",
@@ -1263,9 +1280,7 @@ export class EventsQueryBuilder extends BaseEventsQueryBuilder<
) {
// For events_core/events_full, just use mapFromArrays with metadata_values directly
// The caller should use events_full table if full metadata is needed
- fieldExpressions.push(
- `mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) as metadata`,
- );
+ fieldExpressions.push(`${EVENTS_METADATA_MAP_SQL} as metadata`);
}
// Add raw SELECT expressions (e.g., from CTE joins)
@@ -2207,9 +2222,7 @@ export function buildEventsFullTableSplitQuery(opts: {
ioSelectParts.push("e.input", "e.output");
}
if (opts.includeMetadata) {
- ioSelectParts.push(
- "mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) as metadata",
- );
+ ioSelectParts.push(`${EVENTS_METADATA_MAP_SQL} as metadata`);
}
// The tuple semi-join alone cannot prune events_full's primary key (raw
// start_time/trace_id vs the toStartOfMinute/xxHash32 key expressions), so
diff --git a/packages/shared/src/server/query-ast/compile.ts b/packages/shared/src/server/query-ast/compile.ts
index 28a6bfbc67..02ed0668d9 100644
--- a/packages/shared/src/server/query-ast/compile.ts
+++ b/packages/shared/src/server/query-ast/compile.ts
@@ -1,5 +1,6 @@
import {
createQueryId,
+ type Expression,
type KyselyPlugin,
type RootOperationNode,
} from "kysely";
@@ -9,6 +10,7 @@ import { DedupLoweringPlugin } from "./dedup";
import {
type ExecutionContext,
TenancyInjectionPlugin,
+ assertRelationFree,
requireExecutionContext,
} from "./tenancy";
import { validateTypeCompatibility } from "./typecheck";
@@ -57,3 +59,18 @@ export function compileClickhouseQuery(
params: { ...compiler.namedParameters },
};
}
+
+/**
+ * SQL text of a typed expression, for raw-SQL builders that share an AST
+ * definition: `metadataMap(e.metadata_names,
+ * e.metadata_values)` → `mapFromArrays(arrayReverse(e.metadata_names),
+ * arrayReverse(e.metadata_values))`. Throws on a subquery or raw relation
+ * (nothing scopes it to a project) and on a bound value.
+ */
+export function renderClickhouseExpression(
+ expression: Expression<unknown>,
+): string {
+ const node = expression.toOperationNode();
+ assertRelationFree(node);
+ return new ClickHouseQueryCompiler().compileExpression(node);
+}
diff --git a/packages/shared/src/server/query-ast/compiler.ts b/packages/shared/src/server/query-ast/compiler.ts
index 8dcfc8c33c..6cfdaedb6e 100644
--- a/packages/shared/src/server/query-ast/compiler.ts
+++ b/packages/shared/src/server/query-ast/compiler.ts
@@ -31,6 +31,7 @@ import {
type ClickHouseSelectQueryNode,
type LimitByNode,
} from "./nodes";
+import { QueryCompileError } from "./errors";
import { COLUMN_BIND_TYPES, TABLE_BIND_TYPES } from "./schema";
import { assertTenancyStamped } from "./tenancy";
@@ -229,6 +230,21 @@ export class ClickHouseQueryCompiler extends DefaultQueryCompiler {
return super.compileQuery(node, queryId);
}
+ /**
+ * SQL text of one standalone expression, on a fresh compiler. The caller
+ * checks the tree is relation-free. A bind throws: the text is spliced into
+ * a query whose parameters this compiler does not own.
+ */
+ compileExpression(node: OperationNode): string {
+ this.visitNode(node);
+ if (Object.keys(this.namedParameters).length > 0) {
+ throw new QueryCompileError(
+ "A standalone expression cannot bind parameters; inline the value or build the whole query on the AST.",
+ );
+ }
+ return this.getSql();
+ }
+
// Emit identifiers unquoted. `clickhouse format` normalizes to unquoted
// identifiers, so returning empty wrappers is what keeps raw compiler output
// byte-comparable with the golden snapshots. (Column/table names in this
diff --git a/packages/shared/src/server/query-ast/extensions.test.ts b/packages/shared/src/server/query-ast/extensions.test.ts
index b741be04f9..6ca2018da5 100644
--- a/packages/shared/src/server/query-ast/extensions.test.ts
+++ b/packages/shared/src/server/query-ast/extensions.test.ts
@@ -1,10 +1,23 @@
+import { expressionBuilder, sql } from "kysely";
import { describe, expect, it } from "vitest";
-import { compileClickhouseQuery } from "./compile";
+import { compileClickhouseQuery, renderClickhouseExpression } from "./compile";
import { getClickhouseKysely } from "./dialect";
-import { QueryCompileError, TypeCompatibilityError } from "./errors";
-import { arrayMap, leftUTF8, metadataValue } from "./extensions";
+import {
+ QueryCompileError,
+ TypeCompatibilityError,
+ UnscopedRelationError,
+} from "./errors";
+import {
+ arrayMap,
+ leftUTF8,
+ mapValue,
+ metadataMap,
+ metadataValue,
+ traceSessionId,
+} from "./extensions";
import { ArrayIndexNode } from "./nodes";
+import type { ClickHouseDatabase } from "./schema";
import { TenancyInjectionPlugin } from "./tenancy";
import { defineView, fromView } from "./views";
@@ -192,3 +205,41 @@ describe("arrayMap", () => {
expect(compiled).toContain("arrayMap(v -> leftUTF8(v, 2), e.tags)");
});
});
+
+describe("renderClickhouseExpression", () => {
+ const eb = expressionBuilder<{ e: ClickHouseDatabase["events_core"] }, "e">();
+ const metadata = metadataMap(
+ eb.ref("e.metadata_names"),
+ eb.ref("e.metadata_values"),
+ );
+
+ it("renders a typed expression as the SQL the compiler emits inline", () => {
+ expect(renderClickhouseExpression(metadata)).toBe(
+ "mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))",
+ );
+ expect(renderClickhouseExpression(traceSessionId())).toBe(
+ "argMaxIf(session_id, event_ts, session_id != '')",
+ );
+ });
+
+ it("rejects a subquery", () => {
+ const subquery = getClickhouseKysely()
+ .selectFrom("events_core")
+ .select("trace_id");
+ expect(() => renderClickhouseExpression(subquery)).toThrow(
+ UnscopedRelationError,
+ );
+ });
+
+ it("rejects a raw fragment that introduces a relation", () => {
+ expect(() =>
+ renderClickhouseExpression(sql`(select trace_id from events_core)`),
+ ).toThrow(UnscopedRelationError);
+ });
+
+ it("rejects a bound value", () => {
+ expect(() => renderClickhouseExpression(mapValue(metadata, "key"))).toThrow(
+ /cannot bind parameters/,
+ );
+ });
+});
diff --git a/packages/shared/src/server/query-ast/extensions.ts b/packages/shared/src/server/query-ast/extensions.ts
index ddeff4047c..e389e1b050 100644
--- a/packages/shared/src/server/query-ast/extensions.ts
+++ b/packages/shared/src/server/query-ast/extensions.ts
@@ -226,14 +226,14 @@ export function dateDiff(
);
}
-/** `map[key]` subscript with a bound key. */
+/** `map[key]` subscript with a bound key, on a map column or expression. */
export function mapValue<V = number>(
- column: string,
+ map: string | Expression<Record<string, V>>,
key: string,
-): Expression<V> {
+): ChExpression<V> {
return new ExpressionWrapper<ClickHouseDatabase, keyof ClickHouseDatabase, V>(
ArrayIndexNode.create(
- columnRef(column),
+ typeof map === "string" ? columnRef(map) : map.toOperationNode(),
ValueNode.create(key),
) as unknown as OperationNode,
);
@@ -407,9 +407,7 @@ export function tupleIn(
}
/** `expression != ''`, with the empty string inlined rather than bound. */
-export function nonEmpty(
- expression: Expression<string>,
-): ChExpression<SqlBool> {
+function nonEmpty(expression: Expression<string>): ChExpression<SqlBool> {
return new ExpressionWrapper(
BinaryOperationNode.create(
expression.toOperationNode(),
@@ -420,7 +418,7 @@ export function nonEmpty(
}
/** ClickHouse `argMaxIf(value, by, condition)`. */
-export function argMaxIf<T>(
+function argMaxIf<T>(
value: Expression<T>,
by: Expression<unknown>,
condition: Expression<SqlBool>,
@@ -434,6 +432,24 @@ export function argMaxIf<T>(
);
}
+/**
+ * A trace's session id from its events: the latest non-empty `session_id`,
+ * `argMaxIf(session_id, event_ts, session_id != '')`. Reads unqualified
+ * columns, for a trace-level aggregate over one events table.
+ */
+export function traceSessionId(): ChExpression<string> {
+ const sessionId = new ExpressionWrapper<
+ ClickHouseDatabase,
+ keyof ClickHouseDatabase,
+ string
+ >(columnRef("session_id"));
+ return argMaxIf(
+ sessionId,
+ new ExpressionWrapper(columnRef("event_ts")),
+ nonEmpty(sessionId),
+ );
+}
+
/**
* Lower `metadata[key]` to a traced array-subscript + `indexOf` node.
* The key is a bound `ValueNode`, not a SQL literal.
diff --git a/packages/shared/src/server/query-ast/tenancy.ts b/packages/shared/src/server/query-ast/tenancy.ts
index f393edd0ed..273a8aead7 100644
--- a/packages/shared/src/server/query-ast/tenancy.ts
+++ b/packages/shared/src/server/query-ast/tenancy.ts
@@ -370,3 +370,29 @@ function rejectUnscopedRawSql(node: unknown): void {
rejectUnscopedRawSql(value);
}
}
+
+/**
+ * Rejects a standalone expression that reads a relation: a subquery, or a raw
+ * fragment that introduces one. Outside a query there is no tenancy scope to
+ * inject into it.
+ */
+export function assertRelationFree(node: OperationNode): void {
+ rejectUnscopedRawSql(node);
+ rejectSubqueries(node);
+}
+
+function rejectSubqueries(node: unknown): void {
+ if (!node || typeof node !== "object") return;
+ if (Array.isArray(node)) {
+ for (const item of node) rejectSubqueries(item);
+ return;
+ }
+ if ("kind" in node && node.kind === "SelectQueryNode") {
+ throw new UnscopedRelationError(
+ "A standalone expression cannot contain a subquery: it has no tenancy scope. Compile the query through compileClickhouseQuery() instead.",
+ );
+ }
+ for (const value of Object.values(node)) {
+ rejectSubqueries(value);
+ }
+}
diff --git a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap
index 3eba719374..c845dc649d 100644
--- a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap
+++ b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap
@@ -495,15 +495,235 @@ FORMAT RawBLOB",
]
`;
-exports[`golden: events point-reads & existence family > getAgentGraphDataFromEventsTable 1`] = `
+exports[`golden: events metadata map and session resolution > getEventsForAnalyticsIntegrations 1`] = `
[
{
- "fn": "queryClickhouse",
+ "fn": "queryClickhouseStream",
"params": {
"param1": "golden-project",
- "param2": "golden-trace",
+ "param2": "2025-12-01 00:00:00.000",
"param3": "2026-01-01 00:00:00.000",
- "param4": "2026-01-01 01:00:00.000",
+ },
+ "route": "analytics_integration",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.environment AS environment,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.name AS name,
+ e.level AS level,
+ e.status_message AS status_message,
+ e.version AS version,
+ e.bookmarked AS bookmarked,
+ e.public AS public,
+ e.user_id AS user_id,
+ e.session_id AS session_id,
+ ifNull(
+ COALESCE(
+ nullIf(e.trace_name, ''),
+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)
+ ),
+ ''
+ ) AS trace_name,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.completion_start_time AS completion_start_time,
+ e.created_at AS created_at,
+ e.updated_at AS updated_at,
+ e.provided_model_name AS provided_model_name,
+ e.model_parameters AS model_parameters,
+ e.usage_details AS usage_details,
+ e.cost_details AS cost_details,
+ e.total_cost AS total_cost,
+ e.prompt_id AS prompt_id,
+ e.prompt_name AS prompt_name,
+ e.prompt_version AS prompt_version,
+ e.tool_definitions AS tool_definitions,
+ e.tool_calls AS tool_calls,
+ e.tool_call_names AS tool_call_names,
+ e.usage_pricing_tier_id AS usage_pricing_tier_id,
+ e.usage_pricing_tier_name AS usage_pricing_tier_name,
+ e.tags AS tags,
+ e.release AS release,
+ e.model_id AS model_id,
+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time)) AS latency,
+ if(
+ isNull(e.completion_start_time),
+ NULL,
+ dateDiff('millisecond', e.start_time, e.completion_start_time)
+ ) AS time_to_first_token,
+ e.usage_details['input'] AS input_tokens,
+ e.usage_details['output'] AS output_tokens,
+ e.usage_details['total'] AS total_tokens,
+ mapFromArrays(
+ arrayReverse(e.metadata_names),
+ arrayReverse(e.metadata_values)
+ )['$posthog_session_id'] AS posthog_session_id,
+ mapFromArrays(
+ arrayReverse(e.metadata_names),
+ arrayReverse(e.metadata_values)
+ )['$mixpanel_session_id'] AS mixpanel_session_id
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (e.start_time >= {param2:DateTime64(3)}) AND (e.start_time < {param3:DateTime64(3)}) AND (e.is_deleted = 0)
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events metadata map and session resolution > getObservationsV2FromEventsTableForPublicApi fields=core+metadata expandMetadataKeys=set 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": 51,
+ },
+ "route": "unknown",
+ "sql": "WITH
+ base AS
+ (
+ SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.start_time AS start_time,
+ e.end_time AS end_time
+ FROM events_core AS e
+ WHERE e.project_id = {param1:String}
+ ORDER BY
+ e.project_id DESC,
+ toStartOfMinute(e.start_time) DESC,
+ e.start_time DESC,
+ xxHash32(e.trace_id) DESC,
+ e.span_id DESC
+ LIMIT {param2:Int32}
+ ),
+ io AS
+ (
+ SELECT
+ e.span_id AS _io_id,
+ e.trace_id AS _io_trace_id,
+ e.start_time AS _io_start_time,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata
+ FROM events_full AS e
+ WHERE (e.project_id = {param1:String}) AND (e.start_time >= (((
+ SELECT (min(start_time), max(start_time))
+ FROM base
+ )).1)) AND (e.start_time <= (((
+ SELECT (min(start_time), max(start_time))
+ FROM base
+ )).2)) AND ((e.start_time, e.trace_id, e.span_id) IN (
+ SELECT
+ start_time,
+ trace_id,
+ id
+ FROM base
+ ))
+ )
+SELECT
+ b.id AS id,
+ b.trace_id AS trace_id,
+ b.start_time AS start_time,
+ b.end_time AS end_time,
+ b.project_id AS project_id,
+ b.parent_observation_id AS parent_observation_id,
+ b.type AS type,
+ i.metadata AS metadata
+FROM base AS b
+ANY LEFT JOIN io AS i ON (b.start_time = i._io_start_time) AND (b.trace_id = i._io_trace_id) AND (b.id = i._io_id)
+ORDER BY
+ b.start_time DESC,
+ xxHash32(b.trace_id) DESC,
+ b.id DESC",
+ },
+]
+`;
+
+exports[`golden: events metadata map and session resolution > getTraceBatchEventStream 1`] = `
+[
+ {
+ "fn": "queryClickhouseStream",
+ "params": {
+ "param1": [
+ "golden-project",
+ ],
+ "param2": [
+ "golden-trace",
+ ],
+ "param3": [
+ TupleParam {
+ "values": [
+ "golden-project",
+ "golden-trace",
+ ],
+ },
+ ],
+ "param4": 1764547080000,
+ "param5": 1767225720000,
+ "param6": [
+ "golden-project",
+ ],
+ "param7": [
+ [
+ "golden-trace",
+ ],
+ ],
+ "param8": [
+ 1764547080000,
+ ],
+ "param9": [
+ 1767225720000,
+ ],
+ },
+ "route": "unknown",
+ "sql": "SELECT
+ e.tool_definitions AS tool_definitions,
+ e.tool_calls AS tool_calls,
+ e.tool_call_names AS tool_call_names,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,
+ input,
+ output,
+ e.project_id,
+ e.trace_id,
+ e.environment,
+ e.span_id,
+ e.parent_span_id,
+ e.start_time,
+ e.event_ts,
+ e.type,
+ e.name,
+ e.level,
+ e.status_message
+FROM events_full AS e
+WHERE (e.project_id IN ({param1:Array(String)})) AND (e.trace_id IN ({param2:Array(String)})) AND ((e.project_id, e.trace_id) IN ({param3:Array(Tuple(String, String))})) AND (xxHash32(e.trace_id) IN (
+ SELECT arrayJoin(arrayMap(id -> xxHash32(id), {param2:Array(String)}))
+)) AND (
+ e.start_time >= fromUnixTimestamp64Milli({param4:Int64})
+) AND (e.start_time <= fromUnixTimestamp64Milli({param5:Int64})) AND arrayExists((group_project_id, group_trace_ids, min_ms, max_ms) -> ((e.project_id = group_project_id) AND has(group_trace_ids, e.trace_id) AND ((e.start_time >= fromUnixTimestamp64Milli(min_ms)) AND (e.start_time <= fromUnixTimestamp64Milli(max_ms)))), {param6:Array(String)}, {param7:Array(Array(String))}, {param8:Array(Int64)}, {param9:Array(Int64)})
+ORDER BY
+ e.project_id ASC,
+ e.trace_id ASC",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > getAgentGraphDataFromEventsTable 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "langgraph_node",
+ "param2": "langgraph_step",
+ "param3": "golden-project",
+ "param4": "golden-trace",
+ "param5": "2026-01-01 00:00:00.000",
+ "param6": "2026-01-01 01:00:00.000",
},
"route": "unknown",
"sql": "SELECT
@@ -513,10 +733,10 @@ exports[`golden: events point-reads & existence family > getAgentGraphDataFromEv
e.name AS name,
e.start_time AS start_time,
e.end_time AS end_time,
- mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['langgraph_node'] AS node,
- mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['langgraph_step'] AS step
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))[{param1:String}] AS node,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))[{param2:String}] AS step
FROM events_core AS e
-WHERE (e.project_id = {param1:String}) AND (e.trace_id = {param2:String}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time <= {param4:DateTime64(6)})",
+WHERE (e.project_id = {param3:String}) AND (e.trace_id = {param4:String}) AND (e.start_time >= {param5:DateTime64(6)}) AND (e.start_time <= {param6:DateTime64(6)})",
},
]
`;
diff --git a/packages/shared/src/server/repositories/events.golden.test.ts b/packages/shared/src/server/repositories/events.golden.test.ts
index a48fccad68..5c79475f3e 100644
--- a/packages/shared/src/server/repositories/events.golden.test.ts
+++ b/packages/shared/src/server/repositories/events.golden.test.ts
@@ -28,6 +28,7 @@ vi.mock("./clickhouse", async (importOriginal) => {
import {
getAgentGraphDataFromEventsTable,
+ getEventsForAnalyticsIntegrations,
getLastTraceTimestampsByProjectsFromEventsTable,
getLatestSdkVersionInfoFromEvents,
getObservationByIdFromEventsTable,
@@ -36,6 +37,7 @@ import {
getObservationIOFieldByteLengthFromEventsTable,
getObservationsBatchIOFromEventsTable,
getObservationsTraceIdsFromEventsTable,
+ getObservationsV2FromEventsTableForPublicApi,
getObservationsWithPromptNameFromEvents,
getSessionMetricsFromEvents,
getTraceByIdFromEventsTable,
@@ -45,6 +47,7 @@ import {
hasAnyUserFromEventsTable,
streamObservationIOFieldFromEventsTable,
} from "./events";
+import { getTraceBatchEventStream } from "./trace-batch";
const FIXED_PROJECT_ID = "golden-project";
const FIXED_PROJECT_IDS = ["golden-project", "golden-project-b"] as const;
@@ -458,3 +461,51 @@ describeWithClickhouse("golden: events IO payload reads family", () => {
expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
});
});
+
+describeWithClickhouse(
+ "golden: events metadata map and session resolution",
+ () => {
+ beforeEach(() => {
+ resetCaptures();
+ });
+
+ it("getObservationsV2FromEventsTableForPublicApi fields=core+metadata expandMetadataKeys=set", async () => {
+ await getObservationsV2FromEventsTableForPublicApi({
+ projectId: FIXED_PROJECT_ID,
+ page: 1,
+ limit: 50,
+ fields: ["core", "metadata"],
+ expandMetadataKeys: ["golden-key"],
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getEventsForAnalyticsIntegrations", async () => {
+ for await (const _ of getEventsForAnalyticsIntegrations(
+ FIXED_PROJECT_ID,
+ "golden-project-name",
+ FIXED_FROM_TIMESTAMP,
+ FIXED_START_TIME,
+ )) {
+ // The harness streams no rows.
+ }
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getTraceBatchEventStream", async () => {
+ for await (const _ of getTraceBatchEventStream({
+ traces: [
+ {
+ projectId: FIXED_PROJECT_ID,
+ traceId: FIXED_TRACE_ID,
+ minStart: FIXED_FROM_TIMESTAMP.getTime(),
+ maxStart: FIXED_START_TIME.getTime(),
+ },
+ ],
+ })) {
+ // The harness streams no rows.
+ }
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+ },
+);
diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.ts
index eb574d06fe..d9cabf9dbb 100644
--- a/packages/shared/src/server/repositories/events.ts
+++ b/packages/shared/src/server/repositories/events.ts
@@ -36,7 +36,6 @@ import {
import { getClickhouseKysely } from "../query-ast/dialect";
import { promptGenerationEvents } from "./prompt-events";
import {
- argMaxIf,
arrayMap,
byteLength,
count,
@@ -46,9 +45,9 @@ import {
mapValue,
medianExact,
metadataMap,
- nonEmpty,
sumMapValuesWhereKeyContains,
toJSONString,
+ traceSessionId,
tupleIn,
uniq,
xxHash32,
@@ -148,6 +147,7 @@ import {
type SessionEventsFields,
type SessionEventsMetricsRow,
OrderByEntry,
+ EVENTS_METADATA_MAP_SQL,
} from "../queries/clickhouse-sql/event-query-builder";
import {
buildEventsFilterOptionColumnQuery,
@@ -1055,9 +1055,6 @@ export const getObservationByIdFromEventsTable = async ({
return mapped.shift();
};
-const EVENTS_METADATA_MAP_SQL =
- "mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))";
-
/** ClickHouse DateTime binds stay strings; the column type is Date. */
function dateTimeParam(value: string): Date {
return value as unknown as Date;
@@ -2653,16 +2650,22 @@ export async function getAgentGraphDataFromEventsTable(params: {
const builder = db
.selectFrom("events_core as e")
- .select([
- "e.span_id as id",
- "e.parent_span_id as parent_observation_id",
- "e.type as type",
- "e.name as name",
- "e.start_time as start_time",
- "e.end_time as end_time",
- sql.raw(`${EVENTS_METADATA_MAP_SQL}['langgraph_node']`).as("node"),
- sql.raw(`${EVENTS_METADATA_MAP_SQL}['langgraph_step']`).as("step"),
- ])
+ .select((eb) => {
+ const metadata = metadataMap(
+ eb.ref("e.metadata_names"),
+ eb.ref("e.metadata_values"),
+ );
+ return [
+ "e.span_id as id",
+ "e.parent_span_id as parent_observation_id",
+ "e.type as type",
+ "e.name as name",
+ "e.start_time as start_time",
+ "e.end_time as end_time",
+ mapValue(metadata, "langgraph_node").as("node"),
+ mapValue(metadata, "langgraph_step").as("step"),
+ ];
+ })
.where("e.trace_id", "=", traceId)
.where("e.start_time", ">=", dateTimeParam(chMinStartTime))
.where("e.start_time", "<=", dateTimeParam(chMaxStartTime));
@@ -2855,16 +2858,7 @@ export const getObservationsBatchIOFromEventsTable = async <
.select("trace_id")
.where("trace_id", "in", traceIds)
.groupBy("trace_id")
- .having(
- (eb) =>
- argMaxIf(
- eb.ref("session_id"),
- eb.ref("event_ts"),
- nonEmpty(eb.ref("session_id")),
- ),
- "=",
- opts.sessionId!,
- ),
+ .having(traceSessionId(), "=", opts.sessionId!),
),
)
.where("e.start_time", ">=", chDateTime(minTimestamp))
@@ -3539,8 +3533,8 @@ export const getEventsForAnalyticsIntegrations = async function* (
"e.usage_details['output'] as output_tokens",
"e.usage_details['total'] as total_tokens",
// Analytics integration session IDs from metadata (constructed from array columns)
- "mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['$posthog_session_id'] as posthog_session_id",
- "mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['$mixpanel_session_id'] as mixpanel_session_id",
+ `${EVENTS_METADATA_MAP_SQL}['$posthog_session_id'] as posthog_session_id`,
+ `${EVENTS_METADATA_MAP_SQL}['$mixpanel_session_id'] as mixpanel_session_id`,
)
.whereRaw(
"e.start_time >= {minTimestamp: DateTime64(3)} AND e.start_time < {maxTimestamp: DateTime64(3)}",