🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(query-ast): compile events export streams through the AST (#18867)
Niklas Semmler1744b5d7e864
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.tsindex fe6432d780..b4aecdbac8 100644--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts@@ -1,18 +1,27 @@-import {- OBSERVATION_FIELD_GROUPS_PUBLIC_API,- type ObservationFieldGroupPublicApi,-} from "../../../domain/observation-field-groups";-import {- eventsTableIsRootObservationSql,- eventsTableTraceNameAggregationSql,- eventsTableTraceNameSelectSql,-} from "../../../eventsTable";+import { eventsTableTraceNameAggregationSql } from "../../../eventsTable";import { expressionBuilder } from "kysely";import { renderClickhouseExpression } from "../../query-ast/compile";-import { metadataMap, traceSessionId } from "../../query-ast/extensions";+import {+ metadataMap,+ toBool,+ traceSessionId,+} from "../../query-ast/extensions";import type { ClickHouseDatabase } from "../../query-ast/schema";import { OBSERVATIONS_TO_TRACE_INTERVAL } from "../../repositories/constants";+import {+ FIELD_SETS,+ orderedFieldKeys,+ type EventsFieldKey,+ type FieldSetName,+} from "../../repositories/events-field-sets";+import {+ eventsTraceNameOrEmpty,+ eventsIsRootObservation,+ millisecondsSinceStart,+ type SessionEventsField,+ type SessionEventsMetricsRow,+} from "../../repositories/events-selects";import {FilterList,StringFilter,@@ -29,8 +38,7 @@ const eventsMetadataMap = metadataMap(eventsRef.ref("e.metadata_values"),);-export const EVENTS_METADATA_MAP_SQL =- renderClickhouseExpression(eventsMetadataMap);+const EVENTS_METADATA_MAP_SQL = renderClickhouseExpression(eventsMetadataMap);const TRACE_SESSION_ID_SQL = renderClickhouseExpression(traceSessionId());@@ -146,7 +154,7 @@ const EVENTS_FIELDS = {environment: "e.environment as environment",type: "e.type as type",parentObservationId: "e.parent_span_id as parent_observation_id",- isRootObservation: `toBool(${eventsTableIsRootObservationSql}) as "is_root_observation"`,+ isRootObservation: `${renderClickhouseExpression(toBool(eventsIsRootObservation))} as "is_root_observation"`,isAppRoot: "e.is_app_root as is_app_root",name: "e.name as name",level: "e.level as level",@@ -156,8 +164,7 @@ const EVENTS_FIELDS = {public: "e.public as public",userId: "e.user_id as user_id",sessionId: "e.session_id as session_id",- // String-typed on purpose; see eventsTableTraceNameSelectSql (LFE-14924).- traceName: `${eventsTableTraceNameSelectSql} as "trace_name"`,+ traceName: `${renderClickhouseExpression(eventsTraceNameOrEmpty)} as "trace_name"`,// Time fieldsstartTime: "e.start_time as start_time",@@ -227,285 +234,9 @@ const EVENTS_FIELDS = {experimentItemVersion: "e.experiment_item_version as experiment_item_version",// Calculated fields- latency:- "if(isNull(e.end_time), NULL, date_diff('millisecond', e.start_time, e.end_time)) as latency",- timeToFirstToken:- "if(isNull(e.completion_start_time), NULL, date_diff('millisecond', e.start_time, e.completion_start_time)) as \"time_to_first_token\"",-} as const;--const EVENTS_FIELD_ORDER_INDEX = new Map(- Object.keys(EVENTS_FIELDS).map((key, index) => [key, index]),-);--/**- * Predefined field sets for common query patterns- * Maps set names to arrays of field keys from EVENTS_FIELDS- */-const FIELD_SETS = {- // Aggregates- count: ["count"],- countWithUniqueTraces: ["count", "uniqueTraceCount"],-- // List query field sets (for getObservationsWithModelDataFromEventsTable)- base: [- "id",- "type",- "projectId",- "name",- "modelParameters",- "startTime",- "endTime",- "traceId",- "completionStartTime",- "providedUsageDetails",- "usageDetails",- "providedCostDetails",- "costDetails",- "level",- "environment",- "bookmarked",- "public",- "statusMessage",- "version",- "parentObservationId",- "createdAt",- "updatedAt",- "providedModelName",- "totalCost",- "usagePricingTierId",- "usagePricingTierName",- "promptId",- "promptName",- "promptVersion",- "internalModelId",- "userId",- "sessionId",- "traceName",- "release",- "tags",- "toolDefinitions",- "toolCalls",- "toolCallNames",- ],- baseWithoutTools: [- "id",- "type",- "projectId",- "name",- "modelParameters",- "startTime",- "endTime",- "traceId",- "completionStartTime",- "providedUsageDetails",- "usageDetails",- "providedCostDetails",- "costDetails",- "level",- "environment",- "bookmarked",- "public",- "statusMessage",- "version",- "parentObservationId",- "createdAt",- "updatedAt",- "providedModelName",- "totalCost",- "usagePricingTierId",- "usagePricingTierName",- "promptId",- "promptName",- "promptVersion",- "internalModelId",- "userId",- "sessionId",- "traceName",- "release",- "tags",- // Keep lightweight tool names available even when heavy tool payloads are omitted.- "toolCallNames",- ],- calculated: ["latency", "timeToFirstToken"],- io: ["input", "output"],- metadata: ["metadata"],- tools: ["toolDefinitions", "toolCalls", "toolCallNames"],- trace_context: ["tags", "release", "traceName"],- model_export: ["providedModelName", "modelId", "modelParameters"],- eventTs: ["eventTs"],- publicApiExperimentItemCore: [- "id",- "projectId",- "traceId",- "startTime",- "endTime",- "level",- "environment",- "experimentId",- "experimentName",- "experimentItemId",- ],- publicApiExperimentItemDataset: [- "experimentDatasetId",- "experimentItemVersion",- ],- publicApiExperimentItemExpectedOutput: ["experimentItemExpectedOutput"],- publicApiExperimentItemMetadataFields: ["experimentItemMetadata"],- publicApiExperimentItemExperimentMetadata: [- "experimentMetadata",- "experimentDescription",- ],- // Public API v2 field sets (field groups for selective fetching)- core: [- "id",- "traceId",- "startTime",- "endTime",- "projectId",- "parentObservationId",- "type",- ],- basic: [- "name",- "level",- "statusMessage",- "version",- "environment",- "bookmarked",- "public",- "userId",- "sessionId",- "isRootObservation",- ],- time: ["completionStartTime", "createdAt", "updatedAt"],- model: ["providedModelName", "internalModelId", "modelParameters"],- usage: [- "usageDetails",- "costDetails",- "totalCost",- "usagePricingTierId",- "usagePricingTierName",- ],- prompt: ["promptId", "promptName", "promptVersion"],- metrics: ["latency", "timeToFirstToken"],-- // Batch export field set (all fields needed for CSV/JSON exports)- export: [- "id",- "traceId",- "projectId",- "startTime",- "endTime",- "name",- "type",- "environment",- "version",- "userId",- "sessionId",- "level",- "statusMessage",- "promptName",- "promptId",- "promptVersion",- "modelId",- "providedModelName",- "modelParameters",- "usageDetails",- "costDetails",- "totalCost",- "completionStartTime",- "latency",- "timeToFirstToken",- "tags",- "release",- "traceName",- "parentObservationId",- "bookmarked",- "public",- "createdAt",- "updatedAt",- "toolDefinitions",- "toolCalls",- "toolCallNames",- "usagePricingTierId",- "usagePricingTierName",- ],-- eval: [- "id",- "traceId",- "projectId",- "parentObservationId",- "isAppRoot",- "type",- "name",- "environment",- "version",- "level",- "statusMessage",- "traceName",- "userId",- "sessionId",- "tags",- "release",- "providedModelName",- "modelParameters",- "promptId",- "promptName",- "promptVersion",- "providedUsageDetails",- "usageDetails",- "providedCostDetails",- "costDetails",- "toolDefinitions",- "toolCalls",- "toolCallNames",- "experimentId",- "experimentName",- "experimentItemRootSpanId",- "experimentItemExpectedOutput",- "experimentItemMetadata",- ],-- // Experiment items field set- experimentItems: [- "id", // span_id (observation_id)- "traceId",- "input",- "output",- "startTime",- "level",-- // Experiment metadata- "experimentId",- "experimentName",- "experimentDatasetId",-- // Experiment item metadata- "experimentItemId",- "experimentItemRootSpanId",- "experimentItemVersion",- "experimentItemExpectedOutput",- "experimentItemMetadata",-- // Event metadata- "metadata",- ],-} as const;--export type FieldSetName = keyof typeof FIELD_SETS;--// Canonicalize by EVENTS_FIELDS declaration order so SELECT column order does-// not follow caller field-set order. Clustered ClickHouse reads align result-// blocks by position; incompatible Map types (cost_details vs metadata) 500-// when those columns swap places.-function orderedFieldKeys<K extends string>(fieldKeys: readonly K[]): K[] {- return [...new Set(fieldKeys)].sort(- (a, b) =>- (EVENTS_FIELD_ORDER_INDEX.get(a) ?? Number.MAX_SAFE_INTEGER) -- (EVENTS_FIELD_ORDER_INDEX.get(b) ?? Number.MAX_SAFE_INTEGER),- );-}+ latency: `${renderClickhouseExpression(millisecondsSinceStart("e.end_time"))} as latency`,+ timeToFirstToken: `${renderClickhouseExpression(millisecondsSinceStart("e.completion_start_time"))} as "time_to_first_token"`,+} as const satisfies Record<EventsFieldKey, string>;function eventsFieldSelectExpressions(fieldKeys: readonly string[]): string[] {return orderedFieldKeys(fieldKeys).flatMap((fieldKey) => {@@ -514,15 +245,6 @@ function eventsFieldSelectExpressions(fieldKeys: readonly string[]): string[] {});}-export const OBSERVATION_FIELD_GROUP_FIELD_NAMES = Object.fromEntries(- OBSERVATION_FIELD_GROUPS_PUBLIC_API.map((group) => [- group,- FIELD_SETS[group],- ]),-) as {- [K in ObservationFieldGroupPublicApi]: (typeof FIELD_SETS)[K];-};-/*** Aggregation fields for trace-level queries* These fields use ClickHouse aggregation functions and require GROUP BY@@ -1436,33 +1158,6 @@ export class EventsAggregationQueryBuilder extends BaseEventsQueryBuilder<}}-/**- * ClickHouse row type for session-level metrics queries from the events table.- * Matches the columns produced by EVENTS_SESSION_AGGREGATION_FIELDS below.- */-export type SessionEventsMetricsRow = {- session_id: string;- max_timestamp: string;- min_timestamp: string;- trace_ids: string[];- user_ids: string[];- trace_count: number;- trace_tags: string[];- environment?: string;- total_observations: number;- duration: number;- session_usage_details: Record<string, number>;- session_cost_details: Record<string, number>;- session_input_cost: string;- session_output_cost: string;- session_total_cost: string;- session_input_usage: string;- session_output_usage: string;- session_total_usage: string;-};--export type SessionEventsField = keyof SessionEventsMetricsRow;-/*** ClickHouse returns the UInt64 counts as strings.*/diff --git a/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts b/packages/shared/src/server/queries/clickhouse-sql/query-fragments.tsindex 895d075601..d9584b829a 100644--- a/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts@@ -7,12 +7,12 @@ import {EventsAggQueryBuilder,EventsQueryBuilder,EventsSessionAggregationQueryBuilder,- type SessionEventsField,type SessionEventsFields,ExperimentsAggregationFieldSetName,ExperimentsAggregationQueryBuilder,type CTEWithSchema,} from "./event-query-builder";+import type { SessionEventsField } from "../../repositories/events-selects";import { AGGREGATABLE_SCORE_TYPES } from "../../../domain/scores";import {OBSERVATIONS_TO_TRACE_INTERVAL,diff --git a/packages/shared/src/server/queries/index.ts b/packages/shared/src/server/queries/index.tsindex 32a994364e..6e452fc7b4 100644--- a/packages/shared/src/server/queries/index.ts+++ b/packages/shared/src/server/queries/index.ts@@ -65,18 +65,20 @@ export {EventsSessionAggregationQueryBuilder,EventsQueryBuilder,ExperimentsAggregationQueryBuilder,- OBSERVATION_FIELD_GROUP_FIELD_NAMES,buildEventsFullTableSplitQuery,normalizeSessionCounts,type CTESchema,type CTEWithSchema,type QueryWithParams,type ExperimentsAggregationFieldSetName,- type SessionEventsField,type SessionEventsFields,- type SessionEventsMetricsRow,type SplitQueryBuilder,} from "./clickhouse-sql/event-query-builder";+export { OBSERVATION_FIELD_GROUP_FIELD_NAMES } from "../repositories/events-field-sets";+export type {+ SessionEventsField,+ SessionEventsMetricsRow,+} from "../repositories/events-selects";export {buildEventsFilterOptionColumnQuery,buildEventsFilterOptionsForColumnsQuery,diff --git a/packages/shared/src/server/query-ast/README.md b/packages/shared/src/server/query-ast/README.mdindex f61fef3535..7898c8521d 100644--- a/packages/shared/src/server/query-ast/README.md+++ b/packages/shared/src/server/query-ast/README.md@@ -152,13 +152,14 @@ db.selectFrom("scores as s")// … from scores as s final```-**Select a metadata value** (`metadataValue` — lowers `metadata[key]` to a bound subscript of `metadataMap`, so the last duplicate key wins):+**Select a metadata value** (`metadataValue` — lowers `metadata[key]` to a subscript of `metadataMap`, so the last duplicate key wins; like `mapValue`, a string key is bound and a `literal(...)` key is inlined):```ts+const filterKey: string = input.metadataKey; // caller-supplied, so bounddb.selectFrom("events_core as e")- .select((eb) => [metadataValue("e", "my_key").as("my_val")])- .where((eb) => eb(metadataValue("e", "my_key"), "=", "x"));-// select mapfromarrays(arrayreverse(e.metadata_names), arrayreverse(e.metadata_values))[{p:String}] as my_val …+ .select((eb) => [metadataValue("e", literal("my_key")).as("my_val")])+ .where((eb) => eb(metadataValue("e", filterKey), "=", "x"));+// select mapfromarrays(arrayreverse(e.metadata_names), arrayreverse(e.metadata_values))['my_key'] as my_val …```## Developing the dialect@@ -178,7 +179,7 @@ preserve it:| ARRAY JOIN | Plugin attaches `ArrayJoinNode` as an extra field on `SelectQueryNode`. `ClickHouseOperationNodeTransformer` preserves it. `ClickHouseQueryCompiler.visitSelectQuery` emits it after JOINs / before WHERE. | no || LIMIT BY | Plugin attaches `LimitByNode` the same way. Compiler emits it after ORDER BY / before LIMIT. | no || FINAL | Plugin wraps a FROM/JOIN table expression in `FinalTableNode`. Compiler emits `final` after that expression (`scores as s final`). `$call(useFinal(["scores"]))`. Tables declared `dedup: none` throw. | no |-| metadata subscript | Helper builds an `ArrayIndexNode` over the `metadataMap` `FunctionNode`, indexed by a bound `ValueNode` key. Transformer + compiler special-case the node. No plugin. | no |+| metadata subscript | Helper builds an `ArrayIndexNode` over the `metadataMap` `FunctionNode`, indexed by a string key expression (a bound value or a `literal(...)`). Transformer + compiler special-case the node. No plugin. | no || Virtual view | Plugin rewrites `selectFrom(viewName)` into a WITH CTE. Outer types only expose the view's selected columns. | no |These are real node objects whose children are traced Kyselydiff --git a/packages/shared/src/server/query-ast/extensions.ts b/packages/shared/src/server/query-ast/extensions.tsindex 917fdc87cc..1a12ae18fa 100644--- a/packages/shared/src/server/query-ast/extensions.ts+++ b/packages/shared/src/server/query-ast/extensions.ts@@ -221,15 +221,18 @@ export function dateDiff();}-/** `map[key]` subscript with a bound key. */+/**+ * `map[key]` subscript. A string key is bound; pass `literal(...)` for a code+ * constant so it is inlined.+ */export function mapValue<V = number>(map: Expression<Record<string, V>>,- key: string,+ key: string | Expression<string>,): ChExpression<V> {return new ExpressionWrapper<ClickHouseDatabase, keyof ClickHouseDatabase, V>(ArrayIndexNode.create(map.toOperationNode(),- ValueNode.create(key),+ typeof key === "string" ? ValueNode.create(key) : key.toOperationNode(),) as unknown as OperationNode,);}@@ -349,6 +352,11 @@ export function ifNull<T>(return functionCall("ifNull", [value, fallback]);}+/** ClickHouse `toBool(value)`. */+export function toBool(value: Expression<SqlBool>): ChExpression<boolean> {+ return functionCall("toBool", [value]);+}+/** ClickHouse `isNull(value)`. */export function isNull(value: Expression<unknown>): ChExpression<SqlBool> {return functionCall("isNull", [value]);@@ -629,12 +637,12 @@ export function traceSessionId(): ChExpression<string> {/*** Lower `metadata[key]` to a subscript of {@link metadataMap}, so the last- * entry of a duplicated key wins. The key is a bound `ValueNode`, not a SQL- * literal.+ * entry of a duplicated key wins. Keys follow {@link mapValue}: a string is+ * bound, a `literal(...)` is inlined.*/export function metadataValue(tableAlias: string,- key: string,+ key: string | Expression<string>,): ChExpression<string> {const column = (name: string) =>new ExpressionWrapper<ClickHouseDatabase, "events_core", string[]>(diff --git a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snapindex 1393724258..7cb7132074 100644--- a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap+++ b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap@@ -497,14 +497,900 @@ FORMAT RawBLOB",]`;+exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=basic 1`] = `+[+ {+ "fn": "queryClickhouseStream",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.environment AS environment,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,+ e.name AS name,+ e.level AS level,+ e.status_message AS status_message,+ e.version AS version,+ e.bookmarked AS bookmarked,+ e.public AS public,+ e.user_id AS user_id,+ e.session_id AS session_id,+ e.start_time AS start_time,+ e.end_time AS end_time+FROM events_core AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=core 1`] = `+[+ {+ "fn": "queryClickhouseStream",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ e.start_time AS start_time,+ e.end_time AS end_time+FROM events_core AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=default 1`] = `+[+ {+ "fn": "queryClickhouseStream",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.environment AS environment,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,+ e.name AS name,+ e.level AS level,+ e.status_message AS status_message,+ e.version AS version,+ e.bookmarked AS bookmarked,+ e.public AS public,+ e.user_id AS user_id,+ e.session_id AS session_id,+ ifNull(+ coalesce(+ nullIf(e.trace_name, ''),+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)+ ),+ ''+ ) AS trace_name,+ e.start_time AS start_time,+ e.end_time AS end_time,+ e.completion_start_time AS completion_start_time,+ e.created_at AS created_at,+ e.updated_at AS updated_at,+ e.provided_model_name AS provided_model_name,+ e.model_parameters AS model_parameters,+ e.usage_details AS usage_details,+ e.cost_details AS cost_details,+ e.total_cost AS total_cost,+ e.prompt_id AS prompt_id,+ e.prompt_name AS prompt_name,+ e.prompt_version AS prompt_version,+ e.tool_definitions AS tool_definitions,+ e.tool_calls AS tool_calls,+ e.tool_call_names AS tool_call_names,+ e.usage_pricing_tier_id AS usage_pricing_tier_id,+ e.usage_pricing_tier_name AS usage_pricing_tier_name,+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,+ e.tags AS tags,+ e.release AS release,+ e.model_id AS model_id,+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time)) AS latency,+ if(+ isNull(e.completion_start_time),+ NULL,+ dateDiff('millisecond', e.start_time, e.completion_start_time)+ ) AS time_to_first_token,+ input,+ output+FROM events_full AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=empty 1`] = `+[+ {+ "fn": "queryClickhouseStream",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ e.start_time AS start_time,+ e.end_time AS end_time+FROM events_core AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=io 1`] = `+[+ {+ "fn": "queryClickhouseStream",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ e.start_time AS start_time,+ e.end_time AS end_time,+ input,+ output+FROM events_full AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=metadata 1`] = `+[+ {+ "fn": "queryClickhouseStream",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ e.start_time AS start_time,+ e.end_time AS end_time,+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata+FROM events_core AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=metrics 1`] = `+[+ {+ "fn": "queryClickhouseStream",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ e.start_time AS start_time,+ e.end_time AS end_time,+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time)) AS latency,+ if(+ isNull(e.completion_start_time),+ NULL,+ dateDiff('millisecond', e.start_time, e.completion_start_time)+ ) AS time_to_first_token+FROM events_core AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=model 1`] = `+[+ {+ "fn": "queryClickhouseStream",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ e.start_time AS start_time,+ e.end_time AS end_time,+ e.provided_model_name AS provided_model_name,+ e.model_parameters AS model_parameters,+ e.model_id AS model_id+FROM events_core AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=prompt 1`] = `+[+ {+ "fn": "queryClickhouseStream",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ e.start_time AS start_time,+ e.end_time AS end_time,+ e.prompt_id AS prompt_id,+ e.prompt_name AS prompt_name,+ e.prompt_version AS prompt_version+FROM events_core AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=reversed 1`] = `+[+ {+ "fn": "queryClickhouseStream",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.environment AS environment,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,+ e.name AS name,+ e.level AS level,+ e.status_message AS status_message,+ e.version AS version,+ e.bookmarked AS bookmarked,+ e.public AS public,+ e.user_id AS user_id,+ e.session_id AS session_id,+ ifNull(+ coalesce(+ nullIf(e.trace_name, ''),+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)+ ),+ ''+ ) AS trace_name,+ e.start_time AS start_time,+ e.end_time AS end_time,+ e.completion_start_time AS completion_start_time,+ e.created_at AS created_at,+ e.updated_at AS updated_at,+ e.provided_model_name AS provided_model_name,+ e.model_parameters AS model_parameters,+ e.usage_details AS usage_details,+ e.cost_details AS cost_details,+ e.total_cost AS total_cost,+ e.prompt_id AS prompt_id,+ e.prompt_name AS prompt_name,+ e.prompt_version AS prompt_version,+ e.tool_definitions AS tool_definitions,+ e.tool_calls AS tool_calls,+ e.tool_call_names AS tool_call_names,+ e.usage_pricing_tier_id AS usage_pricing_tier_id,+ e.usage_pricing_tier_name AS usage_pricing_tier_name,+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,+ e.tags AS tags,+ e.release AS release,+ e.model_id AS model_id,+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time)) AS latency,+ if(+ isNull(e.completion_start_time),+ NULL,+ dateDiff('millisecond', e.start_time, e.completion_start_time)+ ) AS time_to_first_token,+ input,+ output+FROM events_full AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=time 1`] = `+[+ {+ "fn": "queryClickhouseStream",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ e.start_time AS start_time,+ e.end_time AS end_time,+ e.completion_start_time AS completion_start_time,+ e.created_at AS created_at,+ e.updated_at AS updated_at+FROM events_core AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=tools 1`] = `+[+ {+ "fn": "queryClickhouseStream",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ e.start_time AS start_time,+ e.end_time AS end_time,+ e.tool_definitions AS tool_definitions,+ e.tool_calls AS tool_calls,+ e.tool_call_names AS tool_call_names+FROM events_core AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=trace_context 1`] = `+[+ {+ "fn": "queryClickhouseStream",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ ifNull(+ coalesce(+ nullIf(e.trace_name, ''),+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)+ ),+ ''+ ) AS trace_name,+ e.start_time AS start_time,+ e.end_time AS end_time,+ e.tags AS tags,+ e.release AS release+FROM events_core AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=usage 1`] = `+[+ {+ "fn": "queryClickhouseStream",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ e.start_time AS start_time,+ e.end_time AS end_time,+ e.usage_details AS usage_details,+ e.cost_details AS cost_details,+ e.total_cost AS total_cost,+ e.usage_pricing_tier_id AS usage_pricing_tier_id,+ e.usage_pricing_tier_name AS usage_pricing_tier_name+FROM events_core AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExportParquet convertLatencyToSeconds=false 1`] = `+[+ {+ "fn": "queryClickhouseExecRaw",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.environment AS environment,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,+ e.name AS name,+ e.level AS level,+ e.status_message AS status_message,+ e.version AS version,+ e.bookmarked AS bookmarked,+ e.public AS public,+ e.user_id AS user_id,+ e.session_id AS session_id,+ ifNull(+ coalesce(+ nullIf(e.trace_name, ''),+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)+ ),+ ''+ ) AS trace_name,+ e.start_time AS start_time,+ e.end_time AS end_time,+ e.completion_start_time AS completion_start_time,+ e.created_at AS created_at,+ e.updated_at AS updated_at,+ e.provided_model_name AS provided_model_name,+ e.model_parameters AS model_parameters,+ e.usage_details AS usage_details,+ e.cost_details AS cost_details,+ e.total_cost AS total_cost,+ e.prompt_id AS prompt_id,+ e.prompt_name AS prompt_name,+ e.prompt_version AS prompt_version,+ e.tool_definitions AS tool_definitions,+ e.tool_calls AS tool_calls,+ e.tool_call_names AS tool_call_names,+ e.usage_pricing_tier_id AS usage_pricing_tier_id,+ e.usage_pricing_tier_name AS usage_pricing_tier_name,+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,+ e.tags AS tags,+ e.release AS release,+ e.model_id AS model_id,+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time)) AS latency,+ if(+ isNull(e.completion_start_time),+ NULL,+ dateDiff('millisecond', e.start_time, e.completion_start_time)+ ) AS time_to_first_token,+ input,+ output+FROM events_full AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id+FORMAT Parquet",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExportParquet convertLatencyToSeconds=true 1`] = `+[+ {+ "fn": "queryClickhouseExecRaw",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.environment AS environment,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,+ e.name AS name,+ e.level AS level,+ e.status_message AS status_message,+ e.version AS version,+ e.bookmarked AS bookmarked,+ e.public AS public,+ e.user_id AS user_id,+ e.session_id AS session_id,+ ifNull(+ coalesce(+ nullIf(e.trace_name, ''),+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)+ ),+ ''+ ) AS trace_name,+ e.start_time AS start_time,+ e.end_time AS end_time,+ e.completion_start_time AS completion_start_time,+ e.created_at AS created_at,+ e.updated_at AS updated_at,+ e.provided_model_name AS provided_model_name,+ e.model_parameters AS model_parameters,+ e.usage_details AS usage_details,+ e.cost_details AS cost_details,+ e.total_cost AS total_cost,+ e.prompt_id AS prompt_id,+ e.prompt_name AS prompt_name,+ e.prompt_version AS prompt_version,+ e.tool_definitions AS tool_definitions,+ e.tool_calls AS tool_calls,+ e.tool_call_names AS tool_call_names,+ e.usage_pricing_tier_id AS usage_pricing_tier_id,+ e.usage_pricing_tier_name AS usage_pricing_tier_name,+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,+ e.tags AS tags,+ e.release AS release,+ e.model_id AS model_id,+ input,+ output,+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time) / 1000) AS latency,+ if(+ isNull(e.completion_start_time),+ NULL,+ dateDiff('millisecond', e.start_time, e.completion_start_time) / 1000+ ) AS time_to_first_token+FROM events_full AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id+FORMAT Parquet",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExportRaw fieldGroups=core+io convertLatencyToSeconds=true 1`] = `+[+ {+ "fn": "queryClickhouseStreamRawText",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ e.start_time AS start_time,+ e.end_time AS end_time,+ input,+ output+FROM events_full AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExportRaw fieldGroups=default convertLatencyToSeconds=true 1`] = `+[+ {+ "fn": "queryClickhouseStreamRawText",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.environment AS environment,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,+ e.name AS name,+ e.level AS level,+ e.status_message AS status_message,+ e.version AS version,+ e.bookmarked AS bookmarked,+ e.public AS public,+ e.user_id AS user_id,+ e.session_id AS session_id,+ ifNull(+ coalesce(+ nullIf(e.trace_name, ''),+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)+ ),+ ''+ ) AS trace_name,+ e.start_time AS start_time,+ e.end_time AS end_time,+ e.completion_start_time AS completion_start_time,+ e.created_at AS created_at,+ e.updated_at AS updated_at,+ e.provided_model_name AS provided_model_name,+ e.model_parameters AS model_parameters,+ e.usage_details AS usage_details,+ e.cost_details AS cost_details,+ e.total_cost AS total_cost,+ e.prompt_id AS prompt_id,+ e.prompt_name AS prompt_name,+ e.prompt_version AS prompt_version,+ e.tool_definitions AS tool_definitions,+ e.tool_calls AS tool_calls,+ e.tool_call_names AS tool_call_names,+ e.usage_pricing_tier_id AS usage_pricing_tier_id,+ e.usage_pricing_tier_name AS usage_pricing_tier_name,+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,+ e.tags AS tags,+ e.release AS release,+ e.model_id AS model_id,+ input,+ output,+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time) / 1000) AS latency,+ if(+ isNull(e.completion_start_time),+ NULL,+ dateDiff('millisecond', e.start_time, e.completion_start_time) / 1000+ ) AS time_to_first_token+FROM events_full AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExportRaw fieldGroups=default convertLatencyToSeconds=undefined 1`] = `+[+ {+ "fn": "queryClickhouseStreamRawText",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.environment AS environment,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,+ e.name AS name,+ e.level AS level,+ e.status_message AS status_message,+ e.version AS version,+ e.bookmarked AS bookmarked,+ e.public AS public,+ e.user_id AS user_id,+ e.session_id AS session_id,+ ifNull(+ coalesce(+ nullIf(e.trace_name, ''),+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)+ ),+ ''+ ) AS trace_name,+ e.start_time AS start_time,+ e.end_time AS end_time,+ e.completion_start_time AS completion_start_time,+ e.created_at AS created_at,+ e.updated_at AS updated_at,+ e.provided_model_name AS provided_model_name,+ e.model_parameters AS model_parameters,+ e.usage_details AS usage_details,+ e.cost_details AS cost_details,+ e.total_cost AS total_cost,+ e.prompt_id AS prompt_id,+ e.prompt_name AS prompt_name,+ e.prompt_version AS prompt_version,+ e.tool_definitions AS tool_definitions,+ e.tool_calls AS tool_calls,+ e.tool_call_names AS tool_call_names,+ e.usage_pricing_tier_id AS usage_pricing_tier_id,+ e.usage_pricing_tier_name AS usage_pricing_tier_name,+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,+ e.tags AS tags,+ e.release AS release,+ e.model_id AS model_id,+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time)) AS latency,+ if(+ isNull(e.completion_start_time),+ NULL,+ dateDiff('millisecond', e.start_time, e.completion_start_time)+ ) AS time_to_first_token,+ input,+ output+FROM events_full AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;++exports[`golden: events export streams family > getEventsForBlobStorageExportRaw fieldGroups=metrics convertLatencyToSeconds=true 1`] = `+[+ {+ "fn": "queryClickhouseStreamRawText",+ "params": {+ "param1": "golden-project",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-02 00:00:00.000",+ },+ "route": "blob_export",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ e.start_time AS start_time,+ e.end_time AS end_time,+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time) / 1000) AS latency,+ if(+ isNull(e.completion_start_time),+ NULL,+ dateDiff('millisecond', e.start_time, e.completion_start_time) / 1000+ ) AS time_to_first_token+FROM events_core AS e+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})+LIMIT 1 BY+ e.span_id,+ e.project_id",+ },+]+`;+exports[`golden: events metadata map and session resolution > getEventsForAnalyticsIntegrations 1`] = `[{"fn": "queryClickhouseStream","params": {"param1": "golden-project",- "param2": "2025-12-01 00:00:00.000",- "param3": "2026-01-01 00:00:00.000",+ "param2": 0,+ "param3": "2025-12-01 00:00:00.000",+ "param4": "2026-01-01 00:00:00.000",},"route": "analytics_integration","sql": "SELECT@@ -523,7 +1409,7 @@ exports[`golden: events metadata map and session resolution > getEventsForAnalyte.user_id AS user_id,e.session_id AS session_id,ifNull(- COALESCE(+ coalesce(nullIf(e.trace_name, ''),if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)),@@ -568,7 +1454,7 @@ exports[`golden: events metadata map and session resolution > getEventsForAnalytarrayReverse(e.metadata_values))['$mixpanel_session_id'] AS mixpanel_session_idFROM events_core AS e-WHERE (e.project_id = {param1:String}) AND (e.start_time >= {param2:DateTime64(3)}) AND (e.start_time < {param3:DateTime64(3)}) AND (e.is_deleted = 0)+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})LIMIT 1 BYe.span_id,e.project_id",@@ -576,6 +1462,61 @@ LIMIT 1 BY]`;+exports[`golden: events metadata map and session resolution > getObservationsV2FromEventsTableForPublicApi fields=basic+metrics+trace_context 1`] = `+[+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": "golden-project",+ "param2": 51,+ },+ "route": "unknown",+ "sql": "SELECT+ e.span_id AS id,+ e.trace_id AS trace_id,+ e.project_id AS project_id,+ e.environment AS environment,+ e.type AS type,+ e.parent_span_id AS parent_observation_id,+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,+ e.name AS name,+ e.level AS level,+ e.status_message AS status_message,+ e.version AS version,+ e.bookmarked AS bookmarked,+ e.public AS public,+ e.user_id AS user_id,+ e.session_id AS session_id,+ ifNull(+ coalesce(+ nullIf(e.trace_name, ''),+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)+ ),+ ''+ ) AS trace_name,+ e.start_time AS start_time,+ e.end_time AS end_time,+ e.tags AS tags,+ e.release AS release,+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time)) AS latency,+ if(+ isNull(e.completion_start_time),+ NULL,+ dateDiff('millisecond', e.start_time, e.completion_start_time)+ ) AS time_to_first_token+FROM events_core AS e+WHERE e.project_id = {param1:String}+ORDER BY+ e.project_id DESC,+ toStartOfMinute(e.start_time) DESC,+ e.start_time DESC,+ xxHash32(e.trace_id) DESC,+ e.span_id DESC+LIMIT {param2:Int32}",+ },+]+`;+exports[`golden: events metadata map and session resolution > getObservationsV2FromEventsTableForPublicApi fields=core+metadata expandMetadataKeys=set 1`] = `[{@@ -720,12 +1661,10 @@ exports[`golden: events point-reads & existence family > getAgentGraphDataFromEv{"fn": "queryClickhouse","params": {- "param1": "langgraph_node",- "param2": "langgraph_step",- "param3": "golden-project",- "param4": "golden-trace",- "param5": "2026-01-01 00:00:00.000",- "param6": "2026-01-01 01:00:00.000",+ "param1": "golden-project",+ "param2": "golden-trace",+ "param3": "2026-01-01 00:00:00.000",+ "param4": "2026-01-01 01:00:00.000",},"route": "unknown","sql": "SELECT@@ -735,10 +1674,10 @@ exports[`golden: events point-reads & existence family > getAgentGraphDataFromEve.name AS name,e.start_time AS start_time,e.end_time AS end_time,- mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))[{param1:String}] AS node,- mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))[{param2:String}] AS step+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['langgraph_node'] AS node,+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['langgraph_step'] AS stepFROM events_core AS e-WHERE (e.project_id = {param3:String}) AND (e.trace_id = {param4:String}) AND (e.start_time >= {param5:DateTime64(6)}) AND (e.start_time <= {param6:DateTime64(6)})",+WHERE (e.project_id = {param1:String}) AND (e.trace_id = {param2:String}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time <= {param4:DateTime64(6)})",},]`;@@ -1640,8 +2579,7 @@ exports[`golden: events simple aggregates family > getObservationMetricsForPromp"param3": [],"param4": "input","param5": "output",- "param6": "total",- "param7": 0,+ "param6": 0,},"route": "unknown","sql": "WITH prompt_events AS@@ -1676,10 +2614,10 @@ SELECTmedianExact(arraySum(mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param5:String}) > 0), e.usage_details)))) AS median_output_usage,- medianExact(e.cost_details[{param6:String}]) AS median_total_cost,+ medianExact(e.cost_details['total']) AS median_total_cost,medianExact(dateDiff('millisecond', e.start_time, e.end_time)) AS median_latency_msFROM prompt_events AS e-WHERE e.is_deleted = {param7:UInt8}+WHERE e.is_deleted = {param6:UInt8}GROUP BYe.prompt_id,e.prompt_version@@ -1702,8 +2640,7 @@ exports[`golden: events simple aggregates family > getObservationMetricsForPromp"param4": "2025-12-01 00:00:00.000","param5": "input","param6": "output",- "param7": "total",- "param8": 0,+ "param7": 0,},"route": "unknown","sql": "WITH prompt_events AS@@ -1738,10 +2675,10 @@ SELECTmedianExact(arraySum(mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param6:String}) > 0), e.usage_details)))) AS median_output_usage,- medianExact(e.cost_details[{param7:String}]) AS median_total_cost,+ medianExact(e.cost_details['total']) AS median_total_cost,medianExact(dateDiff('millisecond', e.start_time, e.end_time)) AS median_latency_msFROM prompt_events AS e-WHERE e.is_deleted = {param8:UInt8}+WHERE e.is_deleted = {param7:UInt8}GROUP BYe.prompt_id,e.prompt_version@@ -1765,8 +2702,7 @@ exports[`golden: events simple aggregates family > getObservationMetricsForPromp"param5": "2026-01-02 00:00:00.000","param6": "input","param7": "output",- "param8": "total",- "param9": 0,+ "param8": 0,},"route": "unknown","sql": "WITH prompt_events AS@@ -1801,10 +2737,10 @@ SELECTmedianExact(arraySum(mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param7:String}) > 0), e.usage_details)))) AS median_output_usage,- medianExact(e.cost_details[{param8:String}]) AS median_total_cost,+ medianExact(e.cost_details['total']) AS median_total_cost,medianExact(dateDiff('millisecond', e.start_time, e.end_time)) AS median_latency_msFROM prompt_events AS e-WHERE e.is_deleted = {param9:UInt8}+WHERE e.is_deleted = {param8:UInt8}GROUP BYe.prompt_id,e.prompt_version@@ -1827,8 +2763,7 @@ exports[`golden: events simple aggregates family > getObservationMetricsForPromp"param4": "2026-01-02 00:00:00.000","param5": "input","param6": "output",- "param7": "total",- "param8": 0,+ "param7": 0,},"route": "unknown","sql": "WITH prompt_events AS@@ -1863,10 +2798,10 @@ SELECTmedianExact(arraySum(mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param6:String}) > 0), e.usage_details)))) AS median_output_usage,- medianExact(e.cost_details[{param7:String}]) AS median_total_cost,+ medianExact(e.cost_details['total']) AS median_total_cost,medianExact(dateDiff('millisecond', e.start_time, e.end_time)) AS median_latency_msFROM prompt_events AS e-WHERE e.is_deleted = {param8:UInt8}+WHERE e.is_deleted = {param7:UInt8}GROUP BYe.prompt_id,e.prompt_version@@ -1888,8 +2823,7 @@ exports[`golden: events simple aggregates family > getObservationMetricsForPromp],"param4": "input","param5": "output",- "param6": "total",- "param7": 0,+ "param6": 0,},"route": "unknown","sql": "WITH prompt_events AS@@ -1924,10 +2858,10 @@ SELECTmedianExact(arraySum(mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param5:String}) > 0), e.usage_details)))) AS median_output_usage,- medianExact(e.cost_details[{param6:String}]) AS median_total_cost,+ medianExact(e.cost_details['total']) AS median_total_cost,medianExact(dateDiff('millisecond', e.start_time, e.end_time)) AS median_latency_msFROM prompt_events AS e-WHERE e.is_deleted = {param7:UInt8}+WHERE e.is_deleted = {param6:UInt8}GROUP BYe.prompt_id,e.prompt_version@@ -2064,14 +2998,13 @@ exports[`golden: events simple aggregates family > getSessionMetricsFromEvents f"params": {"param1": "input","param2": "output",- "param3": "total",- "param4": "golden-project",- "param5": [+ "param3": "golden-project",+ "param4": ["golden-session","golden-session-b",],- "param6": "",- "param7": 2,+ "param5": "",+ "param6": 2,},"route": "unknown","sql": "SELECT@@ -2093,18 +3026,18 @@ exports[`golden: events simple aggregates family > getSessionMetricsFromEvents farraySum(mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param2:String}) > 0), sumMap(cost_details)))) AS session_output_cost,- sumMap(cost_details)[{param3:String}] AS session_total_cost,+ sumMap(cost_details)['total'] AS session_total_cost,arraySum(mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param1:String}) > 0), sumMap(usage_details)))) AS session_input_usage,arraySum(mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param2:String}) > 0), sumMap(usage_details)))) AS session_output_usage,- sumMap(usage_details)[{param3:String}] AS session_total_usage+ sumMap(usage_details)['total'] AS session_total_usageFROM events_core AS e-WHERE (e.project_id = {param4:String}) AND (session_id IN ({param5:Array(String)})) AND (session_id != {param6:String})+WHERE (e.project_id = {param3:String}) AND (session_id IN ({param4:Array(String)})) AND (session_id != {param5:String})GROUP BY session_id-LIMIT {param7:Int64}",+LIMIT {param6:Int64}",},]`;diff --git a/packages/shared/src/server/repositories/blobStorageExportWindow.test.ts b/packages/shared/src/server/repositories/blobStorageExportWindow.test.tsindex 2411ae67a7..4cb29be73b 100644--- a/packages/shared/src/server/repositories/blobStorageExportWindow.test.ts+++ b/packages/shared/src/server/repositories/blobStorageExportWindow.test.ts@@ -1,7 +1,9 @@import { describe, it, expect, beforeEach, vi } from "vitest";+type CapturedCall = { query: string; params: Record<string, unknown> };+const mockQueryClickhouseStream = vi.hoisted(() =>- vi.fn(() => (async function* () {})()),+ vi.fn((_call: CapturedCall) => (async function* () {})()),);vi.mock("./clickhouse", () => ({@@ -24,20 +26,37 @@ import { getTracesForBlobStorageExport } from "./traces";import { getObservationsForBlobStorageExport } from "./observations";import { getScoresForBlobStorageExport } from "./scores";import { getEventsForBlobStorageExport } from "./events";+import { convertDateToClickhouseDateTime } from "../clickhouse/client";const PROJECT = "proj-1";const MIN = new Date("2026-01-01T00:00:00.000Z");const MAX = new Date("2026-01-01T01:00:00.000Z");-const capturedQuery = (): string => {+const capturedCall = (): CapturedCall => {expect(mockQueryClickhouseStream).toHaveBeenCalledTimes(1);- const call = mockQueryClickhouseStream.mock.calls[0] as unknown as [- { query: string },- ];- return call[0].query;+ return mockQueryClickhouseStream.mock.calls[0][0];+};++const capturedQuery = (): string => capturedCall().query;++/** The value bound right after `predicate` (e.g. `e.start_time >= `). */+const boundValue = (predicate: string): unknown => {+ const { query, params } = capturedCall();+ const name = query.split(predicate + "{")[1]?.split(":")[0];+ return name === undefined ? undefined : params[name];+};++const namedParamBounds = {+ lower: ">= {minTimestamp",+ upper: "< {maxTimestamp",+ inclusiveUpper: "<= {maxTimestamp",};-const builders = [+const builders: Array<{+ name: string;+ run: () => Promise<void>;+ bounds?: typeof namedParamBounds;+}> = [{name: "getTracesForBlobStorageExport",run: async () => {@@ -62,6 +81,12 @@ const builders = [run: async () => {getEventsForBlobStorageExport(PROJECT, MIN, MAX);},+ // Query-AST binds are named positionally.+ bounds: {+ lower: "e.start_time >= {",+ upper: "e.start_time < {",+ inclusiveUpper: "e.start_time <= {",+ },},];@@ -72,16 +97,27 @@ describe("blob-storage export window is half-open", () => {it.each(builders)("$name uses an exclusive upper bound and inclusive lower bound",- async ({ run }) => {+ async ({ run, bounds = namedParamBounds }) => {await run();const query = capturedQuery();- expect(query).toContain(">= {minTimestamp");- expect(query).toContain("< {maxTimestamp");- expect(query).not.toContain("<= {maxTimestamp");+ expect(query).toContain(bounds.lower);+ expect(query).toContain(bounds.upper);+ expect(query).not.toContain(bounds.inclusiveUpper);},);+ it("getEventsForBlobStorageExport binds min to the lower and max to the upper bound", async () => {+ getEventsForBlobStorageExport(PROJECT, MIN, MAX);++ expect(boundValue("e.start_time >= ")).toBe(+ convertDateToClickhouseDateTime(MIN),+ );+ expect(boundValue("e.start_time < ")).toBe(+ convertDateToClickhouseDateTime(MAX),+ );+ });+it("boundary row belongs to the next window only (traces)", async () => {// Window 1 [MIN, MAX) excludes a row at exactly MAX...getTracesForBlobStorageExport(PROJECT, MIN, MAX);diff --git a/packages/shared/src/server/repositories/events-field-sets.ts b/packages/shared/src/server/repositories/events-field-sets.tsnew file mode 100644index 0000000000..141445c95d--- /dev/null+++ b/packages/shared/src/server/repositories/events-field-sets.ts@@ -0,0 +1,369 @@+import {+ OBSERVATION_FIELD_GROUPS_PUBLIC_API,+ type ObservationFieldGroupPublicApi,+} from "../../domain/observation-field-groups";++/**+ * Every events row field, in canonical SELECT order. Both the raw-SQL+ * `EventsQueryBuilder` and the query-AST selects order their columns by it.+ */+const EVENTS_FIELD_ORDER = [+ "count",+ "uniqueTraceCount",+ "id",+ "traceId",+ "projectId",+ "environment",+ "type",+ "parentObservationId",+ "isRootObservation",+ "isAppRoot",+ "name",+ "level",+ "statusMessage",+ "version",+ "bookmarked",+ "public",+ "userId",+ "sessionId",+ "traceName",+ "startTime",+ "endTime",+ "completionStartTime",+ "createdAt",+ "updatedAt",+ "eventTs",+ "providedModelName",+ "internalModelId",+ "modelParameters",+ "providedUsageDetails",+ "usageDetails",+ "providedCostDetails",+ "costDetails",+ "totalCost",+ "promptId",+ "promptName",+ "promptVersion",+ "skillsAvailable",+ "skillsResourceLoaded",+ "toolDefinitions",+ "toolCalls",+ "toolCallNames",+ "usagePricingTierId",+ "usagePricingTierName",+ "input",+ "output",+ "metadata",+ "tags",+ "release",+ "modelId",+ "experimentId",+ "experimentName",+ "experimentDescription",+ "experimentDatasetId",+ "experimentMetadata",+ "experimentItemId",+ "experimentItemRootSpanId",+ "experimentItemExpectedOutput",+ "experimentItemMetadata",+ "experimentItemVersion",+ "latency",+ "timeToFirstToken",+] as const;++export type EventsFieldKey = (typeof EVENTS_FIELD_ORDER)[number];++const EVENTS_FIELD_ORDER_INDEX = new Map<string, number>(+ EVENTS_FIELD_ORDER.map((key, index) => [key, index]),+);++/**+ * Predefined field sets for common query patterns+ * Maps set names to arrays of field keys from EVENTS_FIELD_ORDER+ */+export const FIELD_SETS = {+ // Aggregates+ count: ["count"],+ countWithUniqueTraces: ["count", "uniqueTraceCount"],++ // List query field sets (for getObservationsWithModelDataFromEventsTable)+ base: [+ "id",+ "type",+ "projectId",+ "name",+ "modelParameters",+ "startTime",+ "endTime",+ "traceId",+ "completionStartTime",+ "providedUsageDetails",+ "usageDetails",+ "providedCostDetails",+ "costDetails",+ "level",+ "environment",+ "bookmarked",+ "public",+ "statusMessage",+ "version",+ "parentObservationId",+ "createdAt",+ "updatedAt",+ "providedModelName",+ "totalCost",+ "usagePricingTierId",+ "usagePricingTierName",+ "promptId",+ "promptName",+ "promptVersion",+ "internalModelId",+ "userId",+ "sessionId",+ "traceName",+ "release",+ "tags",+ "toolDefinitions",+ "toolCalls",+ "toolCallNames",+ ],+ baseWithoutTools: [+ "id",+ "type",+ "projectId",+ "name",+ "modelParameters",+ "startTime",+ "endTime",+ "traceId",+ "completionStartTime",+ "providedUsageDetails",+ "usageDetails",+ "providedCostDetails",+ "costDetails",+ "level",+ "environment",+ "bookmarked",+ "public",+ "statusMessage",+ "version",+ "parentObservationId",+ "createdAt",+ "updatedAt",+ "providedModelName",+ "totalCost",+ "usagePricingTierId",+ "usagePricingTierName",+ "promptId",+ "promptName",+ "promptVersion",+ "internalModelId",+ "userId",+ "sessionId",+ "traceName",+ "release",+ "tags",+ // Keep lightweight tool names available even when heavy tool payloads are omitted.+ "toolCallNames",+ ],+ calculated: ["latency", "timeToFirstToken"],+ io: ["input", "output"],+ metadata: ["metadata"],+ tools: ["toolDefinitions", "toolCalls", "toolCallNames"],+ trace_context: ["tags", "release", "traceName"],+ model_export: ["providedModelName", "modelId", "modelParameters"],+ eventTs: ["eventTs"],+ publicApiExperimentItemCore: [+ "id",+ "projectId",+ "traceId",+ "startTime",+ "endTime",+ "level",+ "environment",+ "experimentId",+ "experimentName",+ "experimentItemId",+ ],+ publicApiExperimentItemDataset: [+ "experimentDatasetId",+ "experimentItemVersion",+ ],+ publicApiExperimentItemExpectedOutput: ["experimentItemExpectedOutput"],+ publicApiExperimentItemMetadataFields: ["experimentItemMetadata"],+ publicApiExperimentItemExperimentMetadata: [+ "experimentMetadata",+ "experimentDescription",+ ],+ // Public API v2 field sets (field groups for selective fetching)+ core: [+ "id",+ "traceId",+ "startTime",+ "endTime",+ "projectId",+ "parentObservationId",+ "type",+ ],+ basic: [+ "name",+ "level",+ "statusMessage",+ "version",+ "environment",+ "bookmarked",+ "public",+ "userId",+ "sessionId",+ "isRootObservation",+ ],+ time: ["completionStartTime", "createdAt", "updatedAt"],+ model: ["providedModelName", "internalModelId", "modelParameters"],+ usage: [+ "usageDetails",+ "costDetails",+ "totalCost",+ "usagePricingTierId",+ "usagePricingTierName",+ ],+ prompt: ["promptId", "promptName", "promptVersion"],+ metrics: ["latency", "timeToFirstToken"],++ // Batch export field set (all fields needed for CSV/JSON exports)+ export: [+ "id",+ "traceId",+ "projectId",+ "startTime",+ "endTime",+ "name",+ "type",+ "environment",+ "version",+ "userId",+ "sessionId",+ "level",+ "statusMessage",+ "promptName",+ "promptId",+ "promptVersion",+ "modelId",+ "providedModelName",+ "modelParameters",+ "usageDetails",+ "costDetails",+ "totalCost",+ "completionStartTime",+ "latency",+ "timeToFirstToken",+ "tags",+ "release",+ "traceName",+ "parentObservationId",+ "bookmarked",+ "public",+ "createdAt",+ "updatedAt",+ "toolDefinitions",+ "toolCalls",+ "toolCallNames",+ "usagePricingTierId",+ "usagePricingTierName",+ ],++ eval: [+ "id",+ "traceId",+ "projectId",+ "parentObservationId",+ "isAppRoot",+ "type",+ "name",+ "environment",+ "version",+ "level",+ "statusMessage",+ "traceName",+ "userId",+ "sessionId",+ "tags",+ "release",+ "providedModelName",+ "modelParameters",+ "promptId",+ "promptName",+ "promptVersion",+ "providedUsageDetails",+ "usageDetails",+ "providedCostDetails",+ "costDetails",+ "toolDefinitions",+ "toolCalls",+ "toolCallNames",+ "experimentId",+ "experimentName",+ "experimentItemRootSpanId",+ "experimentItemExpectedOutput",+ "experimentItemMetadata",+ ],++ // Experiment items field set+ experimentItems: [+ "id", // span_id (observation_id)+ "traceId",+ "input",+ "output",+ "startTime",+ "level",++ // Experiment metadata+ "experimentId",+ "experimentName",+ "experimentDatasetId",++ // Experiment item metadata+ "experimentItemId",+ "experimentItemRootSpanId",+ "experimentItemVersion",+ "experimentItemExpectedOutput",+ "experimentItemMetadata",++ // Event metadata+ "metadata",+ ],+} as const satisfies Record<string, readonly EventsFieldKey[]>;++export type FieldSetName = keyof typeof FIELD_SETS;++// Canonicalize by EVENTS_FIELD_ORDER so SELECT column order does+// not follow caller field-set order. Clustered ClickHouse reads align result+// blocks by position; incompatible Map types (cost_details vs metadata) 500+// when those columns swap places.+export function orderedFieldKeys<K extends string>(+ fieldKeys: readonly K[],+): K[] {+ return [...new Set(fieldKeys)].sort(+ (a, b) =>+ (EVENTS_FIELD_ORDER_INDEX.get(a) ?? Number.MAX_SAFE_INTEGER) -+ (EVENTS_FIELD_ORDER_INDEX.get(b) ?? Number.MAX_SAFE_INTEGER),+ );+}++export type FieldSetFieldKey<S extends FieldSetName> =+ (typeof FIELD_SETS)[S][number];++/** The fields of `sets`, deduplicated, in the canonical SELECT order. */+export function eventsFieldSetKeys<S extends FieldSetName>(+ ...sets: S[]+): FieldSetFieldKey<S>[] {+ return orderedFieldKeys(sets.flatMap((s) => FIELD_SETS[s]));+}++export const OBSERVATION_FIELD_GROUP_FIELD_NAMES = Object.fromEntries(+ OBSERVATION_FIELD_GROUPS_PUBLIC_API.map((group) => [+ group,+ FIELD_SETS[group],+ ]),+) as {+ [K in ObservationFieldGroupPublicApi]: (typeof FIELD_SETS)[K];+};diff --git a/packages/shared/src/server/repositories/events-selects.ts b/packages/shared/src/server/repositories/events-selects.tsindex ea9354a29f..96c2b52594 100644--- a/packages/shared/src/server/repositories/events-selects.ts+++ b/packages/shared/src/server/repositories/events-selects.ts@@ -29,12 +29,37 @@ import {nullIf,sumMap,sumMapValuesWhereKeyContains,+ toBool,traceSessionId,uniq,uniqIf,} from "../query-ast/extensions";import type { ClickHouseDatabase } from "../query-ast/schema";-import type { SessionEventsField } from "../queries/clickhouse-sql/event-query-builder";+import type { FieldSetFieldKey } from "./events-field-sets";++/** One row of the session aggregation (`SESSION_AGGREGATION_SELECTS`). */+export type SessionEventsMetricsRow = {+ session_id: string;+ max_timestamp: string;+ min_timestamp: string;+ trace_ids: string[];+ user_ids: string[];+ trace_count: number;+ trace_tags: string[];+ environment?: string;+ total_observations: number;+ duration: number;+ session_usage_details: Record<string, number>;+ session_cost_details: Record<string, number>;+ session_input_cost: string;+ session_output_cost: string;+ session_total_cost: string;+ session_input_usage: string;+ session_output_usage: string;+ session_total_usage: string;+};++export type SessionEventsField = keyof SessionEventsMetricsRow;type EventsDb = { e: ClickHouseDatabase["events_core"] };type EventsRow = ClickHouseDatabase["events_core"];@@ -51,7 +76,8 @@ const eventsMetadataMap = metadataMap(eb.ref("e.metadata_values"),);-const isRootObservation = eb.or([+/** The event is a root observation: no parent span, or an app root. */+export const eventsIsRootObservation = eb.or([eb("e.parent_span_id", "=", literal("")),eb("e.is_app_root", "=", literal(true)),]);@@ -63,12 +89,22 @@ const isRootObservation = eb.or([export const eventsTraceName = eb.fn.coalesce(nullIf(eb.ref("e.trace_name"), literal("")),ifElse(- isRootObservation,+ eventsIsRootObservation,nullIf(eb.ref("e.name"), literal("")),literal(null),),);+/**+ * {@link eventsTraceName} as a non-null String, '' when the event has no trace+ * name. Projected under `trace_name`, the nullable form would clash with the+ * physical String column of the same name once a query also filters and sorts+ * on it (ClickHouse AMBIGUOUS_COLUMN_NAME).+ */+export const eventsTraceNameOrEmpty = ifNull(eventsTraceName, literal(""));++const traceNameSelect = eventsTraceNameOrEmpty.as("trace_name");+/** The latest non-empty value of `column` across a trace's events. */const latestNonEmpty = (column: StringColumn) =>argMaxIf(eb.ref(column), eb.ref("event_ts"), eb(column, "<>", literal("")));@@ -104,7 +140,7 @@ export const TRACE_AGGREGATION_SELECTS = [argMaxIf(eb.ref("e.name"),eb.ref("e.event_ts"),- eb.and([isRootObservation, eb("e.name", "<>", literal(""))]),+ eb.and([eventsIsRootObservation, eb("e.name", "<>", literal(""))]),),literal(""),),@@ -222,9 +258,10 @@ export const SESSION_AGGREGATION_SELECTS = {sumMap(eb.ref("cost_details")),"output",).as("session_output_cost"),- session_total_cost: mapValue(sumMap(eb.ref("cost_details")), "total").as(- "session_total_cost",- ),+ session_total_cost: mapValue(+ sumMap(eb.ref("cost_details")),+ literal("total"),+ ).as("session_total_cost"),session_input_usage: sumMapValuesWhereKeyContains(sumMap(eb.ref("usage_details")),"input",@@ -233,9 +270,10 @@ export const SESSION_AGGREGATION_SELECTS = {sumMap(eb.ref("usage_details")),"output",).as("session_output_usage"),- session_total_usage: mapValue(sumMap(eb.ref("usage_details")), "total").as(- "session_total_usage",- ),+ session_total_usage: mapValue(+ sumMap(eb.ref("usage_details")),+ literal("total"),+ ).as("session_total_usage"),} satisfies Record<SessionEventsField, EventsSelection>;/** One observation's fields, without input/output. */@@ -254,8 +292,7 @@ export const OBSERVATION_BY_ID_SELECTS = ["e.public as public","e.user_id as user_id","e.session_id as session_id",- // String-typed on purpose: '' when the event has no trace name.- ifNull(eventsTraceName, literal("")).as("trace_name"),+ traceNameSelect,"e.start_time as start_time","e.end_time as end_time","e.completion_start_time as completion_start_time",@@ -286,3 +323,94 @@ export const OBSERVATION_BY_ID_SELECTS = ["e.tags as tags","e.release as release",] satisfies EventsSelection[];++/** Milliseconds from an event's start to `end`, NULL while `end` is unset. */+export const millisecondsSinceStart = (+ end: "e.end_time" | "e.completion_start_time",+) =>+ ifElse(+ isNull(eb.ref(end)),+ literal(null),+ dateDiff("millisecond", eb.ref("e.start_time"), eb.ref(end)),+ );++/** {@link millisecondsSinceStart} in seconds. */+const secondsSinceStart = (end: "e.end_time" | "e.completion_start_time") =>+ ifElse(+ isNull(eb.ref(end)),+ literal(null),+ eb(+ dateDiff("millisecond", eb.ref("e.start_time"), eb.ref(end)),+ "/",+ literal(1000),+ ),+ );++type RowFieldSet =+ | "core"+ | "basic"+ | "time"+ | "metadata"+ | "model_export"+ | "usage"+ | "prompt"+ | "metrics"+ | "trace_context"+ | "tools"+ | "export";++/**+ * Row-level event fields, keyed like the legacy builder's field sets. Select+ * them in `eventsFieldSetKeys` order: clustered reads align result blocks by+ * column position.+ */+export const EVENTS_ROW_FIELD_SELECTS = {+ id: "e.span_id as id",+ traceId: "e.trace_id as trace_id",+ projectId: "e.project_id as project_id",+ environment: "e.environment as environment",+ type: "e.type as type",+ parentObservationId: "e.parent_span_id as parent_observation_id",+ isRootObservation: toBool(eventsIsRootObservation).as("is_root_observation"),+ name: "e.name as name",+ level: "e.level as level",+ statusMessage: "e.status_message as status_message",+ version: "e.version as version",+ bookmarked: "e.bookmarked as bookmarked",+ public: "e.public as public",+ userId: "e.user_id as user_id",+ sessionId: "e.session_id as session_id",+ traceName: traceNameSelect,+ startTime: "e.start_time as start_time",+ endTime: "e.end_time as end_time",+ completionStartTime: "e.completion_start_time as completion_start_time",+ createdAt: "e.created_at as created_at",+ updatedAt: "e.updated_at as updated_at",+ providedModelName: "e.provided_model_name as provided_model_name",+ modelParameters: "e.model_parameters as model_parameters",+ usageDetails: "e.usage_details as usage_details",+ costDetails: "e.cost_details as cost_details",+ totalCost: "e.total_cost as total_cost",+ promptId: "e.prompt_id as prompt_id",+ promptName: "e.prompt_name as prompt_name",+ promptVersion: "e.prompt_version as prompt_version",+ toolDefinitions: "e.tool_definitions as tool_definitions",+ toolCalls: "e.tool_calls as tool_calls",+ toolCallNames: "e.tool_call_names as tool_call_names",+ usagePricingTierId: "e.usage_pricing_tier_id as usage_pricing_tier_id",+ usagePricingTierName: "e.usage_pricing_tier_name as usage_pricing_tier_name",+ metadata: eventsMetadataMap.as("metadata"),+ tags: "e.tags as tags",+ release: "e.release as release",+ modelId: "e.model_id as model_id",+ latency: millisecondsSinceStart("e.end_time").as("latency"),+ timeToFirstToken: millisecondsSinceStart("e.completion_start_time").as(+ "time_to_first_token",+ ),+} satisfies Record<FieldSetFieldKey<RowFieldSet>, EventsSelection>;++/** The `metrics` field set with latencies in seconds instead of milliseconds. */+export const EVENTS_METRICS_SECONDS_SELECTS = [+ secondsSinceStart("e.end_time").as("latency"),+ secondsSinceStart("e.completion_start_time").as("time_to_first_token"),+] satisfies EventsSelection[];diff --git a/packages/shared/src/server/repositories/events.golden.test.ts b/packages/shared/src/server/repositories/events.golden.test.tsindex c83836402e..aafc467c71 100644--- a/packages/shared/src/server/repositories/events.golden.test.ts+++ b/packages/shared/src/server/repositories/events.golden.test.ts@@ -8,6 +8,7 @@ import {vi,} from "vitest";+import { OBSERVATION_FIELD_GROUPS_FULL } from "../../domain/observation-field-groups";import { env } from "../../env";import { LangfuseNotFoundError } from "../../errors";import {@@ -29,6 +30,9 @@ vi.mock("./clickhouse", async (importOriginal) => {import {getAgentGraphDataFromEventsTable,getEventsForAnalyticsIntegrations,+ getEventsForBlobStorageExport,+ getEventsForBlobStorageExportParquet,+ getEventsForBlobStorageExportRaw,getLastTraceTimestampsByProjectsFromEventsTable,getLatestSdkVersionInfoFromEvents,getObservationByIdFromEventsTable,@@ -490,6 +494,16 @@ describeWithClickhouse(expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();});+ it("getObservationsV2FromEventsTableForPublicApi fields=basic+metrics+trace_context", async () => {+ await getObservationsV2FromEventsTableForPublicApi({+ projectId: FIXED_PROJECT_ID,+ page: 1,+ limit: 50,+ fields: ["basic", "metrics", "trace_context"],+ });+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();+ });+it("getEventsForAnalyticsIntegrations", async () => {for await (const _ of getEventsForAnalyticsIntegrations(FIXED_PROJECT_ID,@@ -519,3 +533,101 @@ describeWithClickhouse(});},);++describeWithClickhouse("golden: events export streams family", () => {+ beforeEach(() => {+ resetCaptures();+ });++ // The harness only records a stream's query once it is iterated.+ async function drain(rows: AsyncIterable<unknown>) {+ for await (const _ of rows) {+ // The harness streams no rows.+ }+ }++ it("getEventsForBlobStorageExport fieldGroups=default", async () => {+ await drain(+ getEventsForBlobStorageExport(+ FIXED_PROJECT_ID,+ FIXED_FROM_TIMESTAMP,+ FIXED_TO_TIMESTAMP,+ ),+ );+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();+ });++ it.each(OBSERVATION_FIELD_GROUPS_FULL)(+ "getEventsForBlobStorageExport fieldGroups=%s",+ async (group) => {+ await drain(+ getEventsForBlobStorageExport(+ FIXED_PROJECT_ID,+ FIXED_FROM_TIMESTAMP,+ FIXED_TO_TIMESTAMP,+ [group],+ ),+ );+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();+ },+ );++ it("getEventsForBlobStorageExport fieldGroups=empty", async () => {+ await drain(+ getEventsForBlobStorageExport(+ FIXED_PROJECT_ID,+ FIXED_FROM_TIMESTAMP,+ FIXED_TO_TIMESTAMP,+ [],+ ),+ );+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();+ });++ it("getEventsForBlobStorageExport fieldGroups=reversed", async () => {+ await drain(+ getEventsForBlobStorageExport(+ FIXED_PROJECT_ID,+ FIXED_FROM_TIMESTAMP,+ FIXED_TO_TIMESTAMP,+ [...OBSERVATION_FIELD_GROUPS_FULL].reverse(),+ ),+ );+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();+ });++ it.each([+ ["default", undefined, undefined],+ ["default", true, [...OBSERVATION_FIELD_GROUPS_FULL]],+ ["metrics", true, ["metrics"]],+ ["core+io", true, ["core", "io"]],+ ] as const)(+ "getEventsForBlobStorageExportRaw fieldGroups=%s convertLatencyToSeconds=%s",+ async (_name, convertLatencyToSeconds, fieldGroups) => {+ await drain(+ getEventsForBlobStorageExportRaw(+ FIXED_PROJECT_ID,+ FIXED_FROM_TIMESTAMP,+ FIXED_TO_TIMESTAMP,+ fieldGroups ? [...fieldGroups] : undefined,+ convertLatencyToSeconds,+ ),+ );+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();+ },+ );++ it.each([false, true])(+ "getEventsForBlobStorageExportParquet convertLatencyToSeconds=%s",+ async (convertLatencyToSeconds) => {+ await getEventsForBlobStorageExportParquet(+ FIXED_PROJECT_ID,+ FIXED_FROM_TIMESTAMP,+ FIXED_TO_TIMESTAMP,+ undefined,+ convertLatencyToSeconds,+ );+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();+ },+ );+});diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex ae3974d132..f1ad3b54a2 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -44,6 +44,7 @@ import {mapValue,medianExact,metadataMap,+ metadataValue,sumMapValuesWhereKeyContains,toDate,toJSONString,@@ -135,10 +136,14 @@ import {} from "./observations";import { convertEventsObservation } from "./observations_converters";import {+ EVENTS_METRICS_SECONDS_SELECTS,+ EVENTS_ROW_FIELD_SELECTS,eventsTraceName,OBSERVATION_BY_ID_SELECTS,SESSION_AGGREGATION_SELECTS,TRACE_AGGREGATION_SELECTS,+ type SessionEventsField,+ type SessionEventsMetricsRow,} from "./events-selects";import {EventsQueryBuilder,@@ -147,12 +152,10 @@ import {buildEventsFullTableSplitQuery,type QueryWithParams,normalizeSessionCounts,- type SessionEventsField,type SessionEventsFields,- type SessionEventsMetricsRow,OrderByEntry,- EVENTS_METADATA_MAP_SQL,} from "../queries/clickhouse-sql/event-query-builder";+import { eventsFieldSetKeys } from "./events-field-sets";import {buildEventsFilterOptionColumnQuery,buildEventsFilterOptionsForColumnsQuery,@@ -683,7 +686,7 @@ export const getObservationMetricsForPromptsFromEvents = async (medianExact(sumMapValuesWhereKeyContains(eb.ref("e.usage_details"), "output"),).as("median_output_usage"),- medianExact(mapValue(eb.ref("e.cost_details"), "total")).as(+ medianExact(mapValue(eb.ref("e.cost_details"), literal("total"))).as("median_total_cost",),medianExact(@@ -2639,8 +2642,8 @@ export async function getAgentGraphDataFromEventsTable(params: {"e.name as name","e.start_time as start_time","e.end_time as end_time",- mapValue<string>(metadata, "langgraph_node").as("node"),- mapValue<string>(metadata, "langgraph_step").as("step"),+ mapValue<string>(metadata, literal("langgraph_node")).as("node"),+ mapValue<string>(metadata, literal("langgraph_step")).as("step"),];}).where("e.trace_id", "=", traceId)@@ -3381,22 +3384,11 @@ export const hasAnyUserFromEventsTable = async (};/**- * Streams events from ClickHouse for blob storage export.- * Uses EventsQueryBuilder for consistent query construction.+ * Query for the events blob export. Full I/O reads events_full; everything+ * else reads events_core. `convertLatencyToSecondsInSql` can be set by the paths+ * that skip the JS row enrichment (raw passthrough, Parquet), so they select+ * latency and time to first token in seconds instead of milliseconds.*/-// metrics field set emits latency / time_to_first_token in MILLISECONDS. The-// raw-passthrough path (LFE-10402) cannot convert ms→s in JS, so when requested-// it selects the seconds-converted expressions directly in SQL instead. The-// standard path keeps ms here and converts conditionally in JS, so it stays-// byte-for-byte unchanged.-const EVENTS_EXPORT_METRICS_SECONDS_SQL = [- "if(isNull(e.end_time), NULL, date_diff('millisecond', e.start_time, e.end_time) / 1000) as latency",- "if(isNull(e.completion_start_time), NULL, date_diff('millisecond', e.start_time, e.completion_start_time) / 1000) as \"time_to_first_token\"",-];--// Shared SQL+params builder for the events blob-export. `convertLatencyToSecondsInSql`-// is only set by the raw-passthrough path; the standard path leaves it false so-// its output is identical to today.const buildEventsForBlobStorageExportQuery = (projectId: string,minTimestamp: Date,@@ -3404,37 +3396,33 @@ const buildEventsForBlobStorageExportQuery = (fieldGroups: ObservationFieldGroupFull[],convertLatencyToSecondsInSql: boolean,) => {- const queryBuilder = new EventsQueryBuilder({ projectId });-// core is always required (provides id, trace_id, start/end_time used for cursor and deduplication)const effectiveGroups = fieldGroups.includes("core")? fieldGroups: (["core", ...fieldGroups] as ObservationFieldGroupFull[]);+ const includeIO = effectiveGroups.includes("io");+ const metricsInSeconds =+ convertLatencyToSecondsInSql && effectiveGroups.includes("metrics");+ const fieldSets = effectiveGroups.flatMap((group) => {+ if (group === "io" || (group === "metrics" && metricsInSeconds)) return [];+ // "model_export" is the field set name for the "model" group+ return group === "model" ? ["model_export" as const] : [group];+ });- for (const group of effectiveGroups) {- if (group === "io") {- queryBuilder.selectIO(false); // Full I/O, no truncation- } else if (group === "model") {- queryBuilder.selectFieldSet("model_export"); // "model_export" is the SQL field set name for the "model" group- } else if (group === "metrics" && convertLatencyToSecondsInSql) {- queryBuilder.selectRaw(...EVENTS_EXPORT_METRICS_SECONDS_SQL);- } else {- queryBuilder.selectFieldSet(group);- }- }-- queryBuilder- .whereRaw(- "e.start_time >= {minTimestamp: DateTime64(3)} AND e.start_time < {maxTimestamp: DateTime64(3)}",- {- minTimestamp: convertDateToClickhouseDateTime(minTimestamp),- maxTimestamp: convertDateToClickhouseDateTime(maxTimestamp),- },+ const builder = getClickhouseKysely()+ .selectFrom(includeIO ? "events_full as e" : "events_core as e")+ .select(+ eventsFieldSetKeys(...fieldSets).map(+ (key) => EVENTS_ROW_FIELD_SELECTS[key],+ ),)- .whereRaw("e.is_deleted = 0")- .limitBy("e.span_id", "e.project_id");+ .$if(includeIO, (qb) => qb.select(["input", "output"]))+ .$if(metricsInSeconds, (qb) => qb.select(EVENTS_METRICS_SECONDS_SELECTS))+ .where("e.start_time", ">=", minTimestamp)+ .where("e.start_time", "<", maxTimestamp)+ .$call(limitBy({ count: 1, columns: ["e.span_id", "e.project_id"] }));- const { query, params } = queryBuilder.buildWithParams();+ const { sql: query, params } = compileClickhouseQuery(builder, projectId);return {query,@@ -3513,7 +3501,6 @@ export const getEventsForBlobStorageExportParquet = function (/*** Streams events from ClickHouse for analytics integrations (PostHog, Mixpanel).- * Uses EventsQueryBuilder for consistent query construction.* All fields come directly from the events table (which has denormalized trace-level data).*/export const getEventsForAnalyticsIntegrations = async function* (@@ -3522,31 +3509,34 @@ export const getEventsForAnalyticsIntegrations = async function* (minTimestamp: Date,maxTimestamp: Date,) {- const queryBuilder = new EventsQueryBuilder({ projectId })+ const builder = getClickhouseKysely()+ .selectFrom("events_core as e")// Use export field set for most fields (id, traceId, name, type, level, version,// environment, userId, sessionId, tags, release, traceName, totalCost, latency, etc.)- .selectFieldSet("export")+ .select(+ eventsFieldSetKeys("export").map((key) => EVENTS_ROW_FIELD_SELECTS[key]),+ )// Add analytics-specific computed fields- .selectRaw(+ .select((eb) => [// Token counts from usage/cost details- "e.usage_details['input'] as input_tokens",- "e.usage_details['output'] as output_tokens",- "e.usage_details['total'] as total_tokens",+ mapValue(eb.ref("e.usage_details"), literal("input")).as("input_tokens"),+ mapValue(eb.ref("e.usage_details"), literal("output")).as(+ "output_tokens",+ ),+ mapValue(eb.ref("e.usage_details"), literal("total")).as("total_tokens"),// Analytics integration session IDs from metadata (constructed from array columns)- `${EVENTS_METADATA_MAP_SQL}['$posthog_session_id'] as posthog_session_id`,- `${EVENTS_METADATA_MAP_SQL}['$mixpanel_session_id'] as mixpanel_session_id`,- )- .whereRaw(- "e.start_time >= {minTimestamp: DateTime64(3)} AND e.start_time < {maxTimestamp: DateTime64(3)}",- {- minTimestamp: convertDateToClickhouseDateTime(minTimestamp),- maxTimestamp: convertDateToClickhouseDateTime(maxTimestamp),- },- )- .whereRaw("e.is_deleted = 0")- .limitBy("e.span_id", "e.project_id");+ metadataValue("e", literal("$posthog_session_id")).as(+ "posthog_session_id",+ ),+ metadataValue("e", literal("$mixpanel_session_id")).as(+ "mixpanel_session_id",+ ),+ ])+ .where("e.start_time", ">=", minTimestamp)+ .where("e.start_time", "<", maxTimestamp)+ .$call(limitBy({ count: 1, columns: ["e.span_id", "e.project_id"] }));- const { query, params } = queryBuilder.buildWithParams();+ const { sql: query, params } = compileClickhouseQuery(builder, projectId);const records = queryClickhouseStream<Record<string, unknown>>({query,@@ -3565,7 +3555,7 @@ export const getEventsForAnalyticsIntegrations = async function* (timestamp: record.start_time,langfuse_observation_name: record.name,// The row type here is Record<string, unknown>; the column is a String- // projection (eventsTableTraceNameSelectSql).+ // projection ('' when the event has no trace name).langfuse_trace_name: normalizeEventsTraceName(record.trace_name as string | null | undefined,),