Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

refactor(query-ast): compile events export streams through the AST (#18867)

Niklas Semmler1744b5d7e864
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
index fe6432d780..b4aecdbac8 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
@@ -1,18 +1,27 @@
-import {
- OBSERVATION_FIELD_GROUPS_PUBLIC_API,
- type ObservationFieldGroupPublicApi,
-} from "../../../domain/observation-field-groups";
-import {
- eventsTableIsRootObservationSql,
- eventsTableTraceNameAggregationSql,
- eventsTableTraceNameSelectSql,
-} from "../../../eventsTable";
+import { eventsTableTraceNameAggregationSql } from "../../../eventsTable";
import { expressionBuilder } from "kysely";
import { renderClickhouseExpression } from "../../query-ast/compile";
-import { metadataMap, traceSessionId } from "../../query-ast/extensions";
+import {
+ metadataMap,
+ toBool,
+ traceSessionId,
+} from "../../query-ast/extensions";
import type { ClickHouseDatabase } from "../../query-ast/schema";
import { OBSERVATIONS_TO_TRACE_INTERVAL } from "../../repositories/constants";
+import {
+ FIELD_SETS,
+ orderedFieldKeys,
+ type EventsFieldKey,
+ type FieldSetName,
+} from "../../repositories/events-field-sets";
+import {
+ eventsTraceNameOrEmpty,
+ eventsIsRootObservation,
+ millisecondsSinceStart,
+ type SessionEventsField,
+ type SessionEventsMetricsRow,
+} from "../../repositories/events-selects";
import {
FilterList,
StringFilter,
@@ -29,8 +38,7 @@ const eventsMetadataMap = metadataMap(
eventsRef.ref("e.metadata_values"),
);
-export const EVENTS_METADATA_MAP_SQL =
- renderClickhouseExpression(eventsMetadataMap);
+const EVENTS_METADATA_MAP_SQL = renderClickhouseExpression(eventsMetadataMap);
const TRACE_SESSION_ID_SQL = renderClickhouseExpression(traceSessionId());
@@ -146,7 +154,7 @@ const EVENTS_FIELDS = {
environment: "e.environment as environment",
type: "e.type as type",
parentObservationId: "e.parent_span_id as parent_observation_id",
- isRootObservation: `toBool(${eventsTableIsRootObservationSql}) as "is_root_observation"`,
+ isRootObservation: `${renderClickhouseExpression(toBool(eventsIsRootObservation))} as "is_root_observation"`,
isAppRoot: "e.is_app_root as is_app_root",
name: "e.name as name",
level: "e.level as level",
@@ -156,8 +164,7 @@ const EVENTS_FIELDS = {
public: "e.public as public",
userId: "e.user_id as user_id",
sessionId: "e.session_id as session_id",
- // String-typed on purpose; see eventsTableTraceNameSelectSql (LFE-14924).
- traceName: `${eventsTableTraceNameSelectSql} as "trace_name"`,
+ traceName: `${renderClickhouseExpression(eventsTraceNameOrEmpty)} as "trace_name"`,
// Time fields
startTime: "e.start_time as start_time",
@@ -227,285 +234,9 @@ const EVENTS_FIELDS = {
experimentItemVersion: "e.experiment_item_version as experiment_item_version",
// Calculated fields
- latency:
- "if(isNull(e.end_time), NULL, date_diff('millisecond', e.start_time, e.end_time)) as latency",
- timeToFirstToken:
- "if(isNull(e.completion_start_time), NULL, date_diff('millisecond', e.start_time, e.completion_start_time)) as \"time_to_first_token\"",
-} as const;
-
-const EVENTS_FIELD_ORDER_INDEX = new Map(
- Object.keys(EVENTS_FIELDS).map((key, index) => [key, index]),
-);
-
-/**
- * Predefined field sets for common query patterns
- * Maps set names to arrays of field keys from EVENTS_FIELDS
- */
-const FIELD_SETS = {
- // Aggregates
- count: ["count"],
- countWithUniqueTraces: ["count", "uniqueTraceCount"],
-
- // List query field sets (for getObservationsWithModelDataFromEventsTable)
- base: [
- "id",
- "type",
- "projectId",
- "name",
- "modelParameters",
- "startTime",
- "endTime",
- "traceId",
- "completionStartTime",
- "providedUsageDetails",
- "usageDetails",
- "providedCostDetails",
- "costDetails",
- "level",
- "environment",
- "bookmarked",
- "public",
- "statusMessage",
- "version",
- "parentObservationId",
- "createdAt",
- "updatedAt",
- "providedModelName",
- "totalCost",
- "usagePricingTierId",
- "usagePricingTierName",
- "promptId",
- "promptName",
- "promptVersion",
- "internalModelId",
- "userId",
- "sessionId",
- "traceName",
- "release",
- "tags",
- "toolDefinitions",
- "toolCalls",
- "toolCallNames",
- ],
- baseWithoutTools: [
- "id",
- "type",
- "projectId",
- "name",
- "modelParameters",
- "startTime",
- "endTime",
- "traceId",
- "completionStartTime",
- "providedUsageDetails",
- "usageDetails",
- "providedCostDetails",
- "costDetails",
- "level",
- "environment",
- "bookmarked",
- "public",
- "statusMessage",
- "version",
- "parentObservationId",
- "createdAt",
- "updatedAt",
- "providedModelName",
- "totalCost",
- "usagePricingTierId",
- "usagePricingTierName",
- "promptId",
- "promptName",
- "promptVersion",
- "internalModelId",
- "userId",
- "sessionId",
- "traceName",
- "release",
- "tags",
- // Keep lightweight tool names available even when heavy tool payloads are omitted.
- "toolCallNames",
- ],
- calculated: ["latency", "timeToFirstToken"],
- io: ["input", "output"],
- metadata: ["metadata"],
- tools: ["toolDefinitions", "toolCalls", "toolCallNames"],
- trace_context: ["tags", "release", "traceName"],
- model_export: ["providedModelName", "modelId", "modelParameters"],
- eventTs: ["eventTs"],
- publicApiExperimentItemCore: [
- "id",
- "projectId",
- "traceId",
- "startTime",
- "endTime",
- "level",
- "environment",
- "experimentId",
- "experimentName",
- "experimentItemId",
- ],
- publicApiExperimentItemDataset: [
- "experimentDatasetId",
- "experimentItemVersion",
- ],
- publicApiExperimentItemExpectedOutput: ["experimentItemExpectedOutput"],
- publicApiExperimentItemMetadataFields: ["experimentItemMetadata"],
- publicApiExperimentItemExperimentMetadata: [
- "experimentMetadata",
- "experimentDescription",
- ],
- // Public API v2 field sets (field groups for selective fetching)
- core: [
- "id",
- "traceId",
- "startTime",
- "endTime",
- "projectId",
- "parentObservationId",
- "type",
- ],
- basic: [
- "name",
- "level",
- "statusMessage",
- "version",
- "environment",
- "bookmarked",
- "public",
- "userId",
- "sessionId",
- "isRootObservation",
- ],
- time: ["completionStartTime", "createdAt", "updatedAt"],
- model: ["providedModelName", "internalModelId", "modelParameters"],
- usage: [
- "usageDetails",
- "costDetails",
- "totalCost",
- "usagePricingTierId",
- "usagePricingTierName",
- ],
- prompt: ["promptId", "promptName", "promptVersion"],
- metrics: ["latency", "timeToFirstToken"],
-
- // Batch export field set (all fields needed for CSV/JSON exports)
- export: [
- "id",
- "traceId",
- "projectId",
- "startTime",
- "endTime",
- "name",
- "type",
- "environment",
- "version",
- "userId",
- "sessionId",
- "level",
- "statusMessage",
- "promptName",
- "promptId",
- "promptVersion",
- "modelId",
- "providedModelName",
- "modelParameters",
- "usageDetails",
- "costDetails",
- "totalCost",
- "completionStartTime",
- "latency",
- "timeToFirstToken",
- "tags",
- "release",
- "traceName",
- "parentObservationId",
- "bookmarked",
- "public",
- "createdAt",
- "updatedAt",
- "toolDefinitions",
- "toolCalls",
- "toolCallNames",
- "usagePricingTierId",
- "usagePricingTierName",
- ],
-
- eval: [
- "id",
- "traceId",
- "projectId",
- "parentObservationId",
- "isAppRoot",
- "type",
- "name",
- "environment",
- "version",
- "level",
- "statusMessage",
- "traceName",
- "userId",
- "sessionId",
- "tags",
- "release",
- "providedModelName",
- "modelParameters",
- "promptId",
- "promptName",
- "promptVersion",
- "providedUsageDetails",
- "usageDetails",
- "providedCostDetails",
- "costDetails",
- "toolDefinitions",
- "toolCalls",
- "toolCallNames",
- "experimentId",
- "experimentName",
- "experimentItemRootSpanId",
- "experimentItemExpectedOutput",
- "experimentItemMetadata",
- ],
-
- // Experiment items field set
- experimentItems: [
- "id", // span_id (observation_id)
- "traceId",
- "input",
- "output",
- "startTime",
- "level",
-
- // Experiment metadata
- "experimentId",
- "experimentName",
- "experimentDatasetId",
-
- // Experiment item metadata
- "experimentItemId",
- "experimentItemRootSpanId",
- "experimentItemVersion",
- "experimentItemExpectedOutput",
- "experimentItemMetadata",
-
- // Event metadata
- "metadata",
- ],
-} as const;
-
-export type FieldSetName = keyof typeof FIELD_SETS;
-
-// Canonicalize by EVENTS_FIELDS declaration order so SELECT column order does
-// not follow caller field-set order. Clustered ClickHouse reads align result
-// blocks by position; incompatible Map types (cost_details vs metadata) 500
-// when those columns swap places.
-function orderedFieldKeys<K extends string>(fieldKeys: readonly K[]): K[] {
- return [...new Set(fieldKeys)].sort(
- (a, b) =>
- (EVENTS_FIELD_ORDER_INDEX.get(a) ?? Number.MAX_SAFE_INTEGER) -
- (EVENTS_FIELD_ORDER_INDEX.get(b) ?? Number.MAX_SAFE_INTEGER),
- );
-}
+ latency: `${renderClickhouseExpression(millisecondsSinceStart("e.end_time"))} as latency`,
+ timeToFirstToken: `${renderClickhouseExpression(millisecondsSinceStart("e.completion_start_time"))} as "time_to_first_token"`,
+} as const satisfies Record<EventsFieldKey, string>;
function eventsFieldSelectExpressions(fieldKeys: readonly string[]): string[] {
return orderedFieldKeys(fieldKeys).flatMap((fieldKey) => {
@@ -514,15 +245,6 @@ function eventsFieldSelectExpressions(fieldKeys: readonly string[]): string[] {
});
}
-export const OBSERVATION_FIELD_GROUP_FIELD_NAMES = Object.fromEntries(
- OBSERVATION_FIELD_GROUPS_PUBLIC_API.map((group) => [
- group,
- FIELD_SETS[group],
- ]),
-) as {
- [K in ObservationFieldGroupPublicApi]: (typeof FIELD_SETS)[K];
-};
-
/**
* Aggregation fields for trace-level queries
* These fields use ClickHouse aggregation functions and require GROUP BY
@@ -1436,33 +1158,6 @@ export class EventsAggregationQueryBuilder extends BaseEventsQueryBuilder<
}
}
-/**
- * ClickHouse row type for session-level metrics queries from the events table.
- * Matches the columns produced by EVENTS_SESSION_AGGREGATION_FIELDS below.
- */
-export type SessionEventsMetricsRow = {
- session_id: string;
- max_timestamp: string;
- min_timestamp: string;
- trace_ids: string[];
- user_ids: string[];
- trace_count: number;
- trace_tags: string[];
- environment?: string;
- total_observations: number;
- duration: number;
- session_usage_details: Record<string, number>;
- session_cost_details: Record<string, number>;
- session_input_cost: string;
- session_output_cost: string;
- session_total_cost: string;
- session_input_usage: string;
- session_output_usage: string;
- session_total_usage: string;
-};
-
-export type SessionEventsField = keyof SessionEventsMetricsRow;
-
/**
* ClickHouse returns the UInt64 counts as strings.
*/
diff --git a/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts b/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts
index 895d075601..d9584b829a 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts
@@ -7,12 +7,12 @@ import {
EventsAggQueryBuilder,
EventsQueryBuilder,
EventsSessionAggregationQueryBuilder,
- type SessionEventsField,
type SessionEventsFields,
ExperimentsAggregationFieldSetName,
ExperimentsAggregationQueryBuilder,
type CTEWithSchema,
} from "./event-query-builder";
+import type { SessionEventsField } from "../../repositories/events-selects";
import { AGGREGATABLE_SCORE_TYPES } from "../../../domain/scores";
import {
OBSERVATIONS_TO_TRACE_INTERVAL,
diff --git a/packages/shared/src/server/queries/index.ts b/packages/shared/src/server/queries/index.ts
index 32a994364e..6e452fc7b4 100644
--- a/packages/shared/src/server/queries/index.ts
+++ b/packages/shared/src/server/queries/index.ts
@@ -65,18 +65,20 @@ export {
EventsSessionAggregationQueryBuilder,
EventsQueryBuilder,
ExperimentsAggregationQueryBuilder,
- OBSERVATION_FIELD_GROUP_FIELD_NAMES,
buildEventsFullTableSplitQuery,
normalizeSessionCounts,
type CTESchema,
type CTEWithSchema,
type QueryWithParams,
type ExperimentsAggregationFieldSetName,
- type SessionEventsField,
type SessionEventsFields,
- type SessionEventsMetricsRow,
type SplitQueryBuilder,
} from "./clickhouse-sql/event-query-builder";
+export { OBSERVATION_FIELD_GROUP_FIELD_NAMES } from "../repositories/events-field-sets";
+export type {
+ SessionEventsField,
+ SessionEventsMetricsRow,
+} from "../repositories/events-selects";
export {
buildEventsFilterOptionColumnQuery,
buildEventsFilterOptionsForColumnsQuery,
diff --git a/packages/shared/src/server/query-ast/README.md b/packages/shared/src/server/query-ast/README.md
index f61fef3535..7898c8521d 100644
--- a/packages/shared/src/server/query-ast/README.md
+++ b/packages/shared/src/server/query-ast/README.md
@@ -152,13 +152,14 @@ db.selectFrom("scores as s")
// … from scores as s final
```
-**Select a metadata value** (`metadataValue` — lowers `metadata[key]` to a bound subscript of `metadataMap`, so the last duplicate key wins):
+**Select a metadata value** (`metadataValue` — lowers `metadata[key]` to a subscript of `metadataMap`, so the last duplicate key wins; like `mapValue`, a string key is bound and a `literal(...)` key is inlined):
```ts
+const filterKey: string = input.metadataKey; // caller-supplied, so bound
db.selectFrom("events_core as e")
- .select((eb) => [metadataValue("e", "my_key").as("my_val")])
- .where((eb) => eb(metadataValue("e", "my_key"), "=", "x"));
-// select mapfromarrays(arrayreverse(e.metadata_names), arrayreverse(e.metadata_values))[{p:String}] as my_val …
+ .select((eb) => [metadataValue("e", literal("my_key")).as("my_val")])
+ .where((eb) => eb(metadataValue("e", filterKey), "=", "x"));
+// select mapfromarrays(arrayreverse(e.metadata_names), arrayreverse(e.metadata_values))['my_key'] as my_val …
```
## Developing the dialect
@@ -178,7 +179,7 @@ preserve it:
| ARRAY JOIN | Plugin attaches `ArrayJoinNode` as an extra field on `SelectQueryNode`. `ClickHouseOperationNodeTransformer` preserves it. `ClickHouseQueryCompiler.visitSelectQuery` emits it after JOINs / before WHERE. | no |
| LIMIT BY | Plugin attaches `LimitByNode` the same way. Compiler emits it after ORDER BY / before LIMIT. | no |
| FINAL | Plugin wraps a FROM/JOIN table expression in `FinalTableNode`. Compiler emits `final` after that expression (`scores as s final`). `$call(useFinal(["scores"]))`. Tables declared `dedup: none` throw. | no |
-| metadata subscript | Helper builds an `ArrayIndexNode` over the `metadataMap` `FunctionNode`, indexed by a bound `ValueNode` key. Transformer + compiler special-case the node. No plugin. | no |
+| metadata subscript | Helper builds an `ArrayIndexNode` over the `metadataMap` `FunctionNode`, indexed by a string key expression (a bound value or a `literal(...)`). Transformer + compiler special-case the node. No plugin. | no |
| Virtual view | Plugin rewrites `selectFrom(viewName)` into a WITH CTE. Outer types only expose the view's selected columns. | no |
These are real node objects whose children are traced Kysely
diff --git a/packages/shared/src/server/query-ast/extensions.ts b/packages/shared/src/server/query-ast/extensions.ts
index 917fdc87cc..1a12ae18fa 100644
--- a/packages/shared/src/server/query-ast/extensions.ts
+++ b/packages/shared/src/server/query-ast/extensions.ts
@@ -221,15 +221,18 @@ export function dateDiff(
);
}
-/** `map[key]` subscript with a bound key. */
+/**
+ * `map[key]` subscript. A string key is bound; pass `literal(...)` for a code
+ * constant so it is inlined.
+ */
export function mapValue<V = number>(
map: Expression<Record<string, V>>,
- key: string,
+ key: string | Expression<string>,
): ChExpression<V> {
return new ExpressionWrapper<ClickHouseDatabase, keyof ClickHouseDatabase, V>(
ArrayIndexNode.create(
map.toOperationNode(),
- ValueNode.create(key),
+ typeof key === "string" ? ValueNode.create(key) : key.toOperationNode(),
) as unknown as OperationNode,
);
}
@@ -349,6 +352,11 @@ export function ifNull<T>(
return functionCall("ifNull", [value, fallback]);
}
+/** ClickHouse `toBool(value)`. */
+export function toBool(value: Expression<SqlBool>): ChExpression<boolean> {
+ return functionCall("toBool", [value]);
+}
+
/** ClickHouse `isNull(value)`. */
export function isNull(value: Expression<unknown>): ChExpression<SqlBool> {
return functionCall("isNull", [value]);
@@ -629,12 +637,12 @@ export function traceSessionId(): ChExpression<string> {
/**
* Lower `metadata[key]` to a subscript of {@link metadataMap}, so the last
- * entry of a duplicated key wins. The key is a bound `ValueNode`, not a SQL
- * literal.
+ * entry of a duplicated key wins. Keys follow {@link mapValue}: a string is
+ * bound, a `literal(...)` is inlined.
*/
export function metadataValue(
tableAlias: string,
- key: string,
+ key: string | Expression<string>,
): ChExpression<string> {
const column = (name: string) =>
new ExpressionWrapper<ClickHouseDatabase, "events_core", string[]>(
diff --git a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap
index 1393724258..7cb7132074 100644
--- a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap
+++ b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap
@@ -497,14 +497,900 @@ FORMAT RawBLOB",
]
`;
+exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=basic 1`] = `
+[
+ {
+ "fn": "queryClickhouseStream",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.environment AS environment,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,
+ e.name AS name,
+ e.level AS level,
+ e.status_message AS status_message,
+ e.version AS version,
+ e.bookmarked AS bookmarked,
+ e.public AS public,
+ e.user_id AS user_id,
+ e.session_id AS session_id,
+ e.start_time AS start_time,
+ e.end_time AS end_time
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=core 1`] = `
+[
+ {
+ "fn": "queryClickhouseStream",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.start_time AS start_time,
+ e.end_time AS end_time
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=default 1`] = `
+[
+ {
+ "fn": "queryClickhouseStream",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.environment AS environment,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,
+ e.name AS name,
+ e.level AS level,
+ e.status_message AS status_message,
+ e.version AS version,
+ e.bookmarked AS bookmarked,
+ e.public AS public,
+ e.user_id AS user_id,
+ e.session_id AS session_id,
+ ifNull(
+ coalesce(
+ nullIf(e.trace_name, ''),
+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)
+ ),
+ ''
+ ) AS trace_name,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.completion_start_time AS completion_start_time,
+ e.created_at AS created_at,
+ e.updated_at AS updated_at,
+ e.provided_model_name AS provided_model_name,
+ e.model_parameters AS model_parameters,
+ e.usage_details AS usage_details,
+ e.cost_details AS cost_details,
+ e.total_cost AS total_cost,
+ e.prompt_id AS prompt_id,
+ e.prompt_name AS prompt_name,
+ e.prompt_version AS prompt_version,
+ e.tool_definitions AS tool_definitions,
+ e.tool_calls AS tool_calls,
+ e.tool_call_names AS tool_call_names,
+ e.usage_pricing_tier_id AS usage_pricing_tier_id,
+ e.usage_pricing_tier_name AS usage_pricing_tier_name,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,
+ e.tags AS tags,
+ e.release AS release,
+ e.model_id AS model_id,
+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time)) AS latency,
+ if(
+ isNull(e.completion_start_time),
+ NULL,
+ dateDiff('millisecond', e.start_time, e.completion_start_time)
+ ) AS time_to_first_token,
+ input,
+ output
+FROM events_full AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=empty 1`] = `
+[
+ {
+ "fn": "queryClickhouseStream",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.start_time AS start_time,
+ e.end_time AS end_time
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=io 1`] = `
+[
+ {
+ "fn": "queryClickhouseStream",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ input,
+ output
+FROM events_full AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=metadata 1`] = `
+[
+ {
+ "fn": "queryClickhouseStream",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=metrics 1`] = `
+[
+ {
+ "fn": "queryClickhouseStream",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time)) AS latency,
+ if(
+ isNull(e.completion_start_time),
+ NULL,
+ dateDiff('millisecond', e.start_time, e.completion_start_time)
+ ) AS time_to_first_token
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=model 1`] = `
+[
+ {
+ "fn": "queryClickhouseStream",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.provided_model_name AS provided_model_name,
+ e.model_parameters AS model_parameters,
+ e.model_id AS model_id
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=prompt 1`] = `
+[
+ {
+ "fn": "queryClickhouseStream",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.prompt_id AS prompt_id,
+ e.prompt_name AS prompt_name,
+ e.prompt_version AS prompt_version
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=reversed 1`] = `
+[
+ {
+ "fn": "queryClickhouseStream",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.environment AS environment,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,
+ e.name AS name,
+ e.level AS level,
+ e.status_message AS status_message,
+ e.version AS version,
+ e.bookmarked AS bookmarked,
+ e.public AS public,
+ e.user_id AS user_id,
+ e.session_id AS session_id,
+ ifNull(
+ coalesce(
+ nullIf(e.trace_name, ''),
+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)
+ ),
+ ''
+ ) AS trace_name,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.completion_start_time AS completion_start_time,
+ e.created_at AS created_at,
+ e.updated_at AS updated_at,
+ e.provided_model_name AS provided_model_name,
+ e.model_parameters AS model_parameters,
+ e.usage_details AS usage_details,
+ e.cost_details AS cost_details,
+ e.total_cost AS total_cost,
+ e.prompt_id AS prompt_id,
+ e.prompt_name AS prompt_name,
+ e.prompt_version AS prompt_version,
+ e.tool_definitions AS tool_definitions,
+ e.tool_calls AS tool_calls,
+ e.tool_call_names AS tool_call_names,
+ e.usage_pricing_tier_id AS usage_pricing_tier_id,
+ e.usage_pricing_tier_name AS usage_pricing_tier_name,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,
+ e.tags AS tags,
+ e.release AS release,
+ e.model_id AS model_id,
+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time)) AS latency,
+ if(
+ isNull(e.completion_start_time),
+ NULL,
+ dateDiff('millisecond', e.start_time, e.completion_start_time)
+ ) AS time_to_first_token,
+ input,
+ output
+FROM events_full AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=time 1`] = `
+[
+ {
+ "fn": "queryClickhouseStream",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.completion_start_time AS completion_start_time,
+ e.created_at AS created_at,
+ e.updated_at AS updated_at
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=tools 1`] = `
+[
+ {
+ "fn": "queryClickhouseStream",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.tool_definitions AS tool_definitions,
+ e.tool_calls AS tool_calls,
+ e.tool_call_names AS tool_call_names
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=trace_context 1`] = `
+[
+ {
+ "fn": "queryClickhouseStream",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ ifNull(
+ coalesce(
+ nullIf(e.trace_name, ''),
+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)
+ ),
+ ''
+ ) AS trace_name,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.tags AS tags,
+ e.release AS release
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExport fieldGroups=usage 1`] = `
+[
+ {
+ "fn": "queryClickhouseStream",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.usage_details AS usage_details,
+ e.cost_details AS cost_details,
+ e.total_cost AS total_cost,
+ e.usage_pricing_tier_id AS usage_pricing_tier_id,
+ e.usage_pricing_tier_name AS usage_pricing_tier_name
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExportParquet convertLatencyToSeconds=false 1`] = `
+[
+ {
+ "fn": "queryClickhouseExecRaw",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.environment AS environment,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,
+ e.name AS name,
+ e.level AS level,
+ e.status_message AS status_message,
+ e.version AS version,
+ e.bookmarked AS bookmarked,
+ e.public AS public,
+ e.user_id AS user_id,
+ e.session_id AS session_id,
+ ifNull(
+ coalesce(
+ nullIf(e.trace_name, ''),
+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)
+ ),
+ ''
+ ) AS trace_name,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.completion_start_time AS completion_start_time,
+ e.created_at AS created_at,
+ e.updated_at AS updated_at,
+ e.provided_model_name AS provided_model_name,
+ e.model_parameters AS model_parameters,
+ e.usage_details AS usage_details,
+ e.cost_details AS cost_details,
+ e.total_cost AS total_cost,
+ e.prompt_id AS prompt_id,
+ e.prompt_name AS prompt_name,
+ e.prompt_version AS prompt_version,
+ e.tool_definitions AS tool_definitions,
+ e.tool_calls AS tool_calls,
+ e.tool_call_names AS tool_call_names,
+ e.usage_pricing_tier_id AS usage_pricing_tier_id,
+ e.usage_pricing_tier_name AS usage_pricing_tier_name,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,
+ e.tags AS tags,
+ e.release AS release,
+ e.model_id AS model_id,
+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time)) AS latency,
+ if(
+ isNull(e.completion_start_time),
+ NULL,
+ dateDiff('millisecond', e.start_time, e.completion_start_time)
+ ) AS time_to_first_token,
+ input,
+ output
+FROM events_full AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id
+FORMAT Parquet",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExportParquet convertLatencyToSeconds=true 1`] = `
+[
+ {
+ "fn": "queryClickhouseExecRaw",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.environment AS environment,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,
+ e.name AS name,
+ e.level AS level,
+ e.status_message AS status_message,
+ e.version AS version,
+ e.bookmarked AS bookmarked,
+ e.public AS public,
+ e.user_id AS user_id,
+ e.session_id AS session_id,
+ ifNull(
+ coalesce(
+ nullIf(e.trace_name, ''),
+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)
+ ),
+ ''
+ ) AS trace_name,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.completion_start_time AS completion_start_time,
+ e.created_at AS created_at,
+ e.updated_at AS updated_at,
+ e.provided_model_name AS provided_model_name,
+ e.model_parameters AS model_parameters,
+ e.usage_details AS usage_details,
+ e.cost_details AS cost_details,
+ e.total_cost AS total_cost,
+ e.prompt_id AS prompt_id,
+ e.prompt_name AS prompt_name,
+ e.prompt_version AS prompt_version,
+ e.tool_definitions AS tool_definitions,
+ e.tool_calls AS tool_calls,
+ e.tool_call_names AS tool_call_names,
+ e.usage_pricing_tier_id AS usage_pricing_tier_id,
+ e.usage_pricing_tier_name AS usage_pricing_tier_name,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,
+ e.tags AS tags,
+ e.release AS release,
+ e.model_id AS model_id,
+ input,
+ output,
+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time) / 1000) AS latency,
+ if(
+ isNull(e.completion_start_time),
+ NULL,
+ dateDiff('millisecond', e.start_time, e.completion_start_time) / 1000
+ ) AS time_to_first_token
+FROM events_full AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id
+FORMAT Parquet",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExportRaw fieldGroups=core+io convertLatencyToSeconds=true 1`] = `
+[
+ {
+ "fn": "queryClickhouseStreamRawText",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ input,
+ output
+FROM events_full AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExportRaw fieldGroups=default convertLatencyToSeconds=true 1`] = `
+[
+ {
+ "fn": "queryClickhouseStreamRawText",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.environment AS environment,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,
+ e.name AS name,
+ e.level AS level,
+ e.status_message AS status_message,
+ e.version AS version,
+ e.bookmarked AS bookmarked,
+ e.public AS public,
+ e.user_id AS user_id,
+ e.session_id AS session_id,
+ ifNull(
+ coalesce(
+ nullIf(e.trace_name, ''),
+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)
+ ),
+ ''
+ ) AS trace_name,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.completion_start_time AS completion_start_time,
+ e.created_at AS created_at,
+ e.updated_at AS updated_at,
+ e.provided_model_name AS provided_model_name,
+ e.model_parameters AS model_parameters,
+ e.usage_details AS usage_details,
+ e.cost_details AS cost_details,
+ e.total_cost AS total_cost,
+ e.prompt_id AS prompt_id,
+ e.prompt_name AS prompt_name,
+ e.prompt_version AS prompt_version,
+ e.tool_definitions AS tool_definitions,
+ e.tool_calls AS tool_calls,
+ e.tool_call_names AS tool_call_names,
+ e.usage_pricing_tier_id AS usage_pricing_tier_id,
+ e.usage_pricing_tier_name AS usage_pricing_tier_name,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,
+ e.tags AS tags,
+ e.release AS release,
+ e.model_id AS model_id,
+ input,
+ output,
+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time) / 1000) AS latency,
+ if(
+ isNull(e.completion_start_time),
+ NULL,
+ dateDiff('millisecond', e.start_time, e.completion_start_time) / 1000
+ ) AS time_to_first_token
+FROM events_full AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExportRaw fieldGroups=default convertLatencyToSeconds=undefined 1`] = `
+[
+ {
+ "fn": "queryClickhouseStreamRawText",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.environment AS environment,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,
+ e.name AS name,
+ e.level AS level,
+ e.status_message AS status_message,
+ e.version AS version,
+ e.bookmarked AS bookmarked,
+ e.public AS public,
+ e.user_id AS user_id,
+ e.session_id AS session_id,
+ ifNull(
+ coalesce(
+ nullIf(e.trace_name, ''),
+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)
+ ),
+ ''
+ ) AS trace_name,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.completion_start_time AS completion_start_time,
+ e.created_at AS created_at,
+ e.updated_at AS updated_at,
+ e.provided_model_name AS provided_model_name,
+ e.model_parameters AS model_parameters,
+ e.usage_details AS usage_details,
+ e.cost_details AS cost_details,
+ e.total_cost AS total_cost,
+ e.prompt_id AS prompt_id,
+ e.prompt_name AS prompt_name,
+ e.prompt_version AS prompt_version,
+ e.tool_definitions AS tool_definitions,
+ e.tool_calls AS tool_calls,
+ e.tool_call_names AS tool_call_names,
+ e.usage_pricing_tier_id AS usage_pricing_tier_id,
+ e.usage_pricing_tier_name AS usage_pricing_tier_name,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,
+ e.tags AS tags,
+ e.release AS release,
+ e.model_id AS model_id,
+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time)) AS latency,
+ if(
+ isNull(e.completion_start_time),
+ NULL,
+ dateDiff('millisecond', e.start_time, e.completion_start_time)
+ ) AS time_to_first_token,
+ input,
+ output
+FROM events_full AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
+exports[`golden: events export streams family > getEventsForBlobStorageExportRaw fieldGroups=metrics convertLatencyToSeconds=true 1`] = `
+[
+ {
+ "fn": "queryClickhouseStreamRawText",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-02 00:00:00.000",
+ },
+ "route": "blob_export",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time) / 1000) AS latency,
+ if(
+ isNull(e.completion_start_time),
+ NULL,
+ dateDiff('millisecond', e.start_time, e.completion_start_time) / 1000
+ ) AS time_to_first_token
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
+LIMIT 1 BY
+ e.span_id,
+ e.project_id",
+ },
+]
+`;
+
exports[`golden: events metadata map and session resolution > getEventsForAnalyticsIntegrations 1`] = `
[
{
"fn": "queryClickhouseStream",
"params": {
"param1": "golden-project",
- "param2": "2025-12-01 00:00:00.000",
- "param3": "2026-01-01 00:00:00.000",
+ "param2": 0,
+ "param3": "2025-12-01 00:00:00.000",
+ "param4": "2026-01-01 00:00:00.000",
},
"route": "analytics_integration",
"sql": "SELECT
@@ -523,7 +1409,7 @@ exports[`golden: events metadata map and session resolution > getEventsForAnalyt
e.user_id AS user_id,
e.session_id AS session_id,
ifNull(
- COALESCE(
+ coalesce(
nullIf(e.trace_name, ''),
if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)
),
@@ -568,7 +1454,7 @@ exports[`golden: events metadata map and session resolution > getEventsForAnalyt
arrayReverse(e.metadata_values)
)['$mixpanel_session_id'] AS mixpanel_session_id
FROM events_core AS e
-WHERE (e.project_id = {param1:String}) AND (e.start_time >= {param2:DateTime64(3)}) AND (e.start_time < {param3:DateTime64(3)}) AND (e.is_deleted = 0)
+WHERE (e.project_id = {param1:String}) AND (e.is_deleted = {param2:UInt8}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time < {param4:DateTime64(6)})
LIMIT 1 BY
e.span_id,
e.project_id",
@@ -576,6 +1462,61 @@ LIMIT 1 BY
]
`;
+exports[`golden: events metadata map and session resolution > getObservationsV2FromEventsTableForPublicApi fields=basic+metrics+trace_context 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": 51,
+ },
+ "route": "unknown",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.environment AS environment,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ toBool((e.parent_span_id = '') OR (e.is_app_root = true)) AS is_root_observation,
+ e.name AS name,
+ e.level AS level,
+ e.status_message AS status_message,
+ e.version AS version,
+ e.bookmarked AS bookmarked,
+ e.public AS public,
+ e.user_id AS user_id,
+ e.session_id AS session_id,
+ ifNull(
+ coalesce(
+ nullIf(e.trace_name, ''),
+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)
+ ),
+ ''
+ ) AS trace_name,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.tags AS tags,
+ e.release AS release,
+ if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time)) AS latency,
+ if(
+ isNull(e.completion_start_time),
+ NULL,
+ dateDiff('millisecond', e.start_time, e.completion_start_time)
+ ) AS time_to_first_token
+FROM events_core AS e
+WHERE e.project_id = {param1:String}
+ORDER BY
+ e.project_id DESC,
+ toStartOfMinute(e.start_time) DESC,
+ e.start_time DESC,
+ xxHash32(e.trace_id) DESC,
+ e.span_id DESC
+LIMIT {param2:Int32}",
+ },
+]
+`;
+
exports[`golden: events metadata map and session resolution > getObservationsV2FromEventsTableForPublicApi fields=core+metadata expandMetadataKeys=set 1`] = `
[
{
@@ -720,12 +1661,10 @@ exports[`golden: events point-reads & existence family > getAgentGraphDataFromEv
{
"fn": "queryClickhouse",
"params": {
- "param1": "langgraph_node",
- "param2": "langgraph_step",
- "param3": "golden-project",
- "param4": "golden-trace",
- "param5": "2026-01-01 00:00:00.000",
- "param6": "2026-01-01 01:00:00.000",
+ "param1": "golden-project",
+ "param2": "golden-trace",
+ "param3": "2026-01-01 00:00:00.000",
+ "param4": "2026-01-01 01:00:00.000",
},
"route": "unknown",
"sql": "SELECT
@@ -735,10 +1674,10 @@ exports[`golden: events point-reads & existence family > getAgentGraphDataFromEv
e.name AS name,
e.start_time AS start_time,
e.end_time AS end_time,
- mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))[{param1:String}] AS node,
- mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))[{param2:String}] AS step
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['langgraph_node'] AS node,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['langgraph_step'] AS step
FROM events_core AS e
-WHERE (e.project_id = {param3:String}) AND (e.trace_id = {param4:String}) AND (e.start_time >= {param5:DateTime64(6)}) AND (e.start_time <= {param6:DateTime64(6)})",
+WHERE (e.project_id = {param1:String}) AND (e.trace_id = {param2:String}) AND (e.start_time >= {param3:DateTime64(6)}) AND (e.start_time <= {param4:DateTime64(6)})",
},
]
`;
@@ -1640,8 +2579,7 @@ exports[`golden: events simple aggregates family > getObservationMetricsForPromp
"param3": [],
"param4": "input",
"param5": "output",
- "param6": "total",
- "param7": 0,
+ "param6": 0,
},
"route": "unknown",
"sql": "WITH prompt_events AS
@@ -1676,10 +2614,10 @@ SELECT
medianExact(
arraySum(mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param5:String}) > 0), e.usage_details)))
) AS median_output_usage,
- medianExact(e.cost_details[{param6:String}]) AS median_total_cost,
+ medianExact(e.cost_details['total']) AS median_total_cost,
medianExact(dateDiff('millisecond', e.start_time, e.end_time)) AS median_latency_ms
FROM prompt_events AS e
-WHERE e.is_deleted = {param7:UInt8}
+WHERE e.is_deleted = {param6:UInt8}
GROUP BY
e.prompt_id,
e.prompt_version
@@ -1702,8 +2640,7 @@ exports[`golden: events simple aggregates family > getObservationMetricsForPromp
"param4": "2025-12-01 00:00:00.000",
"param5": "input",
"param6": "output",
- "param7": "total",
- "param8": 0,
+ "param7": 0,
},
"route": "unknown",
"sql": "WITH prompt_events AS
@@ -1738,10 +2675,10 @@ SELECT
medianExact(
arraySum(mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param6:String}) > 0), e.usage_details)))
) AS median_output_usage,
- medianExact(e.cost_details[{param7:String}]) AS median_total_cost,
+ medianExact(e.cost_details['total']) AS median_total_cost,
medianExact(dateDiff('millisecond', e.start_time, e.end_time)) AS median_latency_ms
FROM prompt_events AS e
-WHERE e.is_deleted = {param8:UInt8}
+WHERE e.is_deleted = {param7:UInt8}
GROUP BY
e.prompt_id,
e.prompt_version
@@ -1765,8 +2702,7 @@ exports[`golden: events simple aggregates family > getObservationMetricsForPromp
"param5": "2026-01-02 00:00:00.000",
"param6": "input",
"param7": "output",
- "param8": "total",
- "param9": 0,
+ "param8": 0,
},
"route": "unknown",
"sql": "WITH prompt_events AS
@@ -1801,10 +2737,10 @@ SELECT
medianExact(
arraySum(mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param7:String}) > 0), e.usage_details)))
) AS median_output_usage,
- medianExact(e.cost_details[{param8:String}]) AS median_total_cost,
+ medianExact(e.cost_details['total']) AS median_total_cost,
medianExact(dateDiff('millisecond', e.start_time, e.end_time)) AS median_latency_ms
FROM prompt_events AS e
-WHERE e.is_deleted = {param9:UInt8}
+WHERE e.is_deleted = {param8:UInt8}
GROUP BY
e.prompt_id,
e.prompt_version
@@ -1827,8 +2763,7 @@ exports[`golden: events simple aggregates family > getObservationMetricsForPromp
"param4": "2026-01-02 00:00:00.000",
"param5": "input",
"param6": "output",
- "param7": "total",
- "param8": 0,
+ "param7": 0,
},
"route": "unknown",
"sql": "WITH prompt_events AS
@@ -1863,10 +2798,10 @@ SELECT
medianExact(
arraySum(mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param6:String}) > 0), e.usage_details)))
) AS median_output_usage,
- medianExact(e.cost_details[{param7:String}]) AS median_total_cost,
+ medianExact(e.cost_details['total']) AS median_total_cost,
medianExact(dateDiff('millisecond', e.start_time, e.end_time)) AS median_latency_ms
FROM prompt_events AS e
-WHERE e.is_deleted = {param8:UInt8}
+WHERE e.is_deleted = {param7:UInt8}
GROUP BY
e.prompt_id,
e.prompt_version
@@ -1888,8 +2823,7 @@ exports[`golden: events simple aggregates family > getObservationMetricsForPromp
],
"param4": "input",
"param5": "output",
- "param6": "total",
- "param7": 0,
+ "param6": 0,
},
"route": "unknown",
"sql": "WITH prompt_events AS
@@ -1924,10 +2858,10 @@ SELECT
medianExact(
arraySum(mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param5:String}) > 0), e.usage_details)))
) AS median_output_usage,
- medianExact(e.cost_details[{param6:String}]) AS median_total_cost,
+ medianExact(e.cost_details['total']) AS median_total_cost,
medianExact(dateDiff('millisecond', e.start_time, e.end_time)) AS median_latency_ms
FROM prompt_events AS e
-WHERE e.is_deleted = {param7:UInt8}
+WHERE e.is_deleted = {param6:UInt8}
GROUP BY
e.prompt_id,
e.prompt_version
@@ -2064,14 +2998,13 @@ exports[`golden: events simple aggregates family > getSessionMetricsFromEvents f
"params": {
"param1": "input",
"param2": "output",
- "param3": "total",
- "param4": "golden-project",
- "param5": [
+ "param3": "golden-project",
+ "param4": [
"golden-session",
"golden-session-b",
],
- "param6": "",
- "param7": 2,
+ "param5": "",
+ "param6": 2,
},
"route": "unknown",
"sql": "SELECT
@@ -2093,18 +3026,18 @@ exports[`golden: events simple aggregates family > getSessionMetricsFromEvents f
arraySum(
mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param2:String}) > 0), sumMap(cost_details)))
) AS session_output_cost,
- sumMap(cost_details)[{param3:String}] AS session_total_cost,
+ sumMap(cost_details)['total'] AS session_total_cost,
arraySum(
mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param1:String}) > 0), sumMap(usage_details)))
) AS session_input_usage,
arraySum(
mapValues(mapFilter(x -> (positionCaseInsensitive(x.1, {param2:String}) > 0), sumMap(usage_details)))
) AS session_output_usage,
- sumMap(usage_details)[{param3:String}] AS session_total_usage
+ sumMap(usage_details)['total'] AS session_total_usage
FROM events_core AS e
-WHERE (e.project_id = {param4:String}) AND (session_id IN ({param5:Array(String)})) AND (session_id != {param6:String})
+WHERE (e.project_id = {param3:String}) AND (session_id IN ({param4:Array(String)})) AND (session_id != {param5:String})
GROUP BY session_id
-LIMIT {param7:Int64}",
+LIMIT {param6:Int64}",
},
]
`;
diff --git a/packages/shared/src/server/repositories/blobStorageExportWindow.test.ts b/packages/shared/src/server/repositories/blobStorageExportWindow.test.ts
index 2411ae67a7..4cb29be73b 100644
--- a/packages/shared/src/server/repositories/blobStorageExportWindow.test.ts
+++ b/packages/shared/src/server/repositories/blobStorageExportWindow.test.ts
@@ -1,7 +1,9 @@
import { describe, it, expect, beforeEach, vi } from "vitest";
+type CapturedCall = { query: string; params: Record<string, unknown> };
+
const mockQueryClickhouseStream = vi.hoisted(() =>
- vi.fn(() => (async function* () {})()),
+ vi.fn((_call: CapturedCall) => (async function* () {})()),
);
vi.mock("./clickhouse", () => ({
@@ -24,20 +26,37 @@ import { getTracesForBlobStorageExport } from "./traces";
import { getObservationsForBlobStorageExport } from "./observations";
import { getScoresForBlobStorageExport } from "./scores";
import { getEventsForBlobStorageExport } from "./events";
+import { convertDateToClickhouseDateTime } from "../clickhouse/client";
const PROJECT = "proj-1";
const MIN = new Date("2026-01-01T00:00:00.000Z");
const MAX = new Date("2026-01-01T01:00:00.000Z");
-const capturedQuery = (): string => {
+const capturedCall = (): CapturedCall => {
expect(mockQueryClickhouseStream).toHaveBeenCalledTimes(1);
- const call = mockQueryClickhouseStream.mock.calls[0] as unknown as [
- { query: string },
- ];
- return call[0].query;
+ return mockQueryClickhouseStream.mock.calls[0][0];
+};
+
+const capturedQuery = (): string => capturedCall().query;
+
+/** The value bound right after `predicate` (e.g. `e.start_time >= `). */
+const boundValue = (predicate: string): unknown => {
+ const { query, params } = capturedCall();
+ const name = query.split(predicate + "{")[1]?.split(":")[0];
+ return name === undefined ? undefined : params[name];
+};
+
+const namedParamBounds = {
+ lower: ">= {minTimestamp",
+ upper: "< {maxTimestamp",
+ inclusiveUpper: "<= {maxTimestamp",
};
-const builders = [
+const builders: Array<{
+ name: string;
+ run: () => Promise<void>;
+ bounds?: typeof namedParamBounds;
+}> = [
{
name: "getTracesForBlobStorageExport",
run: async () => {
@@ -62,6 +81,12 @@ const builders = [
run: async () => {
getEventsForBlobStorageExport(PROJECT, MIN, MAX);
},
+ // Query-AST binds are named positionally.
+ bounds: {
+ lower: "e.start_time >= {",
+ upper: "e.start_time < {",
+ inclusiveUpper: "e.start_time <= {",
+ },
},
];
@@ -72,16 +97,27 @@ describe("blob-storage export window is half-open", () => {
it.each(builders)(
"$name uses an exclusive upper bound and inclusive lower bound",
- async ({ run }) => {
+ async ({ run, bounds = namedParamBounds }) => {
await run();
const query = capturedQuery();
- expect(query).toContain(">= {minTimestamp");
- expect(query).toContain("< {maxTimestamp");
- expect(query).not.toContain("<= {maxTimestamp");
+ expect(query).toContain(bounds.lower);
+ expect(query).toContain(bounds.upper);
+ expect(query).not.toContain(bounds.inclusiveUpper);
},
);
+ it("getEventsForBlobStorageExport binds min to the lower and max to the upper bound", async () => {
+ getEventsForBlobStorageExport(PROJECT, MIN, MAX);
+
+ expect(boundValue("e.start_time >= ")).toBe(
+ convertDateToClickhouseDateTime(MIN),
+ );
+ expect(boundValue("e.start_time < ")).toBe(
+ convertDateToClickhouseDateTime(MAX),
+ );
+ });
+
it("boundary row belongs to the next window only (traces)", async () => {
// Window 1 [MIN, MAX) excludes a row at exactly MAX...
getTracesForBlobStorageExport(PROJECT, MIN, MAX);
diff --git a/packages/shared/src/server/repositories/events-field-sets.ts b/packages/shared/src/server/repositories/events-field-sets.ts
new file mode 100644
index 0000000000..141445c95d
--- /dev/null
+++ b/packages/shared/src/server/repositories/events-field-sets.ts
@@ -0,0 +1,369 @@
+import {
+ OBSERVATION_FIELD_GROUPS_PUBLIC_API,
+ type ObservationFieldGroupPublicApi,
+} from "../../domain/observation-field-groups";
+
+/**
+ * Every events row field, in canonical SELECT order. Both the raw-SQL
+ * `EventsQueryBuilder` and the query-AST selects order their columns by it.
+ */
+const EVENTS_FIELD_ORDER = [
+ "count",
+ "uniqueTraceCount",
+ "id",
+ "traceId",
+ "projectId",
+ "environment",
+ "type",
+ "parentObservationId",
+ "isRootObservation",
+ "isAppRoot",
+ "name",
+ "level",
+ "statusMessage",
+ "version",
+ "bookmarked",
+ "public",
+ "userId",
+ "sessionId",
+ "traceName",
+ "startTime",
+ "endTime",
+ "completionStartTime",
+ "createdAt",
+ "updatedAt",
+ "eventTs",
+ "providedModelName",
+ "internalModelId",
+ "modelParameters",
+ "providedUsageDetails",
+ "usageDetails",
+ "providedCostDetails",
+ "costDetails",
+ "totalCost",
+ "promptId",
+ "promptName",
+ "promptVersion",
+ "skillsAvailable",
+ "skillsResourceLoaded",
+ "toolDefinitions",
+ "toolCalls",
+ "toolCallNames",
+ "usagePricingTierId",
+ "usagePricingTierName",
+ "input",
+ "output",
+ "metadata",
+ "tags",
+ "release",
+ "modelId",
+ "experimentId",
+ "experimentName",
+ "experimentDescription",
+ "experimentDatasetId",
+ "experimentMetadata",
+ "experimentItemId",
+ "experimentItemRootSpanId",
+ "experimentItemExpectedOutput",
+ "experimentItemMetadata",
+ "experimentItemVersion",
+ "latency",
+ "timeToFirstToken",
+] as const;
+
+export type EventsFieldKey = (typeof EVENTS_FIELD_ORDER)[number];
+
+const EVENTS_FIELD_ORDER_INDEX = new Map<string, number>(
+ EVENTS_FIELD_ORDER.map((key, index) => [key, index]),
+);
+
+/**
+ * Predefined field sets for common query patterns
+ * Maps set names to arrays of field keys from EVENTS_FIELD_ORDER
+ */
+export const FIELD_SETS = {
+ // Aggregates
+ count: ["count"],
+ countWithUniqueTraces: ["count", "uniqueTraceCount"],
+
+ // List query field sets (for getObservationsWithModelDataFromEventsTable)
+ base: [
+ "id",
+ "type",
+ "projectId",
+ "name",
+ "modelParameters",
+ "startTime",
+ "endTime",
+ "traceId",
+ "completionStartTime",
+ "providedUsageDetails",
+ "usageDetails",
+ "providedCostDetails",
+ "costDetails",
+ "level",
+ "environment",
+ "bookmarked",
+ "public",
+ "statusMessage",
+ "version",
+ "parentObservationId",
+ "createdAt",
+ "updatedAt",
+ "providedModelName",
+ "totalCost",
+ "usagePricingTierId",
+ "usagePricingTierName",
+ "promptId",
+ "promptName",
+ "promptVersion",
+ "internalModelId",
+ "userId",
+ "sessionId",
+ "traceName",
+ "release",
+ "tags",
+ "toolDefinitions",
+ "toolCalls",
+ "toolCallNames",
+ ],
+ baseWithoutTools: [
+ "id",
+ "type",
+ "projectId",
+ "name",
+ "modelParameters",
+ "startTime",
+ "endTime",
+ "traceId",
+ "completionStartTime",
+ "providedUsageDetails",
+ "usageDetails",
+ "providedCostDetails",
+ "costDetails",
+ "level",
+ "environment",
+ "bookmarked",
+ "public",
+ "statusMessage",
+ "version",
+ "parentObservationId",
+ "createdAt",
+ "updatedAt",
+ "providedModelName",
+ "totalCost",
+ "usagePricingTierId",
+ "usagePricingTierName",
+ "promptId",
+ "promptName",
+ "promptVersion",
+ "internalModelId",
+ "userId",
+ "sessionId",
+ "traceName",
+ "release",
+ "tags",
+ // Keep lightweight tool names available even when heavy tool payloads are omitted.
+ "toolCallNames",
+ ],
+ calculated: ["latency", "timeToFirstToken"],
+ io: ["input", "output"],
+ metadata: ["metadata"],
+ tools: ["toolDefinitions", "toolCalls", "toolCallNames"],
+ trace_context: ["tags", "release", "traceName"],
+ model_export: ["providedModelName", "modelId", "modelParameters"],
+ eventTs: ["eventTs"],
+ publicApiExperimentItemCore: [
+ "id",
+ "projectId",
+ "traceId",
+ "startTime",
+ "endTime",
+ "level",
+ "environment",
+ "experimentId",
+ "experimentName",
+ "experimentItemId",
+ ],
+ publicApiExperimentItemDataset: [
+ "experimentDatasetId",
+ "experimentItemVersion",
+ ],
+ publicApiExperimentItemExpectedOutput: ["experimentItemExpectedOutput"],
+ publicApiExperimentItemMetadataFields: ["experimentItemMetadata"],
+ publicApiExperimentItemExperimentMetadata: [
+ "experimentMetadata",
+ "experimentDescription",
+ ],
+ // Public API v2 field sets (field groups for selective fetching)
+ core: [
+ "id",
+ "traceId",
+ "startTime",
+ "endTime",
+ "projectId",
+ "parentObservationId",
+ "type",
+ ],
+ basic: [
+ "name",
+ "level",
+ "statusMessage",
+ "version",
+ "environment",
+ "bookmarked",
+ "public",
+ "userId",
+ "sessionId",
+ "isRootObservation",
+ ],
+ time: ["completionStartTime", "createdAt", "updatedAt"],
+ model: ["providedModelName", "internalModelId", "modelParameters"],
+ usage: [
+ "usageDetails",
+ "costDetails",
+ "totalCost",
+ "usagePricingTierId",
+ "usagePricingTierName",
+ ],
+ prompt: ["promptId", "promptName", "promptVersion"],
+ metrics: ["latency", "timeToFirstToken"],
+
+ // Batch export field set (all fields needed for CSV/JSON exports)
+ export: [
+ "id",
+ "traceId",
+ "projectId",
+ "startTime",
+ "endTime",
+ "name",
+ "type",
+ "environment",
+ "version",
+ "userId",
+ "sessionId",
+ "level",
+ "statusMessage",
+ "promptName",
+ "promptId",
+ "promptVersion",
+ "modelId",
+ "providedModelName",
+ "modelParameters",
+ "usageDetails",
+ "costDetails",
+ "totalCost",
+ "completionStartTime",
+ "latency",
+ "timeToFirstToken",
+ "tags",
+ "release",
+ "traceName",
+ "parentObservationId",
+ "bookmarked",
+ "public",
+ "createdAt",
+ "updatedAt",
+ "toolDefinitions",
+ "toolCalls",
+ "toolCallNames",
+ "usagePricingTierId",
+ "usagePricingTierName",
+ ],
+
+ eval: [
+ "id",
+ "traceId",
+ "projectId",
+ "parentObservationId",
+ "isAppRoot",
+ "type",
+ "name",
+ "environment",
+ "version",
+ "level",
+ "statusMessage",
+ "traceName",
+ "userId",
+ "sessionId",
+ "tags",
+ "release",
+ "providedModelName",
+ "modelParameters",
+ "promptId",
+ "promptName",
+ "promptVersion",
+ "providedUsageDetails",
+ "usageDetails",
+ "providedCostDetails",
+ "costDetails",
+ "toolDefinitions",
+ "toolCalls",
+ "toolCallNames",
+ "experimentId",
+ "experimentName",
+ "experimentItemRootSpanId",
+ "experimentItemExpectedOutput",
+ "experimentItemMetadata",
+ ],
+
+ // Experiment items field set
+ experimentItems: [
+ "id", // span_id (observation_id)
+ "traceId",
+ "input",
+ "output",
+ "startTime",
+ "level",
+
+ // Experiment metadata
+ "experimentId",
+ "experimentName",
+ "experimentDatasetId",
+
+ // Experiment item metadata
+ "experimentItemId",
+ "experimentItemRootSpanId",
+ "experimentItemVersion",
+ "experimentItemExpectedOutput",
+ "experimentItemMetadata",
+
+ // Event metadata
+ "metadata",
+ ],
+} as const satisfies Record<string, readonly EventsFieldKey[]>;
+
+export type FieldSetName = keyof typeof FIELD_SETS;
+
+// Canonicalize by EVENTS_FIELD_ORDER so SELECT column order does
+// not follow caller field-set order. Clustered ClickHouse reads align result
+// blocks by position; incompatible Map types (cost_details vs metadata) 500
+// when those columns swap places.
+export function orderedFieldKeys<K extends string>(
+ fieldKeys: readonly K[],
+): K[] {
+ return [...new Set(fieldKeys)].sort(
+ (a, b) =>
+ (EVENTS_FIELD_ORDER_INDEX.get(a) ?? Number.MAX_SAFE_INTEGER) -
+ (EVENTS_FIELD_ORDER_INDEX.get(b) ?? Number.MAX_SAFE_INTEGER),
+ );
+}
+
+export type FieldSetFieldKey<S extends FieldSetName> =
+ (typeof FIELD_SETS)[S][number];
+
+/** The fields of `sets`, deduplicated, in the canonical SELECT order. */
+export function eventsFieldSetKeys<S extends FieldSetName>(
+ ...sets: S[]
+): FieldSetFieldKey<S>[] {
+ return orderedFieldKeys(sets.flatMap((s) => FIELD_SETS[s]));
+}
+
+export const OBSERVATION_FIELD_GROUP_FIELD_NAMES = Object.fromEntries(
+ OBSERVATION_FIELD_GROUPS_PUBLIC_API.map((group) => [
+ group,
+ FIELD_SETS[group],
+ ]),
+) as {
+ [K in ObservationFieldGroupPublicApi]: (typeof FIELD_SETS)[K];
+};
diff --git a/packages/shared/src/server/repositories/events-selects.ts b/packages/shared/src/server/repositories/events-selects.ts
index ea9354a29f..96c2b52594 100644
--- a/packages/shared/src/server/repositories/events-selects.ts
+++ b/packages/shared/src/server/repositories/events-selects.ts
@@ -29,12 +29,37 @@ import {
nullIf,
sumMap,
sumMapValuesWhereKeyContains,
+ toBool,
traceSessionId,
uniq,
uniqIf,
} from "../query-ast/extensions";
import type { ClickHouseDatabase } from "../query-ast/schema";
-import type { SessionEventsField } from "../queries/clickhouse-sql/event-query-builder";
+import type { FieldSetFieldKey } from "./events-field-sets";
+
+/** One row of the session aggregation (`SESSION_AGGREGATION_SELECTS`). */
+export type SessionEventsMetricsRow = {
+ session_id: string;
+ max_timestamp: string;
+ min_timestamp: string;
+ trace_ids: string[];
+ user_ids: string[];
+ trace_count: number;
+ trace_tags: string[];
+ environment?: string;
+ total_observations: number;
+ duration: number;
+ session_usage_details: Record<string, number>;
+ session_cost_details: Record<string, number>;
+ session_input_cost: string;
+ session_output_cost: string;
+ session_total_cost: string;
+ session_input_usage: string;
+ session_output_usage: string;
+ session_total_usage: string;
+};
+
+export type SessionEventsField = keyof SessionEventsMetricsRow;
type EventsDb = { e: ClickHouseDatabase["events_core"] };
type EventsRow = ClickHouseDatabase["events_core"];
@@ -51,7 +76,8 @@ const eventsMetadataMap = metadataMap(
eb.ref("e.metadata_values"),
);
-const isRootObservation = eb.or([
+/** The event is a root observation: no parent span, or an app root. */
+export const eventsIsRootObservation = eb.or([
eb("e.parent_span_id", "=", literal("")),
eb("e.is_app_root", "=", literal(true)),
]);
@@ -63,12 +89,22 @@ const isRootObservation = eb.or([
export const eventsTraceName = eb.fn.coalesce(
nullIf(eb.ref("e.trace_name"), literal("")),
ifElse(
- isRootObservation,
+ eventsIsRootObservation,
nullIf(eb.ref("e.name"), literal("")),
literal(null),
),
);
+/**
+ * {@link eventsTraceName} as a non-null String, '' when the event has no trace
+ * name. Projected under `trace_name`, the nullable form would clash with the
+ * physical String column of the same name once a query also filters and sorts
+ * on it (ClickHouse AMBIGUOUS_COLUMN_NAME).
+ */
+export const eventsTraceNameOrEmpty = ifNull(eventsTraceName, literal(""));
+
+const traceNameSelect = eventsTraceNameOrEmpty.as("trace_name");
+
/** The latest non-empty value of `column` across a trace's events. */
const latestNonEmpty = (column: StringColumn) =>
argMaxIf(eb.ref(column), eb.ref("event_ts"), eb(column, "<>", literal("")));
@@ -104,7 +140,7 @@ export const TRACE_AGGREGATION_SELECTS = [
argMaxIf(
eb.ref("e.name"),
eb.ref("e.event_ts"),
- eb.and([isRootObservation, eb("e.name", "<>", literal(""))]),
+ eb.and([eventsIsRootObservation, eb("e.name", "<>", literal(""))]),
),
literal(""),
),
@@ -222,9 +258,10 @@ export const SESSION_AGGREGATION_SELECTS = {
sumMap(eb.ref("cost_details")),
"output",
).as("session_output_cost"),
- session_total_cost: mapValue(sumMap(eb.ref("cost_details")), "total").as(
- "session_total_cost",
- ),
+ session_total_cost: mapValue(
+ sumMap(eb.ref("cost_details")),
+ literal("total"),
+ ).as("session_total_cost"),
session_input_usage: sumMapValuesWhereKeyContains(
sumMap(eb.ref("usage_details")),
"input",
@@ -233,9 +270,10 @@ export const SESSION_AGGREGATION_SELECTS = {
sumMap(eb.ref("usage_details")),
"output",
).as("session_output_usage"),
- session_total_usage: mapValue(sumMap(eb.ref("usage_details")), "total").as(
- "session_total_usage",
- ),
+ session_total_usage: mapValue(
+ sumMap(eb.ref("usage_details")),
+ literal("total"),
+ ).as("session_total_usage"),
} satisfies Record<SessionEventsField, EventsSelection>;
/** One observation's fields, without input/output. */
@@ -254,8 +292,7 @@ export const OBSERVATION_BY_ID_SELECTS = [
"e.public as public",
"e.user_id as user_id",
"e.session_id as session_id",
- // String-typed on purpose: '' when the event has no trace name.
- ifNull(eventsTraceName, literal("")).as("trace_name"),
+ traceNameSelect,
"e.start_time as start_time",
"e.end_time as end_time",
"e.completion_start_time as completion_start_time",
@@ -286,3 +323,94 @@ export const OBSERVATION_BY_ID_SELECTS = [
"e.tags as tags",
"e.release as release",
] satisfies EventsSelection[];
+
+/** Milliseconds from an event's start to `end`, NULL while `end` is unset. */
+export const millisecondsSinceStart = (
+ end: "e.end_time" | "e.completion_start_time",
+) =>
+ ifElse(
+ isNull(eb.ref(end)),
+ literal(null),
+ dateDiff("millisecond", eb.ref("e.start_time"), eb.ref(end)),
+ );
+
+/** {@link millisecondsSinceStart} in seconds. */
+const secondsSinceStart = (end: "e.end_time" | "e.completion_start_time") =>
+ ifElse(
+ isNull(eb.ref(end)),
+ literal(null),
+ eb(
+ dateDiff("millisecond", eb.ref("e.start_time"), eb.ref(end)),
+ "/",
+ literal(1000),
+ ),
+ );
+
+type RowFieldSet =
+ | "core"
+ | "basic"
+ | "time"
+ | "metadata"
+ | "model_export"
+ | "usage"
+ | "prompt"
+ | "metrics"
+ | "trace_context"
+ | "tools"
+ | "export";
+
+/**
+ * Row-level event fields, keyed like the legacy builder's field sets. Select
+ * them in `eventsFieldSetKeys` order: clustered reads align result blocks by
+ * column position.
+ */
+export const EVENTS_ROW_FIELD_SELECTS = {
+ id: "e.span_id as id",
+ traceId: "e.trace_id as trace_id",
+ projectId: "e.project_id as project_id",
+ environment: "e.environment as environment",
+ type: "e.type as type",
+ parentObservationId: "e.parent_span_id as parent_observation_id",
+ isRootObservation: toBool(eventsIsRootObservation).as("is_root_observation"),
+ name: "e.name as name",
+ level: "e.level as level",
+ statusMessage: "e.status_message as status_message",
+ version: "e.version as version",
+ bookmarked: "e.bookmarked as bookmarked",
+ public: "e.public as public",
+ userId: "e.user_id as user_id",
+ sessionId: "e.session_id as session_id",
+ traceName: traceNameSelect,
+ startTime: "e.start_time as start_time",
+ endTime: "e.end_time as end_time",
+ completionStartTime: "e.completion_start_time as completion_start_time",
+ createdAt: "e.created_at as created_at",
+ updatedAt: "e.updated_at as updated_at",
+ providedModelName: "e.provided_model_name as provided_model_name",
+ modelParameters: "e.model_parameters as model_parameters",
+ usageDetails: "e.usage_details as usage_details",
+ costDetails: "e.cost_details as cost_details",
+ totalCost: "e.total_cost as total_cost",
+ promptId: "e.prompt_id as prompt_id",
+ promptName: "e.prompt_name as prompt_name",
+ promptVersion: "e.prompt_version as prompt_version",
+ toolDefinitions: "e.tool_definitions as tool_definitions",
+ toolCalls: "e.tool_calls as tool_calls",
+ toolCallNames: "e.tool_call_names as tool_call_names",
+ usagePricingTierId: "e.usage_pricing_tier_id as usage_pricing_tier_id",
+ usagePricingTierName: "e.usage_pricing_tier_name as usage_pricing_tier_name",
+ metadata: eventsMetadataMap.as("metadata"),
+ tags: "e.tags as tags",
+ release: "e.release as release",
+ modelId: "e.model_id as model_id",
+ latency: millisecondsSinceStart("e.end_time").as("latency"),
+ timeToFirstToken: millisecondsSinceStart("e.completion_start_time").as(
+ "time_to_first_token",
+ ),
+} satisfies Record<FieldSetFieldKey<RowFieldSet>, EventsSelection>;
+
+/** The `metrics` field set with latencies in seconds instead of milliseconds. */
+export const EVENTS_METRICS_SECONDS_SELECTS = [
+ secondsSinceStart("e.end_time").as("latency"),
+ secondsSinceStart("e.completion_start_time").as("time_to_first_token"),
+] satisfies EventsSelection[];
diff --git a/packages/shared/src/server/repositories/events.golden.test.ts b/packages/shared/src/server/repositories/events.golden.test.ts
index c83836402e..aafc467c71 100644
--- a/packages/shared/src/server/repositories/events.golden.test.ts
+++ b/packages/shared/src/server/repositories/events.golden.test.ts
@@ -8,6 +8,7 @@ import {
vi,
} from "vitest";
+import { OBSERVATION_FIELD_GROUPS_FULL } from "../../domain/observation-field-groups";
import { env } from "../../env";
import { LangfuseNotFoundError } from "../../errors";
import {
@@ -29,6 +30,9 @@ vi.mock("./clickhouse", async (importOriginal) => {
import {
getAgentGraphDataFromEventsTable,
getEventsForAnalyticsIntegrations,
+ getEventsForBlobStorageExport,
+ getEventsForBlobStorageExportParquet,
+ getEventsForBlobStorageExportRaw,
getLastTraceTimestampsByProjectsFromEventsTable,
getLatestSdkVersionInfoFromEvents,
getObservationByIdFromEventsTable,
@@ -490,6 +494,16 @@ describeWithClickhouse(
expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
});
+ it("getObservationsV2FromEventsTableForPublicApi fields=basic+metrics+trace_context", async () => {
+ await getObservationsV2FromEventsTableForPublicApi({
+ projectId: FIXED_PROJECT_ID,
+ page: 1,
+ limit: 50,
+ fields: ["basic", "metrics", "trace_context"],
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
it("getEventsForAnalyticsIntegrations", async () => {
for await (const _ of getEventsForAnalyticsIntegrations(
FIXED_PROJECT_ID,
@@ -519,3 +533,101 @@ describeWithClickhouse(
});
},
);
+
+describeWithClickhouse("golden: events export streams family", () => {
+ beforeEach(() => {
+ resetCaptures();
+ });
+
+ // The harness only records a stream's query once it is iterated.
+ async function drain(rows: AsyncIterable<unknown>) {
+ for await (const _ of rows) {
+ // The harness streams no rows.
+ }
+ }
+
+ it("getEventsForBlobStorageExport fieldGroups=default", async () => {
+ await drain(
+ getEventsForBlobStorageExport(
+ FIXED_PROJECT_ID,
+ FIXED_FROM_TIMESTAMP,
+ FIXED_TO_TIMESTAMP,
+ ),
+ );
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it.each(OBSERVATION_FIELD_GROUPS_FULL)(
+ "getEventsForBlobStorageExport fieldGroups=%s",
+ async (group) => {
+ await drain(
+ getEventsForBlobStorageExport(
+ FIXED_PROJECT_ID,
+ FIXED_FROM_TIMESTAMP,
+ FIXED_TO_TIMESTAMP,
+ [group],
+ ),
+ );
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ },
+ );
+
+ it("getEventsForBlobStorageExport fieldGroups=empty", async () => {
+ await drain(
+ getEventsForBlobStorageExport(
+ FIXED_PROJECT_ID,
+ FIXED_FROM_TIMESTAMP,
+ FIXED_TO_TIMESTAMP,
+ [],
+ ),
+ );
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getEventsForBlobStorageExport fieldGroups=reversed", async () => {
+ await drain(
+ getEventsForBlobStorageExport(
+ FIXED_PROJECT_ID,
+ FIXED_FROM_TIMESTAMP,
+ FIXED_TO_TIMESTAMP,
+ [...OBSERVATION_FIELD_GROUPS_FULL].reverse(),
+ ),
+ );
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it.each([
+ ["default", undefined, undefined],
+ ["default", true, [...OBSERVATION_FIELD_GROUPS_FULL]],
+ ["metrics", true, ["metrics"]],
+ ["core+io", true, ["core", "io"]],
+ ] as const)(
+ "getEventsForBlobStorageExportRaw fieldGroups=%s convertLatencyToSeconds=%s",
+ async (_name, convertLatencyToSeconds, fieldGroups) => {
+ await drain(
+ getEventsForBlobStorageExportRaw(
+ FIXED_PROJECT_ID,
+ FIXED_FROM_TIMESTAMP,
+ FIXED_TO_TIMESTAMP,
+ fieldGroups ? [...fieldGroups] : undefined,
+ convertLatencyToSeconds,
+ ),
+ );
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ },
+ );
+
+ it.each([false, true])(
+ "getEventsForBlobStorageExportParquet convertLatencyToSeconds=%s",
+ async (convertLatencyToSeconds) => {
+ await getEventsForBlobStorageExportParquet(
+ FIXED_PROJECT_ID,
+ FIXED_FROM_TIMESTAMP,
+ FIXED_TO_TIMESTAMP,
+ undefined,
+ convertLatencyToSeconds,
+ );
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ },
+ );
+});
diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.ts
index ae3974d132..f1ad3b54a2 100644
--- a/packages/shared/src/server/repositories/events.ts
+++ b/packages/shared/src/server/repositories/events.ts
@@ -44,6 +44,7 @@ import {
mapValue,
medianExact,
metadataMap,
+ metadataValue,
sumMapValuesWhereKeyContains,
toDate,
toJSONString,
@@ -135,10 +136,14 @@ import {
} from "./observations";
import { convertEventsObservation } from "./observations_converters";
import {
+ EVENTS_METRICS_SECONDS_SELECTS,
+ EVENTS_ROW_FIELD_SELECTS,
eventsTraceName,
OBSERVATION_BY_ID_SELECTS,
SESSION_AGGREGATION_SELECTS,
TRACE_AGGREGATION_SELECTS,
+ type SessionEventsField,
+ type SessionEventsMetricsRow,
} from "./events-selects";
import {
EventsQueryBuilder,
@@ -147,12 +152,10 @@ import {
buildEventsFullTableSplitQuery,
type QueryWithParams,
normalizeSessionCounts,
- type SessionEventsField,
type SessionEventsFields,
- type SessionEventsMetricsRow,
OrderByEntry,
- EVENTS_METADATA_MAP_SQL,
} from "../queries/clickhouse-sql/event-query-builder";
+import { eventsFieldSetKeys } from "./events-field-sets";
import {
buildEventsFilterOptionColumnQuery,
buildEventsFilterOptionsForColumnsQuery,
@@ -683,7 +686,7 @@ export const getObservationMetricsForPromptsFromEvents = async (
medianExact(
sumMapValuesWhereKeyContains(eb.ref("e.usage_details"), "output"),
).as("median_output_usage"),
- medianExact(mapValue(eb.ref("e.cost_details"), "total")).as(
+ medianExact(mapValue(eb.ref("e.cost_details"), literal("total"))).as(
"median_total_cost",
),
medianExact(
@@ -2639,8 +2642,8 @@ export async function getAgentGraphDataFromEventsTable(params: {
"e.name as name",
"e.start_time as start_time",
"e.end_time as end_time",
- mapValue<string>(metadata, "langgraph_node").as("node"),
- mapValue<string>(metadata, "langgraph_step").as("step"),
+ mapValue<string>(metadata, literal("langgraph_node")).as("node"),
+ mapValue<string>(metadata, literal("langgraph_step")).as("step"),
];
})
.where("e.trace_id", "=", traceId)
@@ -3381,22 +3384,11 @@ export const hasAnyUserFromEventsTable = async (
};
/**
- * Streams events from ClickHouse for blob storage export.
- * Uses EventsQueryBuilder for consistent query construction.
+ * Query for the events blob export. Full I/O reads events_full; everything
+ * else reads events_core. `convertLatencyToSecondsInSql` can be set by the paths
+ * that skip the JS row enrichment (raw passthrough, Parquet), so they select
+ * latency and time to first token in seconds instead of milliseconds.
*/
-// metrics field set emits latency / time_to_first_token in MILLISECONDS. The
-// raw-passthrough path (LFE-10402) cannot convert ms→s in JS, so when requested
-// it selects the seconds-converted expressions directly in SQL instead. The
-// standard path keeps ms here and converts conditionally in JS, so it stays
-// byte-for-byte unchanged.
-const EVENTS_EXPORT_METRICS_SECONDS_SQL = [
- "if(isNull(e.end_time), NULL, date_diff('millisecond', e.start_time, e.end_time) / 1000) as latency",
- "if(isNull(e.completion_start_time), NULL, date_diff('millisecond', e.start_time, e.completion_start_time) / 1000) as \"time_to_first_token\"",
-];
-
-// Shared SQL+params builder for the events blob-export. `convertLatencyToSecondsInSql`
-// is only set by the raw-passthrough path; the standard path leaves it false so
-// its output is identical to today.
const buildEventsForBlobStorageExportQuery = (
projectId: string,
minTimestamp: Date,
@@ -3404,37 +3396,33 @@ const buildEventsForBlobStorageExportQuery = (
fieldGroups: ObservationFieldGroupFull[],
convertLatencyToSecondsInSql: boolean,
) => {
- const queryBuilder = new EventsQueryBuilder({ projectId });
-
// core is always required (provides id, trace_id, start/end_time used for cursor and deduplication)
const effectiveGroups = fieldGroups.includes("core")
? fieldGroups
: (["core", ...fieldGroups] as ObservationFieldGroupFull[]);
+ const includeIO = effectiveGroups.includes("io");
+ const metricsInSeconds =
+ convertLatencyToSecondsInSql && effectiveGroups.includes("metrics");
+ const fieldSets = effectiveGroups.flatMap((group) => {
+ if (group === "io" || (group === "metrics" && metricsInSeconds)) return [];
+ // "model_export" is the field set name for the "model" group
+ return group === "model" ? ["model_export" as const] : [group];
+ });
- for (const group of effectiveGroups) {
- if (group === "io") {
- queryBuilder.selectIO(false); // Full I/O, no truncation
- } else if (group === "model") {
- queryBuilder.selectFieldSet("model_export"); // "model_export" is the SQL field set name for the "model" group
- } else if (group === "metrics" && convertLatencyToSecondsInSql) {
- queryBuilder.selectRaw(...EVENTS_EXPORT_METRICS_SECONDS_SQL);
- } else {
- queryBuilder.selectFieldSet(group);
- }
- }
-
- queryBuilder
- .whereRaw(
- "e.start_time >= {minTimestamp: DateTime64(3)} AND e.start_time < {maxTimestamp: DateTime64(3)}",
- {
- minTimestamp: convertDateToClickhouseDateTime(minTimestamp),
- maxTimestamp: convertDateToClickhouseDateTime(maxTimestamp),
- },
+ const builder = getClickhouseKysely()
+ .selectFrom(includeIO ? "events_full as e" : "events_core as e")
+ .select(
+ eventsFieldSetKeys(...fieldSets).map(
+ (key) => EVENTS_ROW_FIELD_SELECTS[key],
+ ),
)
- .whereRaw("e.is_deleted = 0")
- .limitBy("e.span_id", "e.project_id");
+ .$if(includeIO, (qb) => qb.select(["input", "output"]))
+ .$if(metricsInSeconds, (qb) => qb.select(EVENTS_METRICS_SECONDS_SELECTS))
+ .where("e.start_time", ">=", minTimestamp)
+ .where("e.start_time", "<", maxTimestamp)
+ .$call(limitBy({ count: 1, columns: ["e.span_id", "e.project_id"] }));
- const { query, params } = queryBuilder.buildWithParams();
+ const { sql: query, params } = compileClickhouseQuery(builder, projectId);
return {
query,
@@ -3513,7 +3501,6 @@ export const getEventsForBlobStorageExportParquet = function (
/**
* Streams events from ClickHouse for analytics integrations (PostHog, Mixpanel).
- * Uses EventsQueryBuilder for consistent query construction.
* All fields come directly from the events table (which has denormalized trace-level data).
*/
export const getEventsForAnalyticsIntegrations = async function* (
@@ -3522,31 +3509,34 @@ export const getEventsForAnalyticsIntegrations = async function* (
minTimestamp: Date,
maxTimestamp: Date,
) {
- const queryBuilder = new EventsQueryBuilder({ projectId })
+ const builder = getClickhouseKysely()
+ .selectFrom("events_core as e")
// Use export field set for most fields (id, traceId, name, type, level, version,
// environment, userId, sessionId, tags, release, traceName, totalCost, latency, etc.)
- .selectFieldSet("export")
+ .select(
+ eventsFieldSetKeys("export").map((key) => EVENTS_ROW_FIELD_SELECTS[key]),
+ )
// Add analytics-specific computed fields
- .selectRaw(
+ .select((eb) => [
// Token counts from usage/cost details
- "e.usage_details['input'] as input_tokens",
- "e.usage_details['output'] as output_tokens",
- "e.usage_details['total'] as total_tokens",
+ mapValue(eb.ref("e.usage_details"), literal("input")).as("input_tokens"),
+ mapValue(eb.ref("e.usage_details"), literal("output")).as(
+ "output_tokens",
+ ),
+ mapValue(eb.ref("e.usage_details"), literal("total")).as("total_tokens"),
// Analytics integration session IDs from metadata (constructed from array columns)
- `${EVENTS_METADATA_MAP_SQL}['$posthog_session_id'] as posthog_session_id`,
- `${EVENTS_METADATA_MAP_SQL}['$mixpanel_session_id'] as mixpanel_session_id`,
- )
- .whereRaw(
- "e.start_time >= {minTimestamp: DateTime64(3)} AND e.start_time < {maxTimestamp: DateTime64(3)}",
- {
- minTimestamp: convertDateToClickhouseDateTime(minTimestamp),
- maxTimestamp: convertDateToClickhouseDateTime(maxTimestamp),
- },
- )
- .whereRaw("e.is_deleted = 0")
- .limitBy("e.span_id", "e.project_id");
+ metadataValue("e", literal("$posthog_session_id")).as(
+ "posthog_session_id",
+ ),
+ metadataValue("e", literal("$mixpanel_session_id")).as(
+ "mixpanel_session_id",
+ ),
+ ])
+ .where("e.start_time", ">=", minTimestamp)
+ .where("e.start_time", "<", maxTimestamp)
+ .$call(limitBy({ count: 1, columns: ["e.span_id", "e.project_id"] }));
- const { query, params } = queryBuilder.buildWithParams();
+ const { sql: query, params } = compileClickhouseQuery(builder, projectId);
const records = queryClickhouseStream<Record<string, unknown>>({
query,
@@ -3565,7 +3555,7 @@ export const getEventsForAnalyticsIntegrations = async function* (
timestamp: record.start_time,
langfuse_observation_name: record.name,
// The row type here is Record<string, unknown>; the column is a String
- // projection (eventsTableTraceNameSelectSql).
+ // projection ('' when the event has no trace name).
langfuse_trace_name: normalizeEventsTraceName(
record.trace_name as string | null | undefined,
),