Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

refactor(query-ast): type-check the events observation-by-id select (#18809)

Niklas Semmler47b198b06ffe
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
index 9c667d27be..3384fbfdab 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
@@ -7,7 +7,7 @@ import {
eventsTableTraceNameAggregationSql,
eventsTableTraceNameSelectSql,
} from "../../../eventsTable";
-import { expressionBuilder } from "kysely";
+import { expressionBuilder, sql, type SelectExpression } from "kysely";
import { renderClickhouseExpression } from "../../query-ast/compile";
import { metadataMap, traceSessionId } from "../../query-ast/extensions";
@@ -24,13 +24,14 @@ const eventsRef = expressionBuilder<
"e"
>();
-export const EVENTS_METADATA_MAP_SQL = renderClickhouseExpression(
- metadataMap(
- eventsRef.ref("e.metadata_names"),
- eventsRef.ref("e.metadata_values"),
- ),
+const eventsMetadataMap = metadataMap(
+ eventsRef.ref("e.metadata_names"),
+ eventsRef.ref("e.metadata_values"),
);
+export const EVENTS_METADATA_MAP_SQL =
+ renderClickhouseExpression(eventsMetadataMap);
+
const TRACE_SESSION_ID_SQL = renderClickhouseExpression(traceSessionId());
/**
@@ -140,58 +141,57 @@ const EVENTS_FIELDS = {
// Identity & basic fields
id: "e.span_id as id",
- traceId: 'e.trace_id as "trace_id"',
- projectId: 'e.project_id as "project_id"',
- environment: 'e.environment as "environment"',
+ traceId: "e.trace_id as trace_id",
+ projectId: "e.project_id as project_id",
+ environment: "e.environment as environment",
type: "e.type as type",
- parentObservationId: 'e.parent_span_id as "parent_observation_id"',
+ parentObservationId: "e.parent_span_id as parent_observation_id",
isRootObservation: `toBool(${eventsTableIsRootObservationSql}) as "is_root_observation"`,
- isAppRoot: 'e.is_app_root as "is_app_root"',
+ isAppRoot: "e.is_app_root as is_app_root",
name: "e.name as name",
level: "e.level as level",
- statusMessage: 'e.status_message as "status_message"',
+ statusMessage: "e.status_message as status_message",
version: "e.version as version",
bookmarked: "e.bookmarked as bookmarked",
public: "e.public as public",
- userId: 'e.user_id as "user_id"',
- sessionId: 'e.session_id as "session_id"',
+ userId: "e.user_id as user_id",
+ sessionId: "e.session_id as session_id",
// String-typed on purpose; see eventsTableTraceNameSelectSql (LFE-14924).
traceName: `${eventsTableTraceNameSelectSql} as "trace_name"`,
// Time fields
- startTime: 'e.start_time as "start_time"',
- endTime: 'e.end_time as "end_time"',
- completionStartTime: 'e.completion_start_time as "completion_start_time"',
- createdAt: 'e.created_at as "created_at"',
- updatedAt: 'e.updated_at as "updated_at"',
+ startTime: "e.start_time as start_time",
+ endTime: "e.end_time as end_time",
+ completionStartTime: "e.completion_start_time as completion_start_time",
+ createdAt: "e.created_at as created_at",
+ updatedAt: "e.updated_at as updated_at",
eventTs: "e.event_ts",
// Model fields
- providedModelName: 'e.provided_model_name as "provided_model_name"',
- internalModelId: 'e.model_id as "internal_model_id"',
- modelParameters: 'e."model_parameters" as model_parameters',
+ providedModelName: "e.provided_model_name as provided_model_name",
+ internalModelId: "e.model_id as internal_model_id",
+ modelParameters: "e.model_parameters as model_parameters",
// Usage & cost fields
- providedUsageDetails: 'e.provided_usage_details as "provided_usage_details"',
- usageDetails: 'e.usage_details as "usage_details"',
- providedCostDetails: 'e.provided_cost_details as "provided_cost_details"',
- costDetails: 'e.cost_details as "cost_details"',
- totalCost: 'e.total_cost as "total_cost"',
+ providedUsageDetails: "e.provided_usage_details as provided_usage_details",
+ usageDetails: "e.usage_details as usage_details",
+ providedCostDetails: "e.provided_cost_details as provided_cost_details",
+ costDetails: "e.cost_details as cost_details",
+ totalCost: "e.total_cost as total_cost",
// Prompt fields
- promptId: 'e.prompt_id as "prompt_id"',
- promptName: 'e.prompt_name as "prompt_name"',
- promptVersion: 'e.prompt_version as "prompt_version"',
+ promptId: "e.prompt_id as prompt_id",
+ promptName: "e.prompt_name as prompt_name",
+ promptVersion: "e.prompt_version as prompt_version",
// Tool fields
- toolDefinitions: 'e.tool_definitions as "tool_definitions"',
- toolCalls: 'e.tool_calls as "tool_calls"',
- toolCallNames: 'e.tool_call_names as "tool_call_names"',
+ toolDefinitions: "e.tool_definitions as tool_definitions",
+ toolCalls: "e.tool_calls as tool_calls",
+ toolCallNames: "e.tool_call_names as tool_call_names",
// Pricing tier
- usagePricingTierId: 'e.usage_pricing_tier_id as "usage_pricing_tier_id"',
- usagePricingTierName:
- 'e.usage_pricing_tier_name as "usage_pricing_tier_name"',
+ usagePricingTierId: "e.usage_pricing_tier_id as usage_pricing_tier_id",
+ usagePricingTierName: "e.usage_pricing_tier_name as usage_pricing_tier_name",
// I/O & metadata fields
input: "e.input",
@@ -202,26 +202,25 @@ const EVENTS_FIELDS = {
release: "e.release as release",
// Model ID with different alias for exports
- modelId: 'e.model_id as "model_id"',
+ modelId: "e.model_id as model_id",
// Experiment fields (denormalized on events table)
- experimentId: 'e.experiment_id as "experiment_id"',
- experimentName: 'e.experiment_name as "experiment_name"',
- experimentDescription: 'e.experiment_description as "experiment_description"',
- experimentDatasetId: 'e.experiment_dataset_id as "experiment_dataset_id"',
+ experimentId: "e.experiment_id as experiment_id",
+ experimentName: "e.experiment_name as experiment_name",
+ experimentDescription: "e.experiment_description as experiment_description",
+ experimentDatasetId: "e.experiment_dataset_id as experiment_dataset_id",
experimentMetadata:
"mapFromArrays(e.experiment_metadata_names, e.experiment_metadata_values) as experiment_metadata",
// Experiment item fields
- experimentItemId: 'e.experiment_item_id as "experiment_item_id"',
+ experimentItemId: "e.experiment_item_id as experiment_item_id",
experimentItemRootSpanId:
- 'e.experiment_item_root_span_id as "experiment_item_root_span_id"',
+ "e.experiment_item_root_span_id as experiment_item_root_span_id",
experimentItemExpectedOutput:
- 'e.experiment_item_expected_output as "experiment_item_expected_output"',
+ "e.experiment_item_expected_output as experiment_item_expected_output",
experimentItemMetadata:
"mapFromArrays(e.experiment_item_metadata_names, e.experiment_item_metadata_values) as experiment_item_metadata",
- experimentItemVersion:
- 'e.experiment_item_version as "experiment_item_version"',
+ experimentItemVersion: "e.experiment_item_version as experiment_item_version",
// Calculated fields
latency:
@@ -534,31 +533,87 @@ const FIELD_SETS = {
export type FieldSetName = keyof typeof FIELD_SETS;
+type EventsFieldKey = keyof typeof EVENTS_FIELDS;
+
+type EventsSelection = SelectExpression<
+ { e: ClickHouseDatabase["events_core"] },
+ "e"
+>;
+
+/** Fields whose EVENTS_FIELDS entry is a registry column reference. */
+type EventsColumnField = {
+ [K in EventsFieldKey]: (typeof EVENTS_FIELDS)[K] extends EventsSelection
+ ? K
+ : never;
+}[EventsFieldKey];
+
/**
- * SELECT expressions (`expr as alias`) for the given field sets, in
- * EVENTS_FIELDS declaration order.
+ * Query-AST selections for fields that are not plain column references,
+ * built from the same source as their EVENTS_FIELDS entry.
*/
-export function eventsFieldSetSelectExpressions(
- ...setNames: FieldSetName[]
-): string[] {
- return eventsFieldSelectExpressions(setNames.flatMap((s) => FIELD_SETS[s]));
+const EVENTS_FIELD_EXPRESSIONS = {
+ traceName: sql.raw<string>(eventsTableTraceNameSelectSql).as("trace_name"),
+ metadata: eventsMetadataMap.as("metadata"),
+} satisfies Partial<
+ Record<Exclude<EventsFieldKey, EventsColumnField>, EventsSelection>
+>;
+
+type EventsExpressionField = keyof typeof EVENTS_FIELD_EXPRESSIONS;
+
+/** Field sets whose every field has a query-AST selection. */
+type SelectableFieldSetName = {
+ [S in FieldSetName]: (typeof FIELD_SETS)[S][number] extends
+ | EventsColumnField
+ | EventsExpressionField
+ ? S
+ : never;
+}[FieldSetName];
+
+type EventsFieldSelection<K> = K extends EventsExpressionField
+ ? (typeof EVENTS_FIELD_EXPRESSIONS)[K]
+ : K extends EventsColumnField
+ ? (typeof EVENTS_FIELDS)[K]
+ : never;
+
+function isEventsExpressionField(key: string): key is EventsExpressionField {
+ return Object.hasOwn(EVENTS_FIELD_EXPRESSIONS, key);
+}
+
+/**
+ * Query-AST selections for the given field sets, in EVENTS_FIELDS declaration
+ * order. Column fields are the EVENTS_FIELDS strings themselves, so Kysely
+ * checks each against the table registry and infers the selected row.
+ */
+export function eventsFieldSetSelections<S extends SelectableFieldSetName>(
+ ...setNames: S[]
+): Array<EventsFieldSelection<(typeof FIELD_SETS)[S][number]>>;
+export function eventsFieldSetSelections(
+ ...setNames: SelectableFieldSetName[]
+): EventsSelection[] {
+ return orderedFieldKeys(setNames.flatMap((s) => FIELD_SETS[s])).map((key) =>
+ isEventsExpressionField(key)
+ ? EVENTS_FIELD_EXPRESSIONS[key]
+ : EVENTS_FIELDS[key],
+ );
}
// Canonicalize by EVENTS_FIELDS declaration order so SELECT column order does
// not follow caller field-set order. Clustered ClickHouse reads align result
// blocks by position; incompatible Map types (cost_details vs metadata) 500
// when those columns swap places.
+function orderedFieldKeys<K extends string>(fieldKeys: readonly K[]): K[] {
+ return [...new Set(fieldKeys)].sort(
+ (a, b) =>
+ (EVENTS_FIELD_ORDER_INDEX.get(a) ?? Number.MAX_SAFE_INTEGER) -
+ (EVENTS_FIELD_ORDER_INDEX.get(b) ?? Number.MAX_SAFE_INTEGER),
+ );
+}
+
function eventsFieldSelectExpressions(fieldKeys: readonly string[]): string[] {
- return [...new Set(fieldKeys)]
- .sort(
- (a, b) =>
- (EVENTS_FIELD_ORDER_INDEX.get(a) ?? Number.MAX_SAFE_INTEGER) -
- (EVENTS_FIELD_ORDER_INDEX.get(b) ?? Number.MAX_SAFE_INTEGER),
- )
- .flatMap((fieldKey) => {
- const fieldExpr = EVENTS_FIELDS[fieldKey as keyof typeof EVENTS_FIELDS];
- return fieldExpr ? [fieldExpr] : [];
- });
+ return orderedFieldKeys(fieldKeys).flatMap((fieldKey) => {
+ const fieldExpr = EVENTS_FIELDS[fieldKey as keyof typeof EVENTS_FIELDS];
+ return fieldExpr ? [fieldExpr] : [];
+ });
}
export const OBSERVATION_FIELD_GROUP_FIELD_NAMES = Object.fromEntries(
diff --git a/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts b/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts
index c28417a7c8..6e59ed242c 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts
@@ -67,8 +67,8 @@ describe("buildEventsStreamQuery", () => {
expect(query).toContain("e.project_id = {projectId: String}");
expect(query).toContain("e.is_deleted = 0");
- expect(query).toContain('e.is_app_root as "is_app_root"');
- expect(query).toContain('e.experiment_name as "experiment_name"');
+ expect(query).toContain("e.is_app_root as is_app_root");
+ expect(query).toContain("e.experiment_name as experiment_name");
const orderIndex = query.lastIndexOf("ORDER BY ");
const deduplicationIndex = query.lastIndexOf("LIMIT 1 BY ");
diff --git a/packages/shared/src/server/query-ast/schema.ts b/packages/shared/src/server/query-ast/schema.ts
index 2a1df2ae47..33978a2ea8 100644
--- a/packages/shared/src/server/query-ast/schema.ts
+++ b/packages/shared/src/server/query-ast/schema.ts
@@ -33,6 +33,8 @@ export type ChColumnType =
| "DateTime64(6)"
| "UInt8"
| "UInt16"
+ | "UInt64"
+ | "Bool"
| "Nullable(UInt16)"
| "Nullable(DateTime64(6))"
| "Array(String)"
@@ -46,23 +48,25 @@ type TsTypeOfColumn<T extends ChColumnType> = T extends "String"
? string | null
: T extends "Float" | "Decimal(18,12)"
? number
- : T extends "UInt8" | "UInt16"
+ : T extends "UInt8" | "UInt16" | "UInt64"
? number
- : T extends "Nullable(UInt16)"
- ? number | null
- : T extends "DateTime" | "DateTime64(6)"
- ? Date
- : T extends "Nullable(DateTime64(6))"
- ? Date | null
- : T extends "Array(String)"
- ? string[]
- : T extends
- | "Map(String, UInt64)"
- | "Map(String, Decimal(18,12))"
- ? Record<string, number>
- : T extends "Map(String, String)"
- ? Record<string, string>
- : never;
+ : T extends "Bool"
+ ? boolean
+ : T extends "Nullable(UInt16)"
+ ? number | null
+ : T extends "DateTime" | "DateTime64(6)"
+ ? Date
+ : T extends "Nullable(DateTime64(6))"
+ ? Date | null
+ : T extends "Array(String)"
+ ? string[]
+ : T extends
+ | "Map(String, UInt64)"
+ | "Map(String, Decimal(18,12))"
+ ? Record<string, number>
+ : T extends "Map(String, String)"
+ ? Record<string, string>
+ : never;
/** Coarse runtime category the aggregate type-check pass reasons about. */
export type ColumnDataType = "string" | "number" | "date" | "array" | "map";
@@ -74,6 +78,9 @@ const RUNTIME_TYPE: Record<ChColumnType, ColumnDataType> = {
"Decimal(18,12)": "number",
UInt8: "number",
UInt16: "number",
+ UInt64: "number",
+ // ClickHouse stores Bool as UInt8, so numeric aggregates accept it.
+ Bool: "number",
"Nullable(UInt16)": "number",
DateTime: "date",
"DateTime64(6)": "date",
@@ -159,6 +166,50 @@ const EVENTS_COLUMNS = {
experiment_item_expected_output: "String",
experiment_item_metadata_names: "Array(String)",
experiment_item_metadata_values: "Array(String)",
+ version: "String",
+ release: "String",
+ trace_name: "String",
+ level: "String",
+ status_message: "String",
+ completion_start_time: "Nullable(DateTime64(6))",
+ is_app_root: "Bool",
+ bookmarked: "Bool",
+ public: "Bool",
+ model_id: "String",
+ provided_model_name: "String",
+ model_parameters: "String",
+ provided_usage_details: "Map(String, UInt64)",
+ provided_cost_details: "Map(String, Decimal(18,12))",
+ calculated_input_cost: "Decimal(18,12)",
+ calculated_output_cost: "Decimal(18,12)",
+ calculated_total_cost: "Decimal(18,12)",
+ usage_pricing_tier_id: "Nullable(String)",
+ usage_pricing_tier_name: "Nullable(String)",
+ tool_definitions: "Map(String, String)",
+ input_length: "UInt64",
+ output_length: "UInt64",
+ evaluator_id: "String",
+ evaluation_rule_id: "String",
+ evaluator_execution_is_test: "Bool",
+ experiment_name: "String",
+ experiment_metadata_names: "Array(String)",
+ experiment_metadata_values: "Array(String)",
+ experiment_description: "String",
+ experiment_dataset_id: "String",
+ experiment_item_id: "String",
+ experiment_item_version: "Nullable(DateTime64(6))",
+ experiment_item_root_span_id: "String",
+ service_name: "String",
+ service_version: "String",
+ scope_name: "String",
+ scope_version: "String",
+ telemetry_sdk_name: "String",
+ telemetry_sdk_version: "String",
+ blob_storage_file_path: "String",
+ event_bytes: "UInt64",
+ created_at: "DateTime64(6)",
+ updated_at: "DateTime64(6)",
+ ingestion_api_key: "String",
} as const satisfies Record<string, ChColumnType>;
const TABLE_REGISTRY = {
@@ -273,6 +324,8 @@ const BIND_TYPE: Record<ChColumnType, string> = {
"Decimal(18,12)": "Decimal(18, 12)",
UInt8: "UInt8",
UInt16: "UInt16",
+ UInt64: "UInt64",
+ Bool: "Bool",
// A bound value is never NULL; NULL checks compile to `IS NULL`.
"Nullable(String)": "String",
"Nullable(UInt16)": "UInt16",
diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.ts
index 428fad0d0a..2507c48bd3 100644
--- a/packages/shared/src/server/repositories/events.ts
+++ b/packages/shared/src/server/repositories/events.ts
@@ -142,7 +142,7 @@ import {
buildEventsFullTableSplitQuery,
EVENTS_AGGREGATION_FIELDS,
EVENTS_SESSION_AGGREGATION_FIELDS,
- eventsFieldSetSelectExpressions,
+ eventsFieldSetSelections,
type QueryWithParams,
normalizeSessionCounts,
type SessionEventsField,
@@ -1092,7 +1092,15 @@ async function getObservationByIdFromEventsTableInternal({
? db.selectFrom("events_full as e")
: db.selectFrom("events_core as e")
)
- .select(observationByIdSelects() as never)
+ // Built per call: events.ts and the builder module import each other.
+ .select(
+ eventsFieldSetSelections(
+ "byIdBase",
+ "byIdModel",
+ "byIdPrompt",
+ "byIdTimestamps",
+ ),
+ )
.$if(fetchWithInputOutput, (qb) =>
qb.select(observationIoSelects(renderingProps.truncated)),
)
@@ -1145,21 +1153,6 @@ function observationIoSelects(
];
}
-/**
- * Raw select expressions (`expr as alias`) taken from the legacy builder field
- * maps, so the AST and the builders still serving other call sites cannot drift.
- * Lazy: events.ts sits in an import cycle with the builder module, so the maps
- * can still be undefined while this module initializes.
- */
-function observationByIdSelects() {
- return eventsFieldSetSelectExpressions(
- "byIdBase",
- "byIdModel",
- "byIdPrompt",
- "byIdTimestamps",
- ).map((field) => sql.raw(field));
-}
-
/**
* Get a trace by ID from the events table.
* Compatible with getTraceById but queries the events table instead.
diff --git a/web/src/__tests__/server/event-query-builder.servertest.ts b/web/src/__tests__/server/event-query-builder.servertest.ts
index 0b0998685e..7898d1d588 100644
--- a/web/src/__tests__/server/event-query-builder.servertest.ts
+++ b/web/src/__tests__/server/event-query-builder.servertest.ts
@@ -1010,13 +1010,13 @@ describe("EventsQueryBuilder", () => {
.selectFieldSet("base", "calculated")
.buildWithParams().query;
- expect(slimQuery).not.toContain('e.tool_definitions as "tool_definitions"');
- expect(slimQuery).not.toContain('e.tool_calls as "tool_calls"');
- expect(slimQuery).toContain('e.tool_call_names as "tool_call_names"');
+ expect(slimQuery).not.toContain("e.tool_definitions as tool_definitions");
+ expect(slimQuery).not.toContain("e.tool_calls as tool_calls");
+ expect(slimQuery).toContain("e.tool_call_names as tool_call_names");
- expect(defaultQuery).toContain('e.tool_definitions as "tool_definitions"');
- expect(defaultQuery).toContain('e.tool_calls as "tool_calls"');
- expect(defaultQuery).toContain('e.tool_call_names as "tool_call_names"');
+ expect(defaultQuery).toContain("e.tool_definitions as tool_definitions");
+ expect(defaultQuery).toContain("e.tool_calls as tool_calls");
+ expect(defaultQuery).toContain("e.tool_call_names as tool_call_names");
});
it("should query events_full when forceFullTable is enabled", () => {
@@ -1054,10 +1054,10 @@ describe("EventsQueryBuilder", () => {
[listQuery, slimListQuery, byIdQuery, usageQuery, exportQuery].forEach(
(query) => {
expect(query).toContain(
- 'e.usage_pricing_tier_id as "usage_pricing_tier_id"',
+ "e.usage_pricing_tier_id as usage_pricing_tier_id",
);
expect(query).toContain(
- 'e.usage_pricing_tier_name as "usage_pricing_tier_name"',
+ "e.usage_pricing_tier_name as usage_pricing_tier_name",
);
},
);