🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(query-ast): type-check the events observation-by-id select (#18809)
Niklas Semmler47b198b06ffe
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.tsindex 9c667d27be..3384fbfdab 100644--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts@@ -7,7 +7,7 @@ import {eventsTableTraceNameAggregationSql,eventsTableTraceNameSelectSql,} from "../../../eventsTable";-import { expressionBuilder } from "kysely";+import { expressionBuilder, sql, type SelectExpression } from "kysely";import { renderClickhouseExpression } from "../../query-ast/compile";import { metadataMap, traceSessionId } from "../../query-ast/extensions";@@ -24,13 +24,14 @@ const eventsRef = expressionBuilder<"e">();-export const EVENTS_METADATA_MAP_SQL = renderClickhouseExpression(- metadataMap(- eventsRef.ref("e.metadata_names"),- eventsRef.ref("e.metadata_values"),- ),+const eventsMetadataMap = metadataMap(+ eventsRef.ref("e.metadata_names"),+ eventsRef.ref("e.metadata_values"),);+export const EVENTS_METADATA_MAP_SQL =+ renderClickhouseExpression(eventsMetadataMap);+const TRACE_SESSION_ID_SQL = renderClickhouseExpression(traceSessionId());/**@@ -140,58 +141,57 @@ const EVENTS_FIELDS = {// Identity & basic fieldsid: "e.span_id as id",- traceId: 'e.trace_id as "trace_id"',- projectId: 'e.project_id as "project_id"',- environment: 'e.environment as "environment"',+ traceId: "e.trace_id as trace_id",+ projectId: "e.project_id as project_id",+ environment: "e.environment as environment",type: "e.type as type",- parentObservationId: 'e.parent_span_id as "parent_observation_id"',+ parentObservationId: "e.parent_span_id as parent_observation_id",isRootObservation: `toBool(${eventsTableIsRootObservationSql}) as "is_root_observation"`,- isAppRoot: 'e.is_app_root as "is_app_root"',+ isAppRoot: "e.is_app_root as is_app_root",name: "e.name as name",level: "e.level as level",- statusMessage: 'e.status_message as "status_message"',+ statusMessage: "e.status_message as status_message",version: "e.version as version",bookmarked: "e.bookmarked as bookmarked",public: "e.public as public",- userId: 'e.user_id as "user_id"',- sessionId: 'e.session_id as "session_id"',+ userId: "e.user_id as user_id",+ sessionId: "e.session_id as session_id",// String-typed on purpose; see eventsTableTraceNameSelectSql (LFE-14924).traceName: `${eventsTableTraceNameSelectSql} as "trace_name"`,// Time fields- startTime: 'e.start_time as "start_time"',- endTime: 'e.end_time as "end_time"',- completionStartTime: 'e.completion_start_time as "completion_start_time"',- createdAt: 'e.created_at as "created_at"',- updatedAt: 'e.updated_at as "updated_at"',+ startTime: "e.start_time as start_time",+ endTime: "e.end_time as end_time",+ completionStartTime: "e.completion_start_time as completion_start_time",+ createdAt: "e.created_at as created_at",+ updatedAt: "e.updated_at as updated_at",eventTs: "e.event_ts",// Model fields- providedModelName: 'e.provided_model_name as "provided_model_name"',- internalModelId: 'e.model_id as "internal_model_id"',- modelParameters: 'e."model_parameters" as model_parameters',+ providedModelName: "e.provided_model_name as provided_model_name",+ internalModelId: "e.model_id as internal_model_id",+ modelParameters: "e.model_parameters as model_parameters",// Usage & cost fields- providedUsageDetails: 'e.provided_usage_details as "provided_usage_details"',- usageDetails: 'e.usage_details as "usage_details"',- providedCostDetails: 'e.provided_cost_details as "provided_cost_details"',- costDetails: 'e.cost_details as "cost_details"',- totalCost: 'e.total_cost as "total_cost"',+ providedUsageDetails: "e.provided_usage_details as provided_usage_details",+ usageDetails: "e.usage_details as usage_details",+ providedCostDetails: "e.provided_cost_details as provided_cost_details",+ costDetails: "e.cost_details as cost_details",+ totalCost: "e.total_cost as total_cost",// Prompt fields- promptId: 'e.prompt_id as "prompt_id"',- promptName: 'e.prompt_name as "prompt_name"',- promptVersion: 'e.prompt_version as "prompt_version"',+ promptId: "e.prompt_id as prompt_id",+ promptName: "e.prompt_name as prompt_name",+ promptVersion: "e.prompt_version as prompt_version",// Tool fields- toolDefinitions: 'e.tool_definitions as "tool_definitions"',- toolCalls: 'e.tool_calls as "tool_calls"',- toolCallNames: 'e.tool_call_names as "tool_call_names"',+ toolDefinitions: "e.tool_definitions as tool_definitions",+ toolCalls: "e.tool_calls as tool_calls",+ toolCallNames: "e.tool_call_names as tool_call_names",// Pricing tier- usagePricingTierId: 'e.usage_pricing_tier_id as "usage_pricing_tier_id"',- usagePricingTierName:- 'e.usage_pricing_tier_name as "usage_pricing_tier_name"',+ usagePricingTierId: "e.usage_pricing_tier_id as usage_pricing_tier_id",+ usagePricingTierName: "e.usage_pricing_tier_name as usage_pricing_tier_name",// I/O & metadata fieldsinput: "e.input",@@ -202,26 +202,25 @@ const EVENTS_FIELDS = {release: "e.release as release",// Model ID with different alias for exports- modelId: 'e.model_id as "model_id"',+ modelId: "e.model_id as model_id",// Experiment fields (denormalized on events table)- experimentId: 'e.experiment_id as "experiment_id"',- experimentName: 'e.experiment_name as "experiment_name"',- experimentDescription: 'e.experiment_description as "experiment_description"',- experimentDatasetId: 'e.experiment_dataset_id as "experiment_dataset_id"',+ experimentId: "e.experiment_id as experiment_id",+ experimentName: "e.experiment_name as experiment_name",+ experimentDescription: "e.experiment_description as experiment_description",+ experimentDatasetId: "e.experiment_dataset_id as experiment_dataset_id",experimentMetadata:"mapFromArrays(e.experiment_metadata_names, e.experiment_metadata_values) as experiment_metadata",// Experiment item fields- experimentItemId: 'e.experiment_item_id as "experiment_item_id"',+ experimentItemId: "e.experiment_item_id as experiment_item_id",experimentItemRootSpanId:- 'e.experiment_item_root_span_id as "experiment_item_root_span_id"',+ "e.experiment_item_root_span_id as experiment_item_root_span_id",experimentItemExpectedOutput:- 'e.experiment_item_expected_output as "experiment_item_expected_output"',+ "e.experiment_item_expected_output as experiment_item_expected_output",experimentItemMetadata:"mapFromArrays(e.experiment_item_metadata_names, e.experiment_item_metadata_values) as experiment_item_metadata",- experimentItemVersion:- 'e.experiment_item_version as "experiment_item_version"',+ experimentItemVersion: "e.experiment_item_version as experiment_item_version",// Calculated fieldslatency:@@ -534,31 +533,87 @@ const FIELD_SETS = {export type FieldSetName = keyof typeof FIELD_SETS;+type EventsFieldKey = keyof typeof EVENTS_FIELDS;++type EventsSelection = SelectExpression<+ { e: ClickHouseDatabase["events_core"] },+ "e"+>;++/** Fields whose EVENTS_FIELDS entry is a registry column reference. */+type EventsColumnField = {+ [K in EventsFieldKey]: (typeof EVENTS_FIELDS)[K] extends EventsSelection+ ? K+ : never;+}[EventsFieldKey];+/**- * SELECT expressions (`expr as alias`) for the given field sets, in- * EVENTS_FIELDS declaration order.+ * Query-AST selections for fields that are not plain column references,+ * built from the same source as their EVENTS_FIELDS entry.*/-export function eventsFieldSetSelectExpressions(- ...setNames: FieldSetName[]-): string[] {- return eventsFieldSelectExpressions(setNames.flatMap((s) => FIELD_SETS[s]));+const EVENTS_FIELD_EXPRESSIONS = {+ traceName: sql.raw<string>(eventsTableTraceNameSelectSql).as("trace_name"),+ metadata: eventsMetadataMap.as("metadata"),+} satisfies Partial<+ Record<Exclude<EventsFieldKey, EventsColumnField>, EventsSelection>+>;++type EventsExpressionField = keyof typeof EVENTS_FIELD_EXPRESSIONS;++/** Field sets whose every field has a query-AST selection. */+type SelectableFieldSetName = {+ [S in FieldSetName]: (typeof FIELD_SETS)[S][number] extends+ | EventsColumnField+ | EventsExpressionField+ ? S+ : never;+}[FieldSetName];++type EventsFieldSelection<K> = K extends EventsExpressionField+ ? (typeof EVENTS_FIELD_EXPRESSIONS)[K]+ : K extends EventsColumnField+ ? (typeof EVENTS_FIELDS)[K]+ : never;++function isEventsExpressionField(key: string): key is EventsExpressionField {+ return Object.hasOwn(EVENTS_FIELD_EXPRESSIONS, key);+}++/**+ * Query-AST selections for the given field sets, in EVENTS_FIELDS declaration+ * order. Column fields are the EVENTS_FIELDS strings themselves, so Kysely+ * checks each against the table registry and infers the selected row.+ */+export function eventsFieldSetSelections<S extends SelectableFieldSetName>(+ ...setNames: S[]+): Array<EventsFieldSelection<(typeof FIELD_SETS)[S][number]>>;+export function eventsFieldSetSelections(+ ...setNames: SelectableFieldSetName[]+): EventsSelection[] {+ return orderedFieldKeys(setNames.flatMap((s) => FIELD_SETS[s])).map((key) =>+ isEventsExpressionField(key)+ ? EVENTS_FIELD_EXPRESSIONS[key]+ : EVENTS_FIELDS[key],+ );}// Canonicalize by EVENTS_FIELDS declaration order so SELECT column order does// not follow caller field-set order. Clustered ClickHouse reads align result// blocks by position; incompatible Map types (cost_details vs metadata) 500// when those columns swap places.+function orderedFieldKeys<K extends string>(fieldKeys: readonly K[]): K[] {+ return [...new Set(fieldKeys)].sort(+ (a, b) =>+ (EVENTS_FIELD_ORDER_INDEX.get(a) ?? Number.MAX_SAFE_INTEGER) -+ (EVENTS_FIELD_ORDER_INDEX.get(b) ?? Number.MAX_SAFE_INTEGER),+ );+}+function eventsFieldSelectExpressions(fieldKeys: readonly string[]): string[] {- return [...new Set(fieldKeys)]- .sort(- (a, b) =>- (EVENTS_FIELD_ORDER_INDEX.get(a) ?? Number.MAX_SAFE_INTEGER) -- (EVENTS_FIELD_ORDER_INDEX.get(b) ?? Number.MAX_SAFE_INTEGER),- )- .flatMap((fieldKey) => {- const fieldExpr = EVENTS_FIELDS[fieldKey as keyof typeof EVENTS_FIELDS];- return fieldExpr ? [fieldExpr] : [];- });+ return orderedFieldKeys(fieldKeys).flatMap((fieldKey) => {+ const fieldExpr = EVENTS_FIELDS[fieldKey as keyof typeof EVENTS_FIELDS];+ return fieldExpr ? [fieldExpr] : [];+ });}export const OBSERVATION_FIELD_GROUP_FIELD_NAMES = Object.fromEntries(diff --git a/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts b/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.tsindex c28417a7c8..6e59ed242c 100644--- a/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts@@ -67,8 +67,8 @@ describe("buildEventsStreamQuery", () => {expect(query).toContain("e.project_id = {projectId: String}");expect(query).toContain("e.is_deleted = 0");- expect(query).toContain('e.is_app_root as "is_app_root"');- expect(query).toContain('e.experiment_name as "experiment_name"');+ expect(query).toContain("e.is_app_root as is_app_root");+ expect(query).toContain("e.experiment_name as experiment_name");const orderIndex = query.lastIndexOf("ORDER BY ");const deduplicationIndex = query.lastIndexOf("LIMIT 1 BY ");diff --git a/packages/shared/src/server/query-ast/schema.ts b/packages/shared/src/server/query-ast/schema.tsindex 2a1df2ae47..33978a2ea8 100644--- a/packages/shared/src/server/query-ast/schema.ts+++ b/packages/shared/src/server/query-ast/schema.ts@@ -33,6 +33,8 @@ export type ChColumnType =| "DateTime64(6)"| "UInt8"| "UInt16"+ | "UInt64"+ | "Bool"| "Nullable(UInt16)"| "Nullable(DateTime64(6))"| "Array(String)"@@ -46,23 +48,25 @@ type TsTypeOfColumn<T extends ChColumnType> = T extends "String"? string | null: T extends "Float" | "Decimal(18,12)"? number- : T extends "UInt8" | "UInt16"+ : T extends "UInt8" | "UInt16" | "UInt64"? number- : T extends "Nullable(UInt16)"- ? number | null- : T extends "DateTime" | "DateTime64(6)"- ? Date- : T extends "Nullable(DateTime64(6))"- ? Date | null- : T extends "Array(String)"- ? string[]- : T extends- | "Map(String, UInt64)"- | "Map(String, Decimal(18,12))"- ? Record<string, number>- : T extends "Map(String, String)"- ? Record<string, string>- : never;+ : T extends "Bool"+ ? boolean+ : T extends "Nullable(UInt16)"+ ? number | null+ : T extends "DateTime" | "DateTime64(6)"+ ? Date+ : T extends "Nullable(DateTime64(6))"+ ? Date | null+ : T extends "Array(String)"+ ? string[]+ : T extends+ | "Map(String, UInt64)"+ | "Map(String, Decimal(18,12))"+ ? Record<string, number>+ : T extends "Map(String, String)"+ ? Record<string, string>+ : never;/** Coarse runtime category the aggregate type-check pass reasons about. */export type ColumnDataType = "string" | "number" | "date" | "array" | "map";@@ -74,6 +78,9 @@ const RUNTIME_TYPE: Record<ChColumnType, ColumnDataType> = {"Decimal(18,12)": "number",UInt8: "number",UInt16: "number",+ UInt64: "number",+ // ClickHouse stores Bool as UInt8, so numeric aggregates accept it.+ Bool: "number","Nullable(UInt16)": "number",DateTime: "date","DateTime64(6)": "date",@@ -159,6 +166,50 @@ const EVENTS_COLUMNS = {experiment_item_expected_output: "String",experiment_item_metadata_names: "Array(String)",experiment_item_metadata_values: "Array(String)",+ version: "String",+ release: "String",+ trace_name: "String",+ level: "String",+ status_message: "String",+ completion_start_time: "Nullable(DateTime64(6))",+ is_app_root: "Bool",+ bookmarked: "Bool",+ public: "Bool",+ model_id: "String",+ provided_model_name: "String",+ model_parameters: "String",+ provided_usage_details: "Map(String, UInt64)",+ provided_cost_details: "Map(String, Decimal(18,12))",+ calculated_input_cost: "Decimal(18,12)",+ calculated_output_cost: "Decimal(18,12)",+ calculated_total_cost: "Decimal(18,12)",+ usage_pricing_tier_id: "Nullable(String)",+ usage_pricing_tier_name: "Nullable(String)",+ tool_definitions: "Map(String, String)",+ input_length: "UInt64",+ output_length: "UInt64",+ evaluator_id: "String",+ evaluation_rule_id: "String",+ evaluator_execution_is_test: "Bool",+ experiment_name: "String",+ experiment_metadata_names: "Array(String)",+ experiment_metadata_values: "Array(String)",+ experiment_description: "String",+ experiment_dataset_id: "String",+ experiment_item_id: "String",+ experiment_item_version: "Nullable(DateTime64(6))",+ experiment_item_root_span_id: "String",+ service_name: "String",+ service_version: "String",+ scope_name: "String",+ scope_version: "String",+ telemetry_sdk_name: "String",+ telemetry_sdk_version: "String",+ blob_storage_file_path: "String",+ event_bytes: "UInt64",+ created_at: "DateTime64(6)",+ updated_at: "DateTime64(6)",+ ingestion_api_key: "String",} as const satisfies Record<string, ChColumnType>;const TABLE_REGISTRY = {@@ -273,6 +324,8 @@ const BIND_TYPE: Record<ChColumnType, string> = {"Decimal(18,12)": "Decimal(18, 12)",UInt8: "UInt8",UInt16: "UInt16",+ UInt64: "UInt64",+ Bool: "Bool",// A bound value is never NULL; NULL checks compile to `IS NULL`."Nullable(String)": "String","Nullable(UInt16)": "UInt16",diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex 428fad0d0a..2507c48bd3 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -142,7 +142,7 @@ import {buildEventsFullTableSplitQuery,EVENTS_AGGREGATION_FIELDS,EVENTS_SESSION_AGGREGATION_FIELDS,- eventsFieldSetSelectExpressions,+ eventsFieldSetSelections,type QueryWithParams,normalizeSessionCounts,type SessionEventsField,@@ -1092,7 +1092,15 @@ async function getObservationByIdFromEventsTableInternal({? db.selectFrom("events_full as e"): db.selectFrom("events_core as e"))- .select(observationByIdSelects() as never)+ // Built per call: events.ts and the builder module import each other.+ .select(+ eventsFieldSetSelections(+ "byIdBase",+ "byIdModel",+ "byIdPrompt",+ "byIdTimestamps",+ ),+ ).$if(fetchWithInputOutput, (qb) =>qb.select(observationIoSelects(renderingProps.truncated)),)@@ -1145,21 +1153,6 @@ function observationIoSelects(];}-/**- * Raw select expressions (`expr as alias`) taken from the legacy builder field- * maps, so the AST and the builders still serving other call sites cannot drift.- * Lazy: events.ts sits in an import cycle with the builder module, so the maps- * can still be undefined while this module initializes.- */-function observationByIdSelects() {- return eventsFieldSetSelectExpressions(- "byIdBase",- "byIdModel",- "byIdPrompt",- "byIdTimestamps",- ).map((field) => sql.raw(field));-}-/*** Get a trace by ID from the events table.* Compatible with getTraceById but queries the events table instead.diff --git a/web/src/__tests__/server/event-query-builder.servertest.ts b/web/src/__tests__/server/event-query-builder.servertest.tsindex 0b0998685e..7898d1d588 100644--- a/web/src/__tests__/server/event-query-builder.servertest.ts+++ b/web/src/__tests__/server/event-query-builder.servertest.ts@@ -1010,13 +1010,13 @@ describe("EventsQueryBuilder", () => {.selectFieldSet("base", "calculated").buildWithParams().query;- expect(slimQuery).not.toContain('e.tool_definitions as "tool_definitions"');- expect(slimQuery).not.toContain('e.tool_calls as "tool_calls"');- expect(slimQuery).toContain('e.tool_call_names as "tool_call_names"');+ expect(slimQuery).not.toContain("e.tool_definitions as tool_definitions");+ expect(slimQuery).not.toContain("e.tool_calls as tool_calls");+ expect(slimQuery).toContain("e.tool_call_names as tool_call_names");- expect(defaultQuery).toContain('e.tool_definitions as "tool_definitions"');- expect(defaultQuery).toContain('e.tool_calls as "tool_calls"');- expect(defaultQuery).toContain('e.tool_call_names as "tool_call_names"');+ expect(defaultQuery).toContain("e.tool_definitions as tool_definitions");+ expect(defaultQuery).toContain("e.tool_calls as tool_calls");+ expect(defaultQuery).toContain("e.tool_call_names as tool_call_names");});it("should query events_full when forceFullTable is enabled", () => {@@ -1054,10 +1054,10 @@ describe("EventsQueryBuilder", () => {[listQuery, slimListQuery, byIdQuery, usageQuery, exportQuery].forEach((query) => {expect(query).toContain(- 'e.usage_pricing_tier_id as "usage_pricing_tier_id"',+ "e.usage_pricing_tier_id as usage_pricing_tier_id",);expect(query).toContain(- 'e.usage_pricing_tier_name as "usage_pricing_tier_name"',+ "e.usage_pricing_tier_name as usage_pricing_tier_name",);},);