Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

refactor(query-ast): compile events point-reads through the AST (#17550)

Niklas Semmler2484ddecdb84
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.md
index 554bec2c5f..8e0e814450 100644
--- a/packages/shared/AGENTS.md
+++ b/packages/shared/AGENTS.md
@@ -50,6 +50,7 @@
`clickhouse format`); each migrated call site keeps its `*.golden.test.ts`
baseline next to the call site (e.g.
`src/server/repositories/environments.golden.test.ts`,
+ `src/server/repositories/events.golden.test.ts`,
`src/server/repositories/scores.golden.test.ts`,
`src/server/queries/clickhouse-sql/event-filter-options.golden.test.ts`).
- Postgres schema: `prisma/schema.prisma`
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
index d12d033ca5..3e5f6be364 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
@@ -512,6 +512,33 @@ const FIELD_SETS = {
export type FieldSetName = keyof typeof FIELD_SETS;
+/**
+ * SELECT expressions (`expr as alias`) for the given field sets, in
+ * EVENTS_FIELDS declaration order.
+ */
+export function eventsFieldSetSelectExpressions(
+ ...setNames: FieldSetName[]
+): string[] {
+ return eventsFieldSelectExpressions(setNames.flatMap((s) => FIELD_SETS[s]));
+}
+
+// Canonicalize by EVENTS_FIELDS declaration order so SELECT column order does
+// not follow caller field-set order. Clustered ClickHouse reads align result
+// blocks by position; incompatible Map types (cost_details vs metadata) 500
+// when those columns swap places.
+function eventsFieldSelectExpressions(fieldKeys: readonly string[]): string[] {
+ return [...new Set(fieldKeys)]
+ .sort(
+ (a, b) =>
+ (EVENTS_FIELD_ORDER_INDEX.get(a) ?? Number.MAX_SAFE_INTEGER) -
+ (EVENTS_FIELD_ORDER_INDEX.get(b) ?? Number.MAX_SAFE_INTEGER),
+ )
+ .flatMap((fieldKey) => {
+ const fieldExpr = EVENTS_FIELDS[fieldKey as keyof typeof EVENTS_FIELDS];
+ return fieldExpr ? [fieldExpr] : [];
+ });
+}
+
export const OBSERVATION_FIELD_GROUP_FIELD_NAMES = Object.fromEntries(
OBSERVATION_FIELD_GROUPS_PUBLIC_API.map((group) => [
group,
@@ -525,7 +552,7 @@ export const OBSERVATION_FIELD_GROUP_FIELD_NAMES = Object.fromEntries(
* Aggregation fields for trace-level queries
* These fields use ClickHouse aggregation functions and require GROUP BY
*/
-const EVENTS_AGGREGATION_FIELDS = {
+export const EVENTS_AGGREGATION_FIELDS = {
// Grouping keys (must be in GROUP BY)
id: "trace_id AS id",
projectId: "project_id",
@@ -1172,22 +1199,9 @@ export class EventsQueryBuilder extends BaseEventsQueryBuilder<
fieldsToExclude.push("metadata");
}
- // Canonicalize by EVENTS_FIELDS declaration order so SELECT column order
- // does not follow caller field-set order. Clustered ClickHouse reads align
- // result blocks by position; incompatible Map types (cost_details vs
- // metadata) 500 when those columns swap places.
- const fieldsToProcess = [...this.selectFields]
- .filter((f) => !fieldsToExclude.includes(f))
- .sort(
- (a, b) =>
- (EVENTS_FIELD_ORDER_INDEX.get(a) ?? Number.MAX_SAFE_INTEGER) -
- (EVENTS_FIELD_ORDER_INDEX.get(b) ?? Number.MAX_SAFE_INTEGER),
- );
-
- const fieldExpressions: string[] = fieldsToProcess.flatMap((fieldKey) => {
- const fieldExpr = EVENTS_FIELDS[fieldKey as keyof typeof EVENTS_FIELDS];
- return fieldExpr ? [fieldExpr] : [];
- });
+ const fieldExpressions = eventsFieldSelectExpressions(
+ [...this.selectFields].filter((f) => !fieldsToExclude.includes(f)),
+ );
// Add I/O fields if configured
// Note: needsFullTable() is responsible for choosing events_core/events_full (truncated vs full I/O)
diff --git a/packages/shared/src/server/query-ast/README.md b/packages/shared/src/server/query-ast/README.md
index ccb9862838..150800d36b 100644
--- a/packages/shared/src/server/query-ast/README.md
+++ b/packages/shared/src/server/query-ast/README.md
@@ -40,6 +40,7 @@ SQL at the `repositories/clickhouse.ts` exec seam in test mode, then normalizes
it via `clickhouse format` + positional param names for snapshot comparison — no
ClickHouse server needed). Each migrated call site keeps its SQL baseline next to
that call site, e.g. `../repositories/environments.golden.test.ts`,
+`../repositories/events.golden.test.ts`,
`../repositories/scores.golden.test.ts`, and
`../queries/clickhouse-sql/event-filter-options.golden.test.ts`.
@@ -74,13 +75,16 @@ suffix to run there. Promote it to a required check once it has proven stable.
- **Compile only through `compileClickhouseQuery(query, ctx)`.** It is the one
supported path from a builder to `{ sql, params }`; the repository layer hands
- that to `queryClickhouse`. `ctx` (an `ExecutionContext` carrying `projectId`)
- is required — omitting it is a compile-time type error and an empty one throws.
+ that to `queryClickhouse`. `ctx` (an `ExecutionContext` carrying exactly one
+ `projectId`) is required — omitting it is a compile-time type error and
+ an empty one throws. There is no multi-project scope: a query reads one
+ project.
- **Never filter `project_id` yourself.** The compile step injects
`project_id = {projectId}` into every tenanted relation, so call sites pass
only `{ projectId }` (see `../repositories/environments.ts`).
- **Dedup is declared per table and must be an existing production idiom.**
- `events_core` is `none` (immutable at read time — no LIMIT BY, no FINAL).
+ `events_core` and `events_full` are `none` (immutable at read time — no
+ LIMIT BY, no FINAL).
`limitBy` is the legacy `ORDER BY <version> DESC LIMIT 1 BY <key>` already
used on traces / observations / scores. `$call(limitBy(...))` remains for
explicit non-version LIMIT BY. Auto-lowering does not declare scores as
@@ -188,8 +192,9 @@ is where tenancy is enforced:
both. It then identity-stamps the tree (`WeakSet`); a copied
`langfuseTenancy` property is not a valid stamp.
3. `DedupLoweringPlugin` applies the table's declared read idiom
- (`none` / `limitBy` / `final`). `events_core` is `none` — immutable at
- read time, so the pass does not inject LIMIT BY or FINAL. `limitBy` is
+ (`none` / `limitBy` / `final`). `events_core` and `events_full` are
+ `none` — immutable at read time, so the pass does not inject LIMIT BY or
+ FINAL. `limitBy` is
the existing legacy `ORDER BY <version> DESC LIMIT 1 BY <key>`.
`final` remains fail-closed on the auto-lowering pass; callers opt in
with `$call(useFinal(["scores"]))`. The pass restamps the rewritten root.
@@ -197,9 +202,9 @@ is where tenancy is enforced:
present, so `qb.compile()` without the plugin also fails. Value binds take
their ClickHouse type from the compared column's registry entry when one is
in scope (`total_cost > 1` → `{p:Float64}`).
-5. Raw-SQL table sources (`selectFrom(sql\`...\`)`) and raw fragments embedding a
- `SELECT`/`FROM`/`JOIN` in SELECT/WHERE throw `UnscopedRelationError`. Kysely's
- own keyword fragments (`asc`/`desc`) are not relations.
+5. Raw-SQL table sources (``selectFrom(sql`...`)``) and raw fragments embedding
+ a `SELECT`/`FROM`/`JOIN` in SELECT/WHERE throw `UnscopedRelationError`.
+ Kysely's own keyword fragments (`asc`/`desc`) are not relations.
So query bodies here never filter `project_id` by hand — it is redundant, and
forgetting it is impossible.
diff --git a/packages/shared/src/server/query-ast/binds.test.ts b/packages/shared/src/server/query-ast/binds.test.ts
index 9548a90b61..3be4215484 100644
--- a/packages/shared/src/server/query-ast/binds.test.ts
+++ b/packages/shared/src/server/query-ast/binds.test.ts
@@ -45,6 +45,57 @@ describe("typed params from the column registry", () => {
expect(sql).not.toMatch(/metadata_names in \(\{p\d+:Array\(String\)\}\)/);
});
+ it("keeps DateTime64(3) when comparing through toStartOfMinute / toDate", () => {
+ const start = "2026-01-01 00:00:00.000";
+ const { sql: minuteSql } = compile(
+ getClickhouseKysely()
+ .selectFrom("events_core")
+ .select("span_id")
+ .where((eb) =>
+ eb(
+ eb.fn("toStartOfMinute", ["start_time"]),
+ "=",
+ eb.fn("toStartOfMinute", [eb.val(start)]),
+ ),
+ ),
+ );
+ expect(minuteSql).toMatch(
+ /toStartOfMinute\(start_time\) = toStartOfMinute\(\{p\d+:DateTime64\(3\)\}\)/,
+ );
+
+ const { sql: dateSql } = compile(
+ getClickhouseKysely()
+ .selectFrom("traces as t")
+ .select("id")
+ .where((eb) =>
+ eb(
+ eb.fn("toDate", [eb.ref("t.timestamp")]),
+ "=",
+ eb.fn("toDate", [eb.val(start)]),
+ ),
+ ),
+ );
+ expect(dateSql).toMatch(
+ /toDate\(t\.timestamp\) = toDate\(\{p\d+:DateTime64\(3\)\}\)/,
+ );
+ });
+
+ it("does not type extra date-wrapper arguments as the column", () => {
+ const { sql } = compile(
+ getClickhouseKysely()
+ .selectFrom("events_core")
+ .select("span_id")
+ .where((eb) =>
+ eb(
+ eb.fn("toStartOfDay", ["start_time", eb.val("UTC")]),
+ ">=",
+ "2026-01-01 00:00:00.000",
+ ),
+ ),
+ );
+ expect(sql).toMatch(/toStartOfDay\(start_time, \{p\d+:String\}\)/);
+ });
+
it("does not type a subquery LIMIT from an outer column comparison", () => {
const { sql } = compile(
getClickhouseKysely()
diff --git a/packages/shared/src/server/query-ast/compiler.ts b/packages/shared/src/server/query-ast/compiler.ts
index 9f868df245..65d5121689 100644
--- a/packages/shared/src/server/query-ast/compiler.ts
+++ b/packages/shared/src/server/query-ast/compiler.ts
@@ -4,6 +4,7 @@ import {
CreateTableNode,
CreateViewNode,
DefaultQueryCompiler,
+ FunctionNode,
InsertQueryNode,
ParensNode,
ReferenceNode,
@@ -33,6 +34,16 @@ const ARRAY_JOIN_SQL: Record<ArrayJoinNode["variant"], string> = {
inner: "inner array join",
};
+const DATE_WRAPPER_FNS = new Set([
+ "toDate",
+ "toDateTime",
+ "toStartOfMinute",
+ "toStartOfHour",
+ "toStartOfDay",
+ "toStartOfMonth",
+ "toStartOfYear",
+]);
+
function canonicalParamValue(value: unknown): string {
if (value instanceof Date) return `d:${value.toISOString()}`;
if (Array.isArray(value)) {
@@ -385,5 +396,15 @@ function bindTypeOfColumnOperand(node: OperationNode): string | undefined {
if (ReferenceNode.is(node) && ColumnNode.is(node.column)) {
return COLUMN_BIND_TYPES[node.column.column.name];
}
+ // Single-argument date wrappers (`toDate(timestamp)`) keep the inner
+ // column's bind type so the compared value is still DateTime64(3). Extra
+ // arguments such as a timezone are not column values.
+ if (
+ FunctionNode.is(node) &&
+ DATE_WRAPPER_FNS.has(node.func) &&
+ node.arguments.length === 1
+ ) {
+ return bindTypeOfColumnOperand(node.arguments[0]);
+ }
return undefined;
}
diff --git a/packages/shared/src/server/query-ast/nodes.test.ts b/packages/shared/src/server/query-ast/nodes.test.ts
index 48d7ab3d5f..7c38839e39 100644
--- a/packages/shared/src/server/query-ast/nodes.test.ts
+++ b/packages/shared/src/server/query-ast/nodes.test.ts
@@ -46,6 +46,17 @@ describe("tenancy injection", () => {
);
});
+ it("has no multi-project scope", () => {
+ const qb = getClickhouseKysely()
+ .selectFrom("events_core")
+ .select("environment");
+
+ expect(() =>
+ // @ts-expect-error - a query is scoped to exactly one project
+ compileClickhouseQuery(qb, { projectIds: ["proj-a", "proj-b"] }),
+ ).toThrow(/no tenancy scope/);
+ });
+
it("refuses kysely.compile() when the tenancy pass did not run", () => {
const qb = getClickhouseKysely()
.selectFrom("traces")
diff --git a/packages/shared/src/server/query-ast/schema.test.ts b/packages/shared/src/server/query-ast/schema.test.ts
index 43c6be5231..bee42c43f1 100644
--- a/packages/shared/src/server/query-ast/schema.test.ts
+++ b/packages/shared/src/server/query-ast/schema.test.ts
@@ -10,14 +10,9 @@ import { schemaTypeAssertions, extensionTypeAssertions } from "./types.assert";
describe("table registry derivation", () => {
it("derives the tenanted table set from the registry", () => {
- // Only relations in the registry are scoped. `events_full` is just as
- // project-gated as `events_core` and would be `tenant: true` if modeled —
- // it is absent only because the builder targets `events_core` (the MV
- // projection) and never selects `events_full`. Since it is not in the row
- // type and raw sources are rejected, it is unreachable, so it needs no
- // entry. Add one (tenant defaults true) the day a query selects it.
expect([...TENANTED_TABLES].sort()).toEqual([
"events_core",
+ "events_full",
"observations",
"scores",
"traces",
@@ -34,8 +29,9 @@ describe("table registry derivation", () => {
expect(COLUMN_DATA_TYPES.metadata).toBe("map");
});
- it("declares events_core as immutable and leaves legacy tables undeclared", () => {
+ it("declares events_core and events_full as immutable and leaves legacy tables undeclared", () => {
expect(DEDUP_SPECS.events_core).toEqual({ strategy: "none" });
+ expect(DEDUP_SPECS.events_full).toEqual({ strategy: "none" });
expect(DEDUP_SPECS.traces).toBeUndefined();
expect(DEDUP_SPECS.observations).toBeUndefined();
expect(DEDUP_SPECS.scores).toBeUndefined();
diff --git a/packages/shared/src/server/query-ast/schema.ts b/packages/shared/src/server/query-ast/schema.ts
index ad4cc6585f..7a6878fe2d 100644
--- a/packages/shared/src/server/query-ast/schema.ts
+++ b/packages/shared/src/server/query-ast/schema.ts
@@ -102,6 +102,28 @@ function defineTable<const Cols extends Record<string, ChColumnType>>(spec: {
};
}
+const EVENTS_COLUMNS = {
+ environment: "String",
+ project_id: "String",
+ start_time: "DateTime",
+ span_id: "String",
+ trace_id: "String",
+ event_ts: "DateTime",
+ type: "String",
+ total_cost: "Float",
+ experiment_id: "String",
+ prompt_id: "String",
+ is_deleted: "UInt8",
+ metadata_names: "Array(String)",
+ metadata_values: "Array(String)",
+ user_id: "String",
+ session_id: "String",
+ name: "String",
+ parent_span_id: "String",
+ end_time: "DateTime",
+ tags: "Array(String)",
+} as const satisfies Record<string, ChColumnType>;
+
const TABLE_REGISTRY = {
traces: defineTable({
columns: {
@@ -122,24 +144,17 @@ const TABLE_REGISTRY = {
},
}),
events_core: defineTable({
- columns: {
- environment: "String",
- project_id: "String",
- start_time: "DateTime",
- span_id: "String",
- trace_id: "String",
- event_ts: "DateTime",
- type: "String",
- total_cost: "Float",
- experiment_id: "String",
- prompt_id: "String",
- is_deleted: "UInt8",
- metadata_names: "Array(String)",
- metadata_values: "Array(String)",
- },
+ columns: EVENTS_COLUMNS,
// Immutable at read time. Do not inject LIMIT BY or FINAL.
dedup: { strategy: "none" },
}),
+ // Same projection as events_core with full I/O. Tenanted and immutable
+ // for the same reasons; selected when a point-read needs untruncated
+ // input/output or metadata.
+ events_full: defineTable({
+ columns: EVENTS_COLUMNS,
+ dedup: { strategy: "none" },
+ }),
scores: defineTable({
columns: {
id: "String",
diff --git a/packages/shared/src/server/query-ast/tenancy.ts b/packages/shared/src/server/query-ast/tenancy.ts
index 599c572332..f393edd0ed 100644
--- a/packages/shared/src/server/query-ast/tenancy.ts
+++ b/packages/shared/src/server/query-ast/tenancy.ts
@@ -34,7 +34,8 @@ import { ClickHouseOperationNodeTransformer } from "./transformer";
/**
* Compile-time tenancy scope. Every ClickHouse query compiled through
* {@link compileClickhouseQuery} must carry one of these; the tenancy
- * injection pass keys off `projectId`.
+ * injection pass keys off `projectId`. A query is always scoped to exactly one
+ * project.
*/
export type ExecutionContext = {
projectId: string;
@@ -66,11 +67,7 @@ type Relation =
*/
export class TenancyInjectionPlugin implements KyselyPlugin {
constructor(private readonly ctx: ExecutionContext) {
- if (!ctx?.projectId) {
- throw new QueryCompileError(
- "ExecutionContext.projectId is required for tenancy injection",
- );
- }
+ requireExecutionContext(ctx);
}
transformQuery(args: PluginTransformQueryArgs): RootOperationNode {
@@ -149,10 +146,8 @@ function injectSelect(
let where = node.where;
for (const table of tenantedFrom) {
- if (
- !predicateCovers(where?.where, table, ctx.projectId, requireQualified)
- ) {
- const predicate = projectIdPredicate(table, ctx.projectId);
+ if (!predicateCovers(where?.where, table, ctx, requireQualified)) {
+ const predicate = projectIdPredicate(table, ctx);
// Prepend: the tenancy scope is the leading WHERE predicate, so its bound
// value keeps a stable (first) parameter position regardless of the other
// predicates a caller wrote.
@@ -168,12 +163,12 @@ function injectSelect(
}
const onExpr = join.on?.on;
if (
- predicateCovers(onExpr, relation, ctx.projectId, requireQualified) ||
- predicateCovers(where?.where, relation, ctx.projectId, requireQualified)
+ predicateCovers(onExpr, relation, ctx, requireQualified) ||
+ predicateCovers(where?.where, relation, ctx, requireQualified)
) {
return join;
}
- const predicate = projectIdPredicate(relation, ctx.projectId);
+ const predicate = projectIdPredicate(relation, ctx);
if (!join.on) {
return JoinNode.createWithOn(join.joinType, join.table, predicate);
}
@@ -228,18 +223,23 @@ function identifierName(node: OperationNode): string | undefined {
return undefined;
}
-function projectIdPredicate(
+function projectIdColumn(
table: Extract<Relation, { kind: "table" }>,
- projectId: string,
): OperationNode {
const column = ColumnNode.create(PROJECT_ID_COLUMN);
- const left = table.alias
+ return table.alias
? ReferenceNode.create(column, TableNode.create(table.alias))
: column;
+}
+
+function projectIdPredicate(
+ table: Extract<Relation, { kind: "table" }>,
+ ctx: ExecutionContext,
+): OperationNode {
return BinaryOperationNode.create(
- left,
+ projectIdColumn(table),
OperatorNode.create("="),
- ValueNode.create(projectId),
+ ValueNode.create(ctx.projectId),
);
}
@@ -255,32 +255,34 @@ function projectIdPredicate(
function predicateCovers(
expr: OperationNode | undefined,
table: Extract<Relation, { kind: "table" }>,
- projectId: string,
+ ctx: ExecutionContext,
requireQualified: boolean,
): boolean {
if (!expr) return false;
if (AndNode.is(expr)) {
return (
- predicateCovers(expr.left, table, projectId, requireQualified) ||
- predicateCovers(expr.right, table, projectId, requireQualified)
+ predicateCovers(expr.left, table, ctx, requireQualified) ||
+ predicateCovers(expr.right, table, ctx, requireQualified)
);
}
if (OrNode.is(expr)) {
return (
- predicateCovers(expr.left, table, projectId, requireQualified) &&
- predicateCovers(expr.right, table, projectId, requireQualified)
+ predicateCovers(expr.left, table, ctx, requireQualified) &&
+ predicateCovers(expr.right, table, ctx, requireQualified)
);
}
if (ParensNode.is(expr)) {
- return predicateCovers(expr.node, table, projectId, requireQualified);
+ return predicateCovers(expr.node, table, ctx, requireQualified);
+ }
+ if (!BinaryOperationNode.is(expr) || !OperatorNode.is(expr.operator)) {
+ return false;
}
- if (!BinaryOperationNode.is(expr)) return false;
- if (!OperatorNode.is(expr.operator) || expr.operator.operator !== "=") {
+ if (!isProjectIdColumn(expr.leftOperand, table, requireQualified)) {
return false;
}
return (
- isProjectIdColumn(expr.leftOperand, table, requireQualified) &&
- isProjectIdValue(expr.rightOperand, projectId)
+ expr.operator.operator === "=" &&
+ isProjectIdValue(expr.rightOperand, ctx.projectId)
);
}
diff --git a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap
new file mode 100644
index 0000000000..9834754530
--- /dev/null
+++ b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap
@@ -0,0 +1,872 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`golden: events point-reads & existence family > getAgentGraphDataFromEventsTable 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": "golden-trace",
+ "param3": "2026-01-01 00:00:00.000",
+ "param4": "2026-01-01 01:00:00.000",
+ },
+ "route": "unknown",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.parent_span_id AS parent_observation_id,
+ e.type AS type,
+ e.name AS name,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['langgraph_node'] AS node,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['langgraph_step'] AS step
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (e.trace_id = {param2:String}) AND (e.start_time >= {param3:DateTime64(3)}) AND (e.start_time <= {param4:DateTime64(3)})",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > getLastTraceTimestampsByProjectsFromEventsTable projectIds=empty 1`] = `[]`;
+
+exports[`golden: events point-reads & existence family > getLastTraceTimestampsByProjectsFromEventsTable projectIds=many 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": [
+ "golden-project",
+ "golden-project-b",
+ ],
+ },
+ "route": "unknown",
+ "sql": "SELECT
+ project_id,
+ max(start_time) AS last_trace_at
+FROM events_core
+WHERE (project_id IN ({param1:Array(String)})) AND (start_time >= (now() - toIntervalDay(30))) AND (is_deleted = 0)
+GROUP BY project_id",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > getLastTraceTimestampsByProjectsFromEventsTable projectIds=one 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": [
+ "golden-project",
+ ],
+ },
+ "route": "unknown",
+ "sql": "SELECT
+ project_id,
+ max(start_time) AS last_trace_at
+FROM events_core
+WHERE (project_id IN ({param1:Array(String)})) AND (start_time >= (now() - toIntervalDay(30))) AND (is_deleted = 0)
+GROUP BY project_id",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > getObservationByIdFromEventsTable base 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": "golden-span",
+ "param3": 1,
+ },
+ "route": "unknown",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.environment AS environment,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.name AS name,
+ e.level AS level,
+ e.status_message AS status_message,
+ e.version AS version,
+ e.bookmarked AS bookmarked,
+ e.public AS public,
+ e.user_id AS user_id,
+ e.session_id AS session_id,
+ ifNull(
+ COALESCE(
+ nullIf(e.trace_name, ''),
+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)
+ ),
+ ''
+ ) AS trace_name,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.completion_start_time AS completion_start_time,
+ e.created_at AS created_at,
+ e.updated_at AS updated_at,
+ e.event_ts,
+ e.provided_model_name AS provided_model_name,
+ e.model_id AS internal_model_id,
+ e.model_parameters AS model_parameters,
+ e.provided_usage_details AS provided_usage_details,
+ e.usage_details AS usage_details,
+ e.provided_cost_details AS provided_cost_details,
+ e.cost_details AS cost_details,
+ e.total_cost AS total_cost,
+ e.prompt_id AS prompt_id,
+ e.prompt_name AS prompt_name,
+ e.prompt_version AS prompt_version,
+ e.tool_definitions AS tool_definitions,
+ e.tool_calls AS tool_calls,
+ e.tool_call_names AS tool_call_names,
+ e.usage_pricing_tier_id AS usage_pricing_tier_id,
+ e.usage_pricing_tier_name AS usage_pricing_tier_name,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,
+ e.tags AS tags,
+ e.release AS release
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (span_id = {param2:String})
+ORDER BY
+ start_time DESC,
+ event_ts DESC
+LIMIT {param3:Int64}",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > getObservationByIdFromEventsTable io=full 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": "golden-span",
+ "param3": 1,
+ },
+ "route": "unknown",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.environment AS environment,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.name AS name,
+ e.level AS level,
+ e.status_message AS status_message,
+ e.version AS version,
+ e.bookmarked AS bookmarked,
+ e.public AS public,
+ e.user_id AS user_id,
+ e.session_id AS session_id,
+ ifNull(
+ COALESCE(
+ nullIf(e.trace_name, ''),
+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)
+ ),
+ ''
+ ) AS trace_name,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.completion_start_time AS completion_start_time,
+ e.created_at AS created_at,
+ e.updated_at AS updated_at,
+ e.event_ts,
+ e.provided_model_name AS provided_model_name,
+ e.model_id AS internal_model_id,
+ e.model_parameters AS model_parameters,
+ e.provided_usage_details AS provided_usage_details,
+ e.usage_details AS usage_details,
+ e.provided_cost_details AS provided_cost_details,
+ e.cost_details AS cost_details,
+ e.total_cost AS total_cost,
+ e.prompt_id AS prompt_id,
+ e.prompt_name AS prompt_name,
+ e.prompt_version AS prompt_version,
+ e.tool_definitions AS tool_definitions,
+ e.tool_calls AS tool_calls,
+ e.tool_call_names AS tool_call_names,
+ e.usage_pricing_tier_id AS usage_pricing_tier_id,
+ e.usage_pricing_tier_name AS usage_pricing_tier_name,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,
+ e.tags AS tags,
+ e.release AS release,
+ input,
+ output
+FROM events_full AS e
+WHERE (e.project_id = {param1:String}) AND (span_id = {param2:String})
+ORDER BY
+ start_time DESC,
+ event_ts DESC
+LIMIT {param3:Int64}",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > getObservationByIdFromEventsTable io=truncated 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": "golden-span",
+ "param3": 1,
+ },
+ "route": "unknown",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.environment AS environment,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.name AS name,
+ e.level AS level,
+ e.status_message AS status_message,
+ e.version AS version,
+ e.bookmarked AS bookmarked,
+ e.public AS public,
+ e.user_id AS user_id,
+ e.session_id AS session_id,
+ ifNull(
+ COALESCE(
+ nullIf(e.trace_name, ''),
+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)
+ ),
+ ''
+ ) AS trace_name,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.completion_start_time AS completion_start_time,
+ e.created_at AS created_at,
+ e.updated_at AS updated_at,
+ e.event_ts,
+ e.provided_model_name AS provided_model_name,
+ e.model_id AS internal_model_id,
+ e.model_parameters AS model_parameters,
+ e.provided_usage_details AS provided_usage_details,
+ e.usage_details AS usage_details,
+ e.provided_cost_details AS provided_cost_details,
+ e.cost_details AS cost_details,
+ e.total_cost AS total_cost,
+ e.prompt_id AS prompt_id,
+ e.prompt_name AS prompt_name,
+ e.prompt_version AS prompt_version,
+ e.tool_definitions AS tool_definitions,
+ e.tool_calls AS tool_calls,
+ e.tool_call_names AS tool_call_names,
+ e.usage_pricing_tier_id AS usage_pricing_tier_id,
+ e.usage_pricing_tier_name AS usage_pricing_tier_name,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,
+ e.tags AS tags,
+ e.release AS release,
+ leftUTF8(input, 1000) AS input,
+ leftUTF8(output, 1000) AS output
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (span_id = {param2:String})
+ORDER BY
+ start_time DESC,
+ event_ts DESC
+LIMIT {param3:Int64}",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > getObservationByIdFromEventsTable startTime+type+traceId 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": "golden-span",
+ "param3": "2026-01-01 00:00:00.000",
+ "param4": 2025-11-29T00:00:00.000Z,
+ "param5": "GENERATION",
+ "param6": "golden-trace",
+ "param7": 1,
+ },
+ "route": "unknown",
+ "sql": "SELECT
+ e.span_id AS id,
+ e.trace_id AS trace_id,
+ e.project_id AS project_id,
+ e.environment AS environment,
+ e.type AS type,
+ e.parent_span_id AS parent_observation_id,
+ e.name AS name,
+ e.level AS level,
+ e.status_message AS status_message,
+ e.version AS version,
+ e.bookmarked AS bookmarked,
+ e.public AS public,
+ e.user_id AS user_id,
+ e.session_id AS session_id,
+ ifNull(
+ COALESCE(
+ nullIf(e.trace_name, ''),
+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)
+ ),
+ ''
+ ) AS trace_name,
+ e.start_time AS start_time,
+ e.end_time AS end_time,
+ e.completion_start_time AS completion_start_time,
+ e.created_at AS created_at,
+ e.updated_at AS updated_at,
+ e.event_ts,
+ e.provided_model_name AS provided_model_name,
+ e.model_id AS internal_model_id,
+ e.model_parameters AS model_parameters,
+ e.provided_usage_details AS provided_usage_details,
+ e.usage_details AS usage_details,
+ e.provided_cost_details AS provided_cost_details,
+ e.cost_details AS cost_details,
+ e.total_cost AS total_cost,
+ e.prompt_id AS prompt_id,
+ e.prompt_name AS prompt_name,
+ e.prompt_version AS prompt_version,
+ e.tool_definitions AS tool_definitions,
+ e.tool_calls AS tool_calls,
+ e.tool_call_names AS tool_call_names,
+ e.usage_pricing_tier_id AS usage_pricing_tier_id,
+ e.usage_pricing_tier_name AS usage_pricing_tier_name,
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,
+ e.tags AS tags,
+ e.release AS release
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (span_id = {param2:String}) AND (toStartOfMinute(start_time) = toStartOfMinute({param3:DateTime64(3)})) AND (start_time >= {param4:DateTime64(3)}) AND (type = {param5:String}) AND (trace_id = {param6:String})
+ORDER BY
+ start_time DESC,
+ event_ts DESC
+LIMIT {param7:Int64}",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > getObservationsTraceIdsFromEventsTable 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": [
+ "golden-span",
+ "golden-span-b",
+ ],
+ },
+ "route": "unknown",
+ "sql": "SELECT
+ e.trace_id AS trace_id,
+ e.span_id AS span_id
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (e.span_id IN ({param2:Array(String)}))",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > getTraceByIdFromEventsTable base 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": [
+ "golden-trace",
+ ],
+ "param3": 1,
+ },
+ "route": "unknown",
+ "sql": "WITH traces AS
+ (
+ SELECT
+ trace_id AS id,
+ project_id,
+ COALESCE(
+ nullIf(argMaxIf(e.trace_name, e.event_ts, e.trace_name != ''), ''),
+ nullIf(
+ argMaxIf(e.name, e.event_ts, ((e.parent_span_id = '') OR (e.is_app_root = true)) AND (e.name != '')),
+ ''
+ )
+ ) AS name,
+ min(start_time) AS timestamp,
+ argMaxIf(environment, event_ts, environment != '') AS environment,
+ argMaxIf(version, event_ts, version != '') AS version,
+ argMaxIf(session_id, event_ts, session_id != '') AS session_id,
+ argMaxIf(user_id, event_ts, user_id != '') AS user_id,
+ argMaxIf(input, event_ts, parent_span_id = '') AS input,
+ argMaxIf(output, event_ts, parent_span_id = '') AS output,
+ argMaxIf(
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)),
+ event_ts,
+ parent_span_id = ''
+ ) AS metadata,
+ min(created_at) AS created_at,
+ max(updated_at) AS updated_at,
+ sum(total_cost) AS total_cost,
+ if(
+ max(end_time) IS NULL,
+ NULL,
+ dateDiff('millisecond', min(start_time), greatest(max(start_time), max(end_time)))
+ ) AS latency_milliseconds,
+ groupUniqArrayIf(span_id, span_id != '') AS observation_ids,
+ length(
+ groupUniqArrayIf(span_id, (span_id != '') AND (span_id != concat('t-', trace_id)))
+ ) AS observation_count,
+ argMaxIf(bookmarked, event_ts, parent_span_id = '') AS bookmarked,
+ max(public) AS public,
+ argMaxIf(experiment_item_id, event_ts, experiment_item_id != '') AS experiment_item_id,
+ sumMap(usage_details) AS usage_details,
+ sumMap(cost_details) AS cost_details,
+ multiIf(
+ arrayExists(x -> (x = 'ERROR'), groupArray(level)),
+ 'ERROR',
+ arrayExists(x -> (x = 'WARNING'), groupArray(level)),
+ 'WARNING',
+ arrayExists(x -> (x = 'DEFAULT'), groupArray(level)),
+ 'DEFAULT',
+ 'DEBUG'
+ ) AS aggregated_level,
+ countIf(level = 'WARNING') AS warning_count,
+ countIf(level = 'ERROR') AS error_count,
+ countIf(level = 'DEFAULT') AS default_count,
+ countIf(level = 'DEBUG') AS debug_count,
+ argMaxIf(tags, event_ts, notEmpty(tags)) AS tags,
+ argMaxIf(release, event_ts, release != '') AS release,
+ argMaxIf(evaluator_id, event_ts, evaluator_id != '') AS evaluator_id,
+ argMaxIf(evaluation_rule_id, event_ts, evaluation_rule_id != '') AS evaluation_rule_id,
+ any(experiment_id) AS experiment_id
+ FROM events_full AS e
+ WHERE (e.project_id = {param1:String}) AND (trace_id IN ({param2:Array(String)}))
+ GROUP BY
+ trace_id,
+ project_id
+ ORDER BY timestamp DESC
+ )
+SELECT
+ t.id,
+ t.name,
+ t.user_id,
+ t.release,
+ t.version,
+ t.project_id,
+ t.environment,
+ t.public,
+ t.bookmarked,
+ t.tags,
+ t.session_id,
+ t.timestamp,
+ t.created_at,
+ t.updated_at,
+ t.metadata,
+ 0 AS is_deleted,
+ t.input,
+ t.output
+FROM traces AS t
+ORDER BY t.timestamp DESC
+LIMIT {param3:Int64}",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > getTraceByIdFromEventsTable excludeInputOutput+excludeMetadata 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": [
+ "golden-trace",
+ ],
+ "param3": 1,
+ },
+ "route": "unknown",
+ "sql": "WITH traces AS
+ (
+ SELECT
+ trace_id AS id,
+ project_id,
+ COALESCE(
+ nullIf(argMaxIf(e.trace_name, e.event_ts, e.trace_name != ''), ''),
+ nullIf(
+ argMaxIf(e.name, e.event_ts, ((e.parent_span_id = '') OR (e.is_app_root = true)) AND (e.name != '')),
+ ''
+ )
+ ) AS name,
+ min(start_time) AS timestamp,
+ argMaxIf(environment, event_ts, environment != '') AS environment,
+ argMaxIf(version, event_ts, version != '') AS version,
+ argMaxIf(session_id, event_ts, session_id != '') AS session_id,
+ argMaxIf(user_id, event_ts, user_id != '') AS user_id,
+ argMaxIf(input, event_ts, parent_span_id = '') AS input,
+ argMaxIf(output, event_ts, parent_span_id = '') AS output,
+ argMaxIf(
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)),
+ event_ts,
+ parent_span_id = ''
+ ) AS metadata,
+ min(created_at) AS created_at,
+ max(updated_at) AS updated_at,
+ sum(total_cost) AS total_cost,
+ if(
+ max(end_time) IS NULL,
+ NULL,
+ dateDiff('millisecond', min(start_time), greatest(max(start_time), max(end_time)))
+ ) AS latency_milliseconds,
+ groupUniqArrayIf(span_id, span_id != '') AS observation_ids,
+ length(
+ groupUniqArrayIf(span_id, (span_id != '') AND (span_id != concat('t-', trace_id)))
+ ) AS observation_count,
+ argMaxIf(bookmarked, event_ts, parent_span_id = '') AS bookmarked,
+ max(public) AS public,
+ argMaxIf(experiment_item_id, event_ts, experiment_item_id != '') AS experiment_item_id,
+ sumMap(usage_details) AS usage_details,
+ sumMap(cost_details) AS cost_details,
+ multiIf(
+ arrayExists(x -> (x = 'ERROR'), groupArray(level)),
+ 'ERROR',
+ arrayExists(x -> (x = 'WARNING'), groupArray(level)),
+ 'WARNING',
+ arrayExists(x -> (x = 'DEFAULT'), groupArray(level)),
+ 'DEFAULT',
+ 'DEBUG'
+ ) AS aggregated_level,
+ countIf(level = 'WARNING') AS warning_count,
+ countIf(level = 'ERROR') AS error_count,
+ countIf(level = 'DEFAULT') AS default_count,
+ countIf(level = 'DEBUG') AS debug_count,
+ argMaxIf(tags, event_ts, notEmpty(tags)) AS tags,
+ argMaxIf(release, event_ts, release != '') AS release,
+ argMaxIf(evaluator_id, event_ts, evaluator_id != '') AS evaluator_id,
+ argMaxIf(evaluation_rule_id, event_ts, evaluation_rule_id != '') AS evaluation_rule_id,
+ any(experiment_id) AS experiment_id
+ FROM events_full AS e
+ WHERE (e.project_id = {param1:String}) AND (trace_id IN ({param2:Array(String)}))
+ GROUP BY
+ trace_id,
+ project_id
+ ORDER BY timestamp DESC
+ )
+SELECT
+ t.id,
+ t.name,
+ t.user_id,
+ t.release,
+ t.version,
+ t.project_id,
+ t.environment,
+ t.public,
+ t.bookmarked,
+ t.tags,
+ t.session_id,
+ t.timestamp,
+ t.created_at,
+ t.updated_at,
+ map() AS metadata,
+ 0 AS is_deleted,
+ '' AS input,
+ '' AS output
+FROM traces AS t
+ORDER BY t.timestamp DESC
+LIMIT {param3:Int64}",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > getTraceByIdFromEventsTable timestamp+fromTimestamp 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": [
+ "golden-trace",
+ ],
+ "param3": 2025-11-29T00:00:00.000Z,
+ "param4": "2026-01-01 00:00:00.000",
+ "param5": 1,
+ },
+ "route": "unknown",
+ "sql": "WITH traces AS
+ (
+ SELECT
+ trace_id AS id,
+ project_id,
+ COALESCE(
+ nullIf(argMaxIf(e.trace_name, e.event_ts, e.trace_name != ''), ''),
+ nullIf(
+ argMaxIf(e.name, e.event_ts, ((e.parent_span_id = '') OR (e.is_app_root = true)) AND (e.name != '')),
+ ''
+ )
+ ) AS name,
+ min(start_time) AS timestamp,
+ argMaxIf(environment, event_ts, environment != '') AS environment,
+ argMaxIf(version, event_ts, version != '') AS version,
+ argMaxIf(session_id, event_ts, session_id != '') AS session_id,
+ argMaxIf(user_id, event_ts, user_id != '') AS user_id,
+ argMaxIf(input, event_ts, parent_span_id = '') AS input,
+ argMaxIf(output, event_ts, parent_span_id = '') AS output,
+ argMaxIf(
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)),
+ event_ts,
+ parent_span_id = ''
+ ) AS metadata,
+ min(created_at) AS created_at,
+ max(updated_at) AS updated_at,
+ sum(total_cost) AS total_cost,
+ if(
+ max(end_time) IS NULL,
+ NULL,
+ dateDiff('millisecond', min(start_time), greatest(max(start_time), max(end_time)))
+ ) AS latency_milliseconds,
+ groupUniqArrayIf(span_id, span_id != '') AS observation_ids,
+ length(
+ groupUniqArrayIf(span_id, (span_id != '') AND (span_id != concat('t-', trace_id)))
+ ) AS observation_count,
+ argMaxIf(bookmarked, event_ts, parent_span_id = '') AS bookmarked,
+ max(public) AS public,
+ argMaxIf(experiment_item_id, event_ts, experiment_item_id != '') AS experiment_item_id,
+ sumMap(usage_details) AS usage_details,
+ sumMap(cost_details) AS cost_details,
+ multiIf(
+ arrayExists(x -> (x = 'ERROR'), groupArray(level)),
+ 'ERROR',
+ arrayExists(x -> (x = 'WARNING'), groupArray(level)),
+ 'WARNING',
+ arrayExists(x -> (x = 'DEFAULT'), groupArray(level)),
+ 'DEFAULT',
+ 'DEBUG'
+ ) AS aggregated_level,
+ countIf(level = 'WARNING') AS warning_count,
+ countIf(level = 'ERROR') AS error_count,
+ countIf(level = 'DEFAULT') AS default_count,
+ countIf(level = 'DEBUG') AS debug_count,
+ argMaxIf(tags, event_ts, notEmpty(tags)) AS tags,
+ argMaxIf(release, event_ts, release != '') AS release,
+ argMaxIf(evaluator_id, event_ts, evaluator_id != '') AS evaluator_id,
+ argMaxIf(evaluation_rule_id, event_ts, evaluation_rule_id != '') AS evaluation_rule_id,
+ any(experiment_id) AS experiment_id
+ FROM events_full AS e
+ WHERE (e.project_id = {param1:String}) AND (trace_id IN ({param2:Array(String)})) AND (start_time >= {param3:DateTime64(3)})
+ GROUP BY
+ trace_id,
+ project_id
+ ORDER BY timestamp DESC
+ )
+SELECT
+ t.id,
+ t.name,
+ t.user_id,
+ t.release,
+ t.version,
+ t.project_id,
+ t.environment,
+ t.public,
+ t.bookmarked,
+ t.tags,
+ t.session_id,
+ t.timestamp,
+ t.created_at,
+ t.updated_at,
+ t.metadata,
+ 0 AS is_deleted,
+ t.input,
+ t.output
+FROM traces AS t
+WHERE toDate(t.timestamp) = toDate({param4:DateTime64(3)})
+ORDER BY t.timestamp DESC
+LIMIT {param5:Int64}",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > getTraceByIdFromEventsTable truncated 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": [
+ "golden-trace",
+ ],
+ "param3": 1,
+ },
+ "route": "unknown",
+ "sql": "WITH traces AS
+ (
+ SELECT
+ trace_id AS id,
+ project_id,
+ COALESCE(
+ nullIf(argMaxIf(e.trace_name, e.event_ts, e.trace_name != ''), ''),
+ nullIf(
+ argMaxIf(e.name, e.event_ts, ((e.parent_span_id = '') OR (e.is_app_root = true)) AND (e.name != '')),
+ ''
+ )
+ ) AS name,
+ min(start_time) AS timestamp,
+ argMaxIf(environment, event_ts, environment != '') AS environment,
+ argMaxIf(version, event_ts, version != '') AS version,
+ argMaxIf(session_id, event_ts, session_id != '') AS session_id,
+ argMaxIf(user_id, event_ts, user_id != '') AS user_id,
+ argMaxIf(input, event_ts, parent_span_id = '') AS input,
+ argMaxIf(output, event_ts, parent_span_id = '') AS output,
+ argMaxIf(
+ mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)),
+ event_ts,
+ parent_span_id = ''
+ ) AS metadata,
+ min(created_at) AS created_at,
+ max(updated_at) AS updated_at,
+ sum(total_cost) AS total_cost,
+ if(
+ max(end_time) IS NULL,
+ NULL,
+ dateDiff('millisecond', min(start_time), greatest(max(start_time), max(end_time)))
+ ) AS latency_milliseconds,
+ groupUniqArrayIf(span_id, span_id != '') AS observation_ids,
+ length(
+ groupUniqArrayIf(span_id, (span_id != '') AND (span_id != concat('t-', trace_id)))
+ ) AS observation_count,
+ argMaxIf(bookmarked, event_ts, parent_span_id = '') AS bookmarked,
+ max(public) AS public,
+ argMaxIf(experiment_item_id, event_ts, experiment_item_id != '') AS experiment_item_id,
+ sumMap(usage_details) AS usage_details,
+ sumMap(cost_details) AS cost_details,
+ multiIf(
+ arrayExists(x -> (x = 'ERROR'), groupArray(level)),
+ 'ERROR',
+ arrayExists(x -> (x = 'WARNING'), groupArray(level)),
+ 'WARNING',
+ arrayExists(x -> (x = 'DEFAULT'), groupArray(level)),
+ 'DEFAULT',
+ 'DEBUG'
+ ) AS aggregated_level,
+ countIf(level = 'WARNING') AS warning_count,
+ countIf(level = 'ERROR') AS error_count,
+ countIf(level = 'DEFAULT') AS default_count,
+ countIf(level = 'DEBUG') AS debug_count,
+ argMaxIf(tags, event_ts, notEmpty(tags)) AS tags,
+ argMaxIf(release, event_ts, release != '') AS release,
+ argMaxIf(evaluator_id, event_ts, evaluator_id != '') AS evaluator_id,
+ argMaxIf(evaluation_rule_id, event_ts, evaluation_rule_id != '') AS evaluation_rule_id,
+ any(experiment_id) AS experiment_id
+ FROM events_core AS e
+ WHERE (e.project_id = {param1:String}) AND (trace_id IN ({param2:Array(String)}))
+ GROUP BY
+ trace_id,
+ project_id
+ ORDER BY timestamp DESC
+ )
+SELECT
+ t.id,
+ t.name,
+ t.user_id,
+ t.release,
+ t.version,
+ t.project_id,
+ t.environment,
+ t.public,
+ t.bookmarked,
+ t.tags,
+ t.session_id,
+ t.timestamp,
+ t.created_at,
+ t.updated_at,
+ t.metadata,
+ 0 AS is_deleted,
+ leftUTF8(t.input, 1000) AS input,
+ leftUTF8(t.output, 1000) AS output
+FROM traces AS t
+ORDER BY t.timestamp DESC
+LIMIT {param3:Int64}",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > getTraceMetadataByIdsFromEvents traceIds=empty 1`] = `[]`;
+
+exports[`golden: events point-reads & existence family > getTraceMetadataByIdsFromEvents traceIds=set 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": [
+ "golden-trace",
+ "golden-trace-b",
+ ],
+ },
+ "route": "unknown",
+ "sql": "SELECT
+ e.trace_id AS id,
+ COALESCE(
+ nullIf(e.trace_name, ''),
+ if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)
+ ) AS name,
+ e.user_id AS user_id,
+ e.tags AS tags
+FROM events_core AS e
+WHERE (e.project_id = {param1:String}) AND (COALESCE(nullIf(e.trace_name, ''), if((e.parent_span_id = '') OR (e.is_app_root = true), nullIf(e.name, ''), NULL)) IS NOT NULL) AND (e.is_deleted = {param2:UInt8}) AND (e.trace_id IN ({param3:Array(String)}))
+LIMIT 1 BY e.trace_id",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > hasAnySessionFromEventsTable 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": "",
+ "param3": 0,
+ "param4": 1,
+ },
+ "route": "unknown",
+ "sql": "SELECT 1 AS one
+FROM events_core
+WHERE (project_id = {param1:String}) AND (session_id IS NOT NULL) AND (session_id != {param2:String}) AND (is_deleted = {param3:UInt8})
+LIMIT {param4:Int64}",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > hasAnyTraceFromEventsTable 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": 0,
+ "param3": 1,
+ },
+ "route": "unknown",
+ "sql": "SELECT 1 AS one
+FROM events_core
+WHERE (project_id = {param1:String}) AND (is_deleted = {param2:UInt8})
+LIMIT {param3:Int64}",
+ },
+]
+`;
+
+exports[`golden: events point-reads & existence family > hasAnyUserFromEventsTable 1`] = `
+[
+ {
+ "fn": "queryClickhouse",
+ "params": {
+ "param1": "golden-project",
+ "param2": "",
+ "param3": 0,
+ "param4": 1,
+ },
+ "route": "unknown",
+ "sql": "SELECT 1 AS one
+FROM events_core
+WHERE (project_id = {param1:String}) AND (user_id IS NOT NULL) AND (user_id != {param2:String}) AND (is_deleted = {param3:UInt8})
+LIMIT {param4:Int64}",
+ },
+]
+`;
diff --git a/packages/shared/src/server/repositories/constants.ts b/packages/shared/src/server/repositories/constants.ts
index d8693c21b5..45372350fc 100644
--- a/packages/shared/src/server/repositories/constants.ts
+++ b/packages/shared/src/server/repositories/constants.ts
@@ -1,7 +1,8 @@
// Rule of thumb: If you join observations from left, use observations to trace and vice versa
// t.timestamp > observation.start_time - 2 days
-export const OBSERVATIONS_TO_TRACE_INTERVAL = "INTERVAL 2 DAY";
+export const OBSERVATIONS_TO_TRACE_INTERVAL_DAYS = 2;
+export const OBSERVATIONS_TO_TRACE_INTERVAL = `INTERVAL ${OBSERVATIONS_TO_TRACE_INTERVAL_DAYS} DAY`;
// observation.start_time > t.timestamp - 1 hour
export const TRACE_TO_OBSERVATIONS_INTERVAL = "INTERVAL 1 HOUR";
// observation.start_time > s.timestamp - 1 hour
diff --git a/packages/shared/src/server/repositories/events.golden.test.ts b/packages/shared/src/server/repositories/events.golden.test.ts
new file mode 100644
index 0000000000..a6c87f99c7
--- /dev/null
+++ b/packages/shared/src/server/repositories/events.golden.test.ts
@@ -0,0 +1,218 @@
+import { afterAll, beforeEach, describe, expect, it, vi } from "vitest";
+
+import { env } from "../../env";
+import { LangfuseNotFoundError } from "../../errors";
+import {
+ capturedQueries,
+ clickhouseFormatAvailable,
+ normalizeCapturedQueries,
+ resetCaptures,
+} from "./goldenHarness";
+
+// Record the exec seam instead of hitting ClickHouse. The factory is hoisted
+// above imports, so it pulls the harness in via dynamic import; the captured
+// store is a module singleton shared with the assertions below.
+vi.mock("./clickhouse", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("./clickhouse")>();
+ const { buildClickhouseMock } = await import("./goldenHarness.js");
+ return buildClickhouseMock(actual);
+});
+
+import {
+ getAgentGraphDataFromEventsTable,
+ getLastTraceTimestampsByProjectsFromEventsTable,
+ getObservationByIdFromEventsTable,
+ getObservationsTraceIdsFromEventsTable,
+ getTraceByIdFromEventsTable,
+ getTraceMetadataByIdsFromEvents,
+ hasAnySessionFromEventsTable,
+ hasAnyTraceFromEventsTable,
+ hasAnyUserFromEventsTable,
+} from "./events";
+
+const FIXED_PROJECT_ID = "golden-project";
+const FIXED_PROJECT_IDS = ["golden-project", "golden-project-b"] as const;
+const FIXED_TRACE_ID = "golden-trace";
+const FIXED_OBSERVATION_ID = "golden-span";
+const FIXED_START_TIME = new Date("2026-01-01T00:00:00.000Z");
+const FIXED_FROM_TIMESTAMP = new Date("2025-12-01T00:00:00.000Z");
+const FIXED_CH_MIN = "2026-01-01 00:00:00.000";
+const FIXED_CH_MAX = "2026-01-01 01:00:00.000";
+
+const describeWithClickhouse = clickhouseFormatAvailable()
+ ? describe
+ : describe.skip;
+
+if (!clickhouseFormatAvailable()) {
+ console.warn(
+ "[golden-harness] `clickhouse format` unavailable — skipping golden SQL tests. Install clickhouse-local to run them.",
+ );
+}
+
+async function captureObservationById(
+ args: Parameters<typeof getObservationByIdFromEventsTable>[0],
+) {
+ try {
+ await getObservationByIdFromEventsTable(args);
+ } catch (error) {
+ // The harness returns no rows; the wrapper throws after the exec seam.
+ if (!(error instanceof LangfuseNotFoundError)) throw error;
+ }
+}
+
+describeWithClickhouse("golden: events point-reads & existence family", () => {
+ const originalWriteMode = env.LANGFUSE_MIGRATION_V4_WRITE_MODE;
+
+ beforeEach(() => {
+ resetCaptures();
+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";
+ });
+ afterAll(() => {
+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = originalWriteMode;
+ });
+
+ it("hasAnyTraceFromEventsTable", async () => {
+ await hasAnyTraceFromEventsTable(FIXED_PROJECT_ID);
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("hasAnyUserFromEventsTable", async () => {
+ await hasAnyUserFromEventsTable(FIXED_PROJECT_ID);
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("hasAnySessionFromEventsTable", async () => {
+ await hasAnySessionFromEventsTable(FIXED_PROJECT_ID);
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getLastTraceTimestampsByProjectsFromEventsTable projectIds=empty", async () => {
+ await getLastTraceTimestampsByProjectsFromEventsTable({ projectIds: [] });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getLastTraceTimestampsByProjectsFromEventsTable projectIds=one", async () => {
+ await getLastTraceTimestampsByProjectsFromEventsTable({
+ projectIds: [FIXED_PROJECT_ID],
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getLastTraceTimestampsByProjectsFromEventsTable projectIds=many", async () => {
+ await getLastTraceTimestampsByProjectsFromEventsTable({
+ projectIds: [...FIXED_PROJECT_IDS],
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getAgentGraphDataFromEventsTable", async () => {
+ await getAgentGraphDataFromEventsTable({
+ projectId: FIXED_PROJECT_ID,
+ traceId: FIXED_TRACE_ID,
+ chMinStartTime: FIXED_CH_MIN,
+ chMaxStartTime: FIXED_CH_MAX,
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getObservationsTraceIdsFromEventsTable", async () => {
+ await getObservationsTraceIdsFromEventsTable({
+ projectId: FIXED_PROJECT_ID,
+ observationIds: [FIXED_OBSERVATION_ID, "golden-span-b"],
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getTraceMetadataByIdsFromEvents traceIds=empty", async () => {
+ await getTraceMetadataByIdsFromEvents({
+ projectId: FIXED_PROJECT_ID,
+ traceIds: [],
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getTraceMetadataByIdsFromEvents traceIds=set", async () => {
+ await getTraceMetadataByIdsFromEvents({
+ projectId: FIXED_PROJECT_ID,
+ traceIds: [FIXED_TRACE_ID, "golden-trace-b"],
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getObservationByIdFromEventsTable base", async () => {
+ await captureObservationById({
+ id: FIXED_OBSERVATION_ID,
+ projectId: FIXED_PROJECT_ID,
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getObservationByIdFromEventsTable startTime+type+traceId", async () => {
+ await captureObservationById({
+ id: FIXED_OBSERVATION_ID,
+ projectId: FIXED_PROJECT_ID,
+ startTime: FIXED_START_TIME,
+ startTimeLowerBound: FIXED_FROM_TIMESTAMP,
+ type: "GENERATION",
+ traceId: FIXED_TRACE_ID,
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getObservationByIdFromEventsTable io=truncated", async () => {
+ await captureObservationById({
+ id: FIXED_OBSERVATION_ID,
+ projectId: FIXED_PROJECT_ID,
+ fetchWithInputOutput: true,
+ renderingProps: { truncated: true, shouldJsonParse: true },
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getObservationByIdFromEventsTable io=full", async () => {
+ await captureObservationById({
+ id: FIXED_OBSERVATION_ID,
+ projectId: FIXED_PROJECT_ID,
+ fetchWithInputOutput: true,
+ renderingProps: { truncated: false, shouldJsonParse: true },
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getTraceByIdFromEventsTable base", async () => {
+ await getTraceByIdFromEventsTable({
+ traceId: FIXED_TRACE_ID,
+ projectId: FIXED_PROJECT_ID,
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getTraceByIdFromEventsTable timestamp+fromTimestamp", async () => {
+ await getTraceByIdFromEventsTable({
+ traceId: FIXED_TRACE_ID,
+ projectId: FIXED_PROJECT_ID,
+ timestamp: FIXED_START_TIME,
+ fromTimestamp: FIXED_FROM_TIMESTAMP,
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getTraceByIdFromEventsTable excludeInputOutput+excludeMetadata", async () => {
+ await getTraceByIdFromEventsTable({
+ traceId: FIXED_TRACE_ID,
+ projectId: FIXED_PROJECT_ID,
+ excludeInputOutput: true,
+ excludeMetadata: true,
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+
+ it("getTraceByIdFromEventsTable truncated", async () => {
+ await getTraceByIdFromEventsTable({
+ traceId: FIXED_TRACE_ID,
+ projectId: FIXED_PROJECT_ID,
+ renderingProps: { truncated: true, shouldJsonParse: true },
+ });
+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();
+ });
+});
diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.ts
index 7863054b4a..59ce450621 100644
--- a/packages/shared/src/server/repositories/events.ts
+++ b/packages/shared/src/server/repositories/events.ts
@@ -5,6 +5,7 @@ import {
type ClickHouseClientConfigOptions,
type ClickHouseSettings,
} from "@clickhouse/client";
+import { sql, type SqlBool } from "kysely";
import type {
EventsObservation,
MetadataDomain,
@@ -22,7 +23,12 @@ import {
} from "../clickhouse/client";
import { recordDistribution } from "../instrumentation";
import { logger } from "../logger";
-import { OBSERVATIONS_TO_TRACE_INTERVAL } from "./constants";
+import {
+ compileClickhouseQuery,
+ type ExecutionContext,
+} from "../query-ast/compile";
+import { getClickhouseKysely } from "../query-ast/dialect";
+import { limitBy } from "../query-ast/extensions";
import {
convertClickhouseToDomain,
convertClickhouseTracesListToDomain,
@@ -63,7 +69,6 @@ import type { EventsTableFilterState, FilterState } from "../../types";
import type { TracingSearchType } from "../../interfaces/search";
import {
eventsSessionsAggregation,
- eventsTraceMetadata,
eventsTracesAggregation,
eventsTracesScoresAggregation,
promptEventsForMetrics,
@@ -112,6 +117,8 @@ import {
CTEQueryBuilder,
EventsAggQueryBuilder,
buildEventsFullTableSplitQuery,
+ EVENTS_AGGREGATION_FIELDS,
+ eventsFieldSetSelectExpressions,
type QueryWithParams,
type SessionEventsMetricsRow,
OrderByEntry,
@@ -130,9 +137,11 @@ import {
import { type EventsObservationPublic } from "../queries/createGenerationsQuery";
import {
eventsTableCols,
+ eventsTableTraceNameSql,
normalizeEventsTraceName,
type NumericEventsTableColumnId,
} from "../../eventsTable";
+import { OBSERVATIONS_TO_TRACE_INTERVAL_DAYS } from "./constants";
import type { TraceDeleteBatchActionCursor } from "../../features/batchAction/types";
import {
findUiColumnMapping,
@@ -1016,6 +1025,14 @@ export const getObservationByIdFromEventsTable = async ({
return mapped.shift();
};
+const EVENTS_METADATA_MAP_SQL =
+ "mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))";
+
+/** ClickHouse DateTime64(3) binds stay strings; the column type is Date. */
+function dateTimeParam(value: string): Date {
+ return value as unknown as Date;
+}
+
async function getObservationByIdFromEventsTableInternal({
id,
projectId,
@@ -1037,51 +1054,53 @@ async function getObservationByIdFromEventsTableInternal({
renderingProps?: RenderingProps;
preferredClickhouseService?: PreferredClickhouseService;
}) {
- const queryBuilder = new EventsQueryBuilder({ projectId })
- .selectFieldSet("byIdBase", "byIdModel", "byIdPrompt", "byIdTimestamps")
- .when(fetchWithInputOutput, (b) =>
- b.selectIO(
- renderingProps.truncated,
- env.LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT,
+ // Minute-resolution start_time matches the events_full primary key
+ // (project_id, toStartOfMinute(start_time), ...) so the lookup can prune.
+ // The lower bound subtracts the observation-to-trace skew interval because
+ // an observation may start slightly before its anchor (e.g. the parent
+ // trace).
+ const db = getClickhouseKysely();
+ const ioSelects = observationIoSelects(
+ fetchWithInputOutput,
+ renderingProps.truncated,
+ );
+ const needsFullTable =
+ fetchWithInputOutput && renderingProps.truncated !== true;
+
+ const builder = (
+ needsFullTable
+ ? db.selectFrom("events_full as e")
+ : db.selectFrom("events_core as e")
+ )
+ .select([...observationByIdSelects(), ...ioSelects] as never)
+ .where("span_id", "=", id)
+ .$if(startTime != null, (qb) =>
+ qb.where((eb) =>
+ eb(
+ eb.fn("toStartOfMinute", ["start_time"]),
+ "=",
+ eb.fn("toStartOfMinute", [
+ eb.val(convertDateToClickhouseDateTime(startTime!)),
+ ]),
+ ),
),
)
- .whereRaw("span_id = {id: String}", { id })
- // Matched at minute resolution: minute is the finest the events_full primary
- // key (project_id, toStartOfMinute(start_time), ...) can prune on, and
- // flooring absorbs sub-minute precision differences in the caller-supplied
- // start time.
- .when(Boolean(startTime), (b) =>
- b.whereRaw(
- "toStartOfMinute(start_time) = toStartOfMinute({startTime: DateTime64(3)})",
- {
- startTime: convertDateToClickhouseDateTime(startTime!),
- },
+ .$if(startTimeLowerBound != null, (qb) =>
+ qb.where(
+ "start_time",
+ ">=",
+ observationsToTraceLowerBound(startTimeLowerBound!),
),
)
- // Lower-bound start_time on an anchor (e.g. the parent trace's timestamp) so
- // the lookup can prune events_full parts/partitions. Subtract the skew
- // interval because an observation may start slightly before its anchor.
- .when(Boolean(startTimeLowerBound), (b) =>
- b.whereRaw(
- `start_time >= {startTimeLowerBound: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}`,
- {
- startTimeLowerBound: convertDateToClickhouseDateTime(
- startTimeLowerBound!,
- ),
- },
- ),
- )
- .when(Boolean(type), (b) => b.whereRaw("type = {type: String}", { type }))
- .when(Boolean(traceId), (b) =>
- b.whereRaw("trace_id = {traceId: String}", { traceId }),
- )
- .orderByColumns([
- { column: "start_time", direction: "DESC" },
- { column: "event_ts", direction: "DESC" },
- ])
- .limit(1, 0);
+ .$if(type != null, (qb) => qb.where("type", "=", type!))
+ .$if(traceId != null, (qb) => qb.where("trace_id", "=", traceId!))
+ .orderBy("start_time", "desc")
+ .orderBy("event_ts", "desc")
+ .limit(1);
- const { query, params } = queryBuilder.buildWithParams();
+ const { sql: query, params } = compileClickhouseQuery(builder, {
+ projectId,
+ });
return await queryClickhouse<EventsObservationRecordReadType>({
query,
@@ -1091,6 +1110,36 @@ async function getObservationByIdFromEventsTableInternal({
});
}
+function observationIoSelects(
+ fetchWithInputOutput: boolean,
+ truncated: boolean | undefined,
+) {
+ if (!fetchWithInputOutput) return [];
+ if (truncated) {
+ const charLimit = env.LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT;
+ return [
+ sql.raw(`leftUTF8(input, ${charLimit})`).as("input"),
+ sql.raw(`leftUTF8(output, ${charLimit})`).as("output"),
+ ];
+ }
+ return [sql`input`, sql`output`];
+}
+
+/**
+ * Raw select expressions (`expr as alias`) taken from the legacy builder field
+ * maps, so the AST and the builders still serving other call sites cannot drift.
+ * Lazy: events.ts sits in an import cycle with the builder module, so the maps
+ * can still be undefined while this module initializes.
+ */
+function observationByIdSelects() {
+ return eventsFieldSetSelectExpressions(
+ "byIdBase",
+ "byIdModel",
+ "byIdPrompt",
+ "byIdTimestamps",
+ ).map((field) => sql.raw(field));
+}
+
/**
* Get a trace by ID from the events table.
* Compatible with getTraceById but queries the events table instead.
@@ -1119,68 +1168,72 @@ export const getTraceByIdFromEventsTable = async ({
/** When true, sets metadata column to empty in the query to reduce database load */
excludeMetadata?: boolean;
}) => {
- // Build traces CTE using eventsTracesAggregation
- // Pass truncated flag to select events_core (truncated) or events_full (full I/O)
- const tracesBuilder = eventsTracesAggregation({
- projectId,
- traceIds: [traceId],
- startTimeFrom: fromTimestamp
- ? convertDateToClickhouseDateTime(fromTimestamp)
- : null,
- truncated: renderingProps.truncated,
- });
-
- // Build the final query
- const queryBuilder = new CTEQueryBuilder()
- .withCTEFromBuilder("traces", tracesBuilder)
- .from("traces", "t")
- .selectColumns(
- "t.id",
- "t.name",
- "t.user_id",
- "t.release",
- "t.version",
- "t.project_id",
- "t.environment",
- "t.public",
- "t.bookmarked",
- "t.tags",
- "t.session_id",
- "t.timestamp",
- "t.created_at",
- "t.updated_at",
- )
- .select(excludeMetadata ? "map() as metadata" : "t.metadata")
- .select("0 as is_deleted");
-
- if (timestamp) {
- queryBuilder.whereRaw(
- `toDate(t.timestamp) = toDate({timestamp: DateTime64(3)})`,
- {
- timestamp: convertDateToClickhouseDateTime(timestamp),
- },
- );
- }
-
- // Handle input/output with truncation
- // Note: eventsTracesAggregation above is responsible for choosing events_core/events_full
- if (excludeInputOutput) {
- queryBuilder.select("'' as input").select("'' as output");
- } else if (renderingProps.truncated) {
- queryBuilder
- .select(
- `leftUTF8(t.input, ${env.LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT}) as input`,
+ const db = getClickhouseKysely();
+ const truncated = renderingProps.truncated === true;
+
+ const tracesCte = (qb: ReturnType<typeof getClickhouseKysely>) => {
+ const from = truncated
+ ? qb.selectFrom("events_core as e")
+ : qb.selectFrom("events_full as e");
+ return from
+ .select(traceAggregationSelects() as never)
+ .where("trace_id", "in", [traceId])
+ .$if(fromTimestamp != null, (inner) =>
+ inner.where(
+ "start_time",
+ ">=",
+ observationsToTraceLowerBound(fromTimestamp!),
+ ),
)
- .select(
- `leftUTF8(t.output, ${env.LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT}) as output`,
- );
- } else {
- queryBuilder.selectColumns("t.input", "t.output");
- }
+ .groupBy(["trace_id", "project_id"])
+ .orderBy(sql`timestamp`, "desc");
+ };
- queryBuilder.orderBy("ORDER BY t.timestamp DESC").limit(1);
+ const metadataSelect = excludeMetadata
+ ? sql`map()`.as("metadata")
+ : sql`t.metadata`;
- const { query, params } = queryBuilder.buildWithParams();
+ const builder = db
+ .with("traces", (qb) =>
+ tracesCte(qb as ReturnType<typeof getClickhouseKysely>),
+ )
+ .selectFrom("traces as t")
+ .select([
+ sql`t.id`,
+ sql`t.name`,
+ sql`t.user_id`,
+ sql`t.release`,
+ sql`t.version`,
+ sql`t.project_id`,
+ sql`t.environment`,
+ sql`t.public`,
+ sql`t.bookmarked`,
+ sql`t.tags`,
+ sql`t.session_id`,
+ sql`t.timestamp`,
+ sql`t.created_at`,
+ sql`t.updated_at`,
+ metadataSelect,
+ sql`0`.as("is_deleted"),
+ ...traceIoSelects(excludeInputOutput, truncated),
+ ])
+ .$if(timestamp != null, (qb) =>
+ qb.where((eb) =>
+ eb(
+ eb.fn("toDate", [eb.ref("t.timestamp")]),
+ "=",
+ eb.fn("toDate", [
+ eb.val(convertDateToClickhouseDateTime(timestamp!)),
+ ]),
+ ),
+ ),
+ )
+ .orderBy("t.timestamp", "desc")
+ .limit(1);
+
+ const { sql: query, params } = compileClickhouseQuery(builder, {
+ projectId,
+ });
const records = await queryClickhouse<TraceRecordReadType>({
query,
@@ -1206,6 +1259,32 @@ export const getTraceByIdFromEventsTable = async ({
return res.shift();
};
+function traceAggregationSelects() {
+ return Object.values(EVENTS_AGGREGATION_FIELDS).map((field) =>
+ sql.raw(field),
+ );
+}
+
+/** Earliest start_time an observation of a trace anchored at `anchor` can have. */
+function observationsToTraceLowerBound(anchor: Date): Date {
+ return new Date(
+ anchor.getTime() -
+ OBSERVATIONS_TO_TRACE_INTERVAL_DAYS * 24 * 60 * 60 * 1000,
+ );
+}
+
+function traceIoSelects(excludeInputOutput: boolean, truncated: boolean) {
+ if (excludeInputOutput) return [sql`''`.as("input"), sql`''`.as("output")];
+ if (truncated) {
+ const charLimit = env.LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT;
+ return [
+ sql.raw(`leftUTF8(t.input, ${charLimit})`).as("input"),
+ sql.raw(`leftUTF8(t.output, ${charLimit})`).as("output"),
+ ];
+ }
+ return [sql`t.input`, sql`t.output`];
+}
+
/**
* Routing wrapper for "trace by id" reads.
*
@@ -1270,17 +1349,14 @@ export const getTracesIdentifierForSession = async (
export const hasAnyTraceFromEventsTable = async (
projectId: string,
): Promise<boolean> => {
- const query = `
- SELECT 1
- FROM events_core
- WHERE project_id = {projectId: String}
- AND is_deleted = 0
- LIMIT 1
- `;
+ const { sql: query, params } = compileHasAnyFromEventsTable({
+ projectId,
+ kind: "trace",
+ });
- const rows = await queryClickhouse<{ 1: number }>({
+ const rows = await queryClickhouse<{ one: number }>({
query,
- params: { projectId },
+ params,
tags: { projectId },
preferredClickhouseService: "EventsReadOnly",
clickhouseSettings: {
@@ -1291,6 +1367,30 @@ export const hasAnyTraceFromEventsTable = async (
return rows.length > 0;
};
+type HasAnyEventsKind = "trace" | "user" | "session";
+
+function compileHasAnyFromEventsTable(opts: {
+ projectId: string;
+ kind: HasAnyEventsKind;
+}) {
+ const db = getClickhouseKysely();
+ const ctx: ExecutionContext = { projectId: opts.projectId };
+
+ const query = db
+ .selectFrom("events_core")
+ .select(sql<number>`1`.as("one"))
+ .$if(opts.kind === "user", (qb) =>
+ qb.where("user_id", "is not", null).where("user_id", "!=", ""),
+ )
+ .$if(opts.kind === "session", (qb) =>
+ qb.where("session_id", "is not", null).where("session_id", "!=", ""),
+ )
+ .where("is_deleted", "=", 0)
+ .limit(1);
+
+ return compileClickhouseQuery(query, ctx);
+}
+
/**
* Routing wrapper for the tracing onboarding gate ("has this project ingested
* any tracing data yet?").
@@ -1326,6 +1426,8 @@ export const getLastTraceTimestampsByProjectsFromEventsTable = async ({
}) => {
if (projectIds.length === 0) return [];
+ // Spans several projects, so it cannot go through compileClickhouseQuery,
+ // whose tenancy scope is always a single project.
const query = `
SELECT
project_id,
@@ -2512,27 +2614,31 @@ export async function getAgentGraphDataFromEventsTable(params: {
}) {
const { projectId, traceId, chMinStartTime, chMaxStartTime } = params;
- const query = `
- SELECT
- e.span_id as id,
- e.parent_span_id as parent_observation_id,
- e.type as type,
- e.name as name,
- e.start_time as start_time,
- e.end_time as end_time,
- mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['langgraph_node'] AS node,
- mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))['langgraph_step'] AS step
- FROM events_core e
- WHERE
- e.project_id = {projectId: String}
- AND e.trace_id = {traceId: String}
- AND e.start_time >= {chMinStartTime: DateTime64(3)}
- AND e.start_time <= {chMaxStartTime: DateTime64(3)}
- `;
+ const db = getClickhouseKysely();
+
+ const builder = db
+ .selectFrom("events_core as e")
+ .select([
+ "e.span_id as id",
+ "e.parent_span_id as parent_observation_id",
+ "e.type as type",
+ "e.name as name",
+ "e.start_time as start_time",
+ "e.end_time as end_time",
+ sql.raw(`${EVENTS_METADATA_MAP_SQL}['langgraph_node']`).as("node"),
+ sql.raw(`${EVENTS_METADATA_MAP_SQL}['langgraph_step']`).as("step"),
+ ])
+ .where("e.trace_id", "=", traceId)
+ .where("e.start_time", ">=", dateTimeParam(chMinStartTime))
+ .where("e.start_time", "<=", dateTimeParam(chMaxStartTime));
+
+ const { sql: query, params: queryParams } = compileClickhouseQuery(builder, {
+ projectId,
+ });
return queryClickhouse({
query,
- params: { projectId, traceId, chMinStartTime, chMaxStartTime },
+ params: queryParams,
tags: { projectId },
preferredClickhouseService: "EventsReadOnly",
});
@@ -2998,13 +3104,16 @@ export const getObservationsTraceIdsFromEventsTable = async (opts: {
}) => {
const { projectId, observationIds } = opts;
- const queryBuilder = new EventsQueryBuilder({ projectId })
- .selectRaw("e.trace_id AS trace_id", "e.span_id AS span_id")
- .whereRaw("e.span_id IN {observationIds: Array(String)}", {
- observationIds,
- });
+ const db = getClickhouseKysely();
- const { query, params: queryParams } = queryBuilder.buildWithParams();
+ const builder = db
+ .selectFrom("events_core as e")
+ .select(["e.trace_id as trace_id", "e.span_id as span_id"])
+ .where("e.span_id", "in", observationIds);
+
+ const { sql: query, params: queryParams } = compileClickhouseQuery(builder, {
+ projectId,
+ });
const result = await queryClickhouse<{
trace_id: string;
@@ -3217,20 +3326,14 @@ export const getUserMetricsFromEventsTable = async (
export const hasAnyUserFromEventsTable = async (
projectId: string,
): Promise<boolean> => {
- // Filter out deleted rows
- const query = `
- SELECT 1
- FROM events_core
- WHERE project_id = {projectId: String}
- AND user_id IS NOT NULL
- AND user_id != ''
- AND is_deleted = 0
- LIMIT 1
- `;
+ const { sql: query, params } = compileHasAnyFromEventsTable({
+ projectId,
+ kind: "user",
+ });
- const rows = await queryClickhouse<{ 1: number }>({
+ const rows = await queryClickhouse<{ one: number }>({
query,
- params: { projectId },
+ params,
tags: { projectId },
preferredClickhouseService: "EventsReadOnly",
});
@@ -3464,19 +3567,14 @@ export const getEventsForAnalyticsIntegrations = async function* (
export const hasAnySessionFromEventsTable = async (
projectId: string,
): Promise<boolean> => {
- const query = `
- SELECT 1
- FROM events_core
- WHERE project_id = {projectId: String}
- AND session_id IS NOT NULL
- AND session_id != ''
- AND is_deleted = 0
- LIMIT 1
- `;
+ const { sql: query, params } = compileHasAnyFromEventsTable({
+ projectId,
+ kind: "session",
+ });
- const rows = await queryClickhouse<{ 1: number }>({
+ const rows = await queryClickhouse<{ one: number }>({
query,
- params: { projectId },
+ params,
tags: { projectId },
preferredClickhouseService: "EventsReadOnly",
});
@@ -3495,12 +3593,24 @@ export const getTraceMetadataByIdsFromEvents = async (props: {
}) => {
if (props.traceIds.length === 0) return [];
- const builder = eventsTraceMetadata(props.projectId).whereRaw(
- "e.trace_id IN ({traceIds: Array(String)})",
- { traceIds: props.traceIds },
- );
+ const db = getClickhouseKysely();
- const { query, params } = builder.buildWithParams();
+ const builder = db
+ .selectFrom("events_core as e")
+ .select([
+ "e.trace_id as id",
+ sql.raw(eventsTableTraceNameSql).as("name"),
+ "e.user_id as user_id",
+ "e.tags as tags",
+ ])
+ .where(sql<SqlBool>`${sql.raw(eventsTableTraceNameSql)} IS NOT NULL`)
+ .where("e.is_deleted", "=", 0)
+ .where("e.trace_id", "in", props.traceIds)
+ .$call(limitBy({ count: 1, columns: ["e.trace_id"] }));
+
+ const { sql: query, params } = compileClickhouseQuery(builder, {
+ projectId: props.projectId,
+ });
return queryClickhouse<{
id: string;